cleveragents-core/simulations/sim4/scraper.py

#!/usr/bin/env python3
"""
Hacker News Scraper

A web scraper that fetches stories from the Hacker News front page
and outputs them in JSON format.
"""

import json
import logging
import re
import sys
from dataclasses import dataclass, asdict
from typing import List, Optional
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Constants
HACKER_NEWS_URL = "https://news.ycombinator.com/"
REQUEST_TIMEOUT = 10
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"


@dataclass
class Story:
    """Represents a Hacker News story with metadata."""
    title: str
    url: str
    points: int
    comments: int

    def to_dict(self) -> dict:
        """Convert Story to dictionary for JSON serialization."""
        return asdict(self)


class HackerNewsScraper:
    """Scraper for Hacker News front page stories."""
    
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': USER_AGENT})
    
    def fetch_page(self, url: str) -> Optional[str]:
        """
        Fetch HTML content from the given URL.
        
        Args:
            url: The URL to fetch
            
        Returns:
            HTML content as string, or None if request failed
        """
        try:
            logger.info(f"Fetching page: {url}")
            response = self.session.get(url, timeout=REQUEST_TIMEOUT)
            response.raise_for_status()
            return response.text
        except requests.exceptions.RequestException as e:
            logger.error(f"Failed to fetch {url}: {e}")
            return None
    
    def extract_number(self, text: str) -> int:
        """
        Extract numeric value from text string.
        
        Args:
            text: Text containing a number
            
        Returns:
            Extracted number, or 0 if no number found
        """
        if not text:
            return 0
        
        match = re.search(r'(\d+)', text.strip())
        return int(match.group(1)) if match else 0
    
    def parse_story_row(self, title_row, subtext_row) -> Optional[Story]:
        """
        Parse a story from title row and subtext row elements.
        
        Args:
            title_row: BeautifulSoup element containing title and URL
            subtext_row: BeautifulSoup element containing points and comments
            
        Returns:
            Story object or None if parsing failed
        """
        try:
            # Extract title and URL
            title_link = title_row.find('span', class_='titleline')
            if not title_link:
                return None
                
            link_elem = title_link.find('a')
            if not link_elem:
                return None
            
            title = link_elem.get_text(strip=True)
            url = link_elem.get('href', '')
            
            # Handle relative URLs
            if url.startswith('item?'):
                url = urljoin(HACKER_NEWS_URL, url)
            elif not urlparse(url).netloc:
                url = urljoin(HACKER_NEWS_URL, url)
            
            # Extract points
            points = 0
            score_elem = subtext_row.find('span', class_='score')
            if score_elem:
                points = self.extract_number(score_elem.get_text())
            
            # Extract comments count
            comments = 0
            comment_links = subtext_row.find_all('a')
            for link in comment_links:
                link_text = link.get_text(strip=True)
                if 'comment' in link_text.lower():
                    comments = self.extract_number(link_text)
                    break
            
            return Story(
                title=title,
                url=url,
                points=points,
                comments=comments
            )
        
        except Exception as e:
            logger.warning(f"Failed to parse story row: {e}")
            return None
    
    def parse_stories(self, html: str) -> List[Story]:
        """
        Parse stories from Hacker News HTML content.
        
        Args:
            html: HTML content string
            
        Returns:
            List of Story objects
        """
        try:
            soup = BeautifulSoup(html, 'lxml')
            stories = []
            
            # Find the main table containing stories
            main_table = soup.find('table', id='hnmain')
            if not main_table:
                logger.error("Could not find main stories table")
                return stories
            
            # Find all story rows (they have class 'athing')
            story_rows = main_table.find_all('tr', class_='athing')
            
            for story_row in story_rows:
                # The subtext row immediately follows the story row
                subtext_row = story_row.find_next_sibling('tr')
                
                if subtext_row and subtext_row.find('td', class_='subtext'):
                    story = self.parse_story_row(story_row, subtext_row)
                    if story:
                        stories.append(story)
                        logger.debug(f"Parsed story: {story.title}")
            
            logger.info(f"Successfully parsed {len(stories)} stories")
            return stories
        
        except Exception as e:
            logger.error(f"Failed to parse HTML: {e}")
            return []
    
    def scrape_front_page(self) -> List[Story]:
        """
        Scrape stories from Hacker News front page.
        
        Returns:
            List of Story objects
        """
        html = self.fetch_page(HACKER_NEWS_URL)
        if not html:
            logger.error("Failed to fetch Hacker News front page")
            return []
        
        return self.parse_stories(html)
    
    def close(self):
        """Close the requests session."""
        self.session.close()


def stories_to_json(stories: List[Story]) -> str:
    """
    Convert list of stories to JSON string.
    
    Args:
        stories: List of Story objects
        
    Returns:
        JSON string representation
    """
    try:
        stories_dict = [story.to_dict() for story in stories]
        return json.dumps(stories_dict, indent=2, ensure_ascii=False)
    except Exception as e:
        logger.error(f"Failed to serialize stories to JSON: {e}")
        return "[]"


def main():
    """Main function to run the scraper."""
    scraper = HackerNewsScraper()
    
    try:
        # Scrape stories
        stories = scraper.scrape_front_page()
        
        if not stories:
            logger.warning("No stories were scraped")
            print("[]")
            sys.exit(1)
        
        # Output as JSON
        json_output = stories_to_json(stories)
        print(json_output)
        
        logger.info(f"Successfully scraped and output {len(stories)} stories")
        
    except KeyboardInterrupt:
        logger.info("Scraping interrupted by user")
        sys.exit(1)
    except Exception as e:
        logger.error(f"Unexpected error: {e}")
        sys.exit(1)
    finally:
        scraper.close()


if __name__ == "__main__":
    main()