""" Base scraper class with common functionality """ import logging from abc import ABC, abstractmethod from dataclasses import dataclass from typing import List, Optional logger = logging.getLogger(__name__) @dataclass class Product: """Represents a product listing""" name: str url: str price: Optional[str] in_stock: bool image_url: Optional[str] = None site: str = "" product_id: Optional[str] = None # Unique identifier for tracking def __hash__(self): return hash(self.url) def __eq__(self, other): if isinstance(other, Product): return self.url == other.url return False class BaseScraper(ABC): """Base class for site-specific scrapers""" site_name: str = "unknown" # Terms that indicate a Pokemon product POKEMON_TERMS = [ "pokemon", "pokémon", "poke", "tcg", "pikachu", "charizard", "mewtwo", "eevee", "snorlax", "booster", "elite trainer", "etb", "scarlet", "violet", "prismatic", "evolutions", ] # Terms that indicate NOT a Pokemon product (false positives from search) EXCLUDE_TERMS = [ "ice cube", "oven", "barbie", "hot wheels", "lego", "furniture", "appliance", "kitchen", "bedding", "glitter girls", "masters of the universe", "transformers", "room essentials", "threshold", ] def is_pokemon_product(self, product: Product) -> bool: """ Check if a product is actually a Pokemon product. Filters out false positives from search results. """ name_lower = product.name.lower() # Check for exclusion terms first for term in self.EXCLUDE_TERMS: if term in name_lower: return False # Check for Pokemon terms for term in self.POKEMON_TERMS: if term in name_lower: return True # If no Pokemon terms found, reject it return False def filter_pokemon_products(self, products: List[Product]) -> List[Product]: """Filter to only include valid Pokemon products""" filtered = [p for p in products if self.is_pokemon_product(p)] rejected = len(products) - len(filtered) if rejected > 0: logger.info(f"Filtered out {rejected} non-Pokemon products") return filtered @abstractmethod def scrape_category_page(self, url: str) -> List[Product]: """ Scrape a category/search page and return all products found Args: url: URL of the category page Returns: List of Product objects """ pass @abstractmethod def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]: """ Check if a specific product is in stock Args: product_url: URL of the product page Returns: Tuple of (is_in_stock, price) """ pass def filter_by_keywords(self, products: List[Product], keywords: List[str]) -> List[Product]: """Filter products by keywords in name""" if not keywords: return products filtered = [] for product in products: name_lower = product.name.lower() if any(kw.lower() in name_lower for kw in keywords): filtered.append(product) return filtered