mmcghen
8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
120 lines
3.3 KiB
Python
120 lines
3.3 KiB
Python
"""
|
|
Base scraper class with common functionality
|
|
"""
|
|
|
|
import logging
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import List, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class Product:
|
|
"""Represents a product listing"""
|
|
name: str
|
|
url: str
|
|
price: Optional[str]
|
|
in_stock: bool
|
|
image_url: Optional[str] = None
|
|
site: str = ""
|
|
product_id: Optional[str] = None # Unique identifier for tracking
|
|
|
|
def __hash__(self):
|
|
return hash(self.url)
|
|
|
|
def __eq__(self, other):
|
|
if isinstance(other, Product):
|
|
return self.url == other.url
|
|
return False
|
|
|
|
|
|
class BaseScraper(ABC):
|
|
"""Base class for site-specific scrapers"""
|
|
|
|
site_name: str = "unknown"
|
|
|
|
# Terms that indicate a Pokemon product
|
|
POKEMON_TERMS = [
|
|
"pokemon", "pokémon", "poke", "tcg",
|
|
"pikachu", "charizard", "mewtwo", "eevee", "snorlax",
|
|
"booster", "elite trainer", "etb",
|
|
"scarlet", "violet", "prismatic", "evolutions",
|
|
]
|
|
|
|
# Terms that indicate NOT a Pokemon product (false positives from search)
|
|
EXCLUDE_TERMS = [
|
|
"ice cube", "oven", "barbie", "hot wheels", "lego",
|
|
"furniture", "appliance", "kitchen", "bedding",
|
|
"glitter girls", "masters of the universe", "transformers",
|
|
"room essentials", "threshold",
|
|
]
|
|
|
|
def is_pokemon_product(self, product: Product) -> bool:
|
|
"""
|
|
Check if a product is actually a Pokemon product.
|
|
Filters out false positives from search results.
|
|
"""
|
|
name_lower = product.name.lower()
|
|
|
|
# Check for exclusion terms first
|
|
for term in self.EXCLUDE_TERMS:
|
|
if term in name_lower:
|
|
return False
|
|
|
|
# Check for Pokemon terms
|
|
for term in self.POKEMON_TERMS:
|
|
if term in name_lower:
|
|
return True
|
|
|
|
# If no Pokemon terms found, reject it
|
|
return False
|
|
|
|
def filter_pokemon_products(self, products: List[Product]) -> List[Product]:
|
|
"""Filter to only include valid Pokemon products"""
|
|
filtered = [p for p in products if self.is_pokemon_product(p)]
|
|
rejected = len(products) - len(filtered)
|
|
if rejected > 0:
|
|
logger.info(f"Filtered out {rejected} non-Pokemon products")
|
|
return filtered
|
|
|
|
@abstractmethod
|
|
def scrape_category_page(self, url: str) -> List[Product]:
|
|
"""
|
|
Scrape a category/search page and return all products found
|
|
|
|
Args:
|
|
url: URL of the category page
|
|
|
|
Returns:
|
|
List of Product objects
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
|
"""
|
|
Check if a specific product is in stock
|
|
|
|
Args:
|
|
product_url: URL of the product page
|
|
|
|
Returns:
|
|
Tuple of (is_in_stock, price)
|
|
"""
|
|
pass
|
|
|
|
def filter_by_keywords(self, products: List[Product], keywords: List[str]) -> List[Product]:
|
|
"""Filter products by keywords in name"""
|
|
if not keywords:
|
|
return products
|
|
|
|
filtered = []
|
|
for product in products:
|
|
name_lower = product.name.lower()
|
|
if any(kw.lower() in name_lower for kw in keywords):
|
|
filtered.append(product)
|
|
|
|
return filtered
|