feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
@@ -16,14 +16,25 @@ from config import (
    SITES_ENABLED,
    POKEMON_CENTER_URLS,
    TARGET_URLS,
+    GAMESTOP_URLS,
+    BESTBUY_URLS,
+    WALMART_URLS,
    KEYWORD_FILTER_ENABLED,
    KEYWORDS,
    LOG_LEVEL,
+    NEW_DROP_NOTIFICATIONS,
+    MAX_PAGES,
+)
+from src.browser import get_browser, shutdown_browser
+from src.product_tracker import ProductTracker
+from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
+from scrapers import (
+    PokemonCenterScraper,
+    TargetScraper,
+    GameStopScraper,
+    BestBuyScraper,
+    WalmartScraper,
 )
-from browser import get_browser, shutdown_browser
-from product_tracker import ProductTracker
-from discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
-from scrapers import PokemonCenterScraper, TargetScraper

 # Setup logging
 logging.basicConfig(
@@ -43,6 +54,9 @@ tracker = ProductTracker()
 scrapers = {
    "pokemoncenter": PokemonCenterScraper(),
    "target": TargetScraper(),
+    "gamestop": GameStopScraper(),
+    "bestbuy": BestBuyScraper(),
+    "walmart": WalmartScraper(),
 }


@@ -54,7 +68,7 @@ def check_pokemoncenter():
    try:
        for url in POKEMON_CENTER_URLS:
            logger.info(f"Scraping: {url}")
-            products = scraper.scrape_category_page(url)
+            products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1))

            # Apply keyword filter if enabled
            if KEYWORD_FILTER_ENABLED and KEYWORDS:
@@ -64,18 +78,19 @@ def check_pokemoncenter():
            # Process and detect changes
            new_products, restocked_products = tracker.process_products(products)

-            # Send notifications for new products
-            for product in new_products:
-                if product.in_stock:
-                    send_stock_alert(
-                        product_name=product.name,
-                        product_url=product.url,
-                        price=product.price or "See link",
-                        site="pokemoncenter",
-                        alert_type="new_drop",
-                        image_url=product.image_url,
-                    )
-                    logger.info(f"Sent notification for new product: {product.name}")
+            # Send notifications for new products (if enabled)
+            if NEW_DROP_NOTIFICATIONS:
+                for product in new_products:
+                    if product.in_stock:
+                        send_stock_alert(
+                            product_name=product.name,
+                            product_url=product.url,
+                            price=product.price or "See link",
+                            site="pokemoncenter",
+                            alert_type="new_drop",
+                            image_url=product.image_url,
+                        )
+                        logger.info(f"Sent notification for new product: {product.name}")

            # Send notifications for restocks
            for product in restocked_products:
@@ -110,7 +125,10 @@ def check_target():
    try:
        for url in TARGET_URLS:
            logger.info(f"Scraping: {url}")
-            products = scraper.scrape_category_page(url)
+            products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("target", 1))
+
+            # Always filter to Pokemon products only
+            products = scraper.filter_pokemon_products(products)

            # Apply keyword filter if enabled
            if KEYWORD_FILTER_ENABLED and KEYWORDS:
@@ -120,18 +138,19 @@ def check_target():
            # Process and detect changes
            new_products, restocked_products = tracker.process_products(products)

-            # Send notifications for new products
-            for product in new_products:
-                if product.in_stock:
-                    send_stock_alert(
-                        product_name=product.name,
-                        product_url=product.url,
-                        price=product.price or "See link",
-                        site="target",
-                        alert_type="new_drop",
-                        image_url=product.image_url,
-                    )
-                    logger.info(f"Sent notification for new product: {product.name}")
+            # Send notifications for new products (if enabled)
+            if NEW_DROP_NOTIFICATIONS:
+                for product in new_products:
+                    if product.in_stock:
+                        send_stock_alert(
+                            product_name=product.name,
+                            product_url=product.url,
+                            price=product.price or "See link",
+                            site="target",
+                            alert_type="new_drop",
+                            image_url=product.image_url,
+                        )
+                        logger.info(f"Sent notification for new product: {product.name}")

            # Send notifications for restocks
            for product in restocked_products:
@@ -158,6 +177,186 @@ def check_target():
        send_error_notification(f"Error checking Target: {str(e)}", "Target")


+def check_gamestop():
+    """Check GameStop for restocks and new drops"""
+    logger.info("Checking GameStop...")
+    scraper = scrapers["gamestop"]
+
+    try:
+        for url in GAMESTOP_URLS:
+            logger.info(f"Scraping: {url}")
+            products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("gamestop", 1))
+
+            # Always filter to Pokemon products only
+            products = scraper.filter_pokemon_products(products)
+
+            # Apply keyword filter if enabled
+            if KEYWORD_FILTER_ENABLED and KEYWORDS:
+                products = scraper.filter_by_keywords(products, KEYWORDS)
+                logger.info(f"After keyword filter: {len(products)} products")
+
+            # Process and detect changes
+            new_products, restocked_products = tracker.process_products(products)
+
+            # Send notifications for new products (if enabled)
+            if NEW_DROP_NOTIFICATIONS:
+                for product in new_products:
+                    if product.in_stock:
+                        send_stock_alert(
+                            product_name=product.name,
+                            product_url=product.url,
+                            price=product.price or "See link",
+                            site="gamestop",
+                            alert_type="new_drop",
+                            image_url=product.image_url,
+                        )
+                        logger.info(f"Sent notification for new product: {product.name}")
+
+            # Send notifications for restocks
+            for product in restocked_products:
+                send_stock_alert(
+                    product_name=product.name,
+                    product_url=product.url,
+                    price=product.price or "See link",
+                    site="gamestop",
+                    alert_type="restock",
+                    image_url=product.image_url,
+                )
+                logger.info(f"Sent notification for restock: {product.name}")
+
+            # Log stats
+            stats = tracker.get_stats()
+            logger.info(
+                f"Stats: {stats['total_products']} total, "
+                f"{stats['in_stock']} in stock, "
+                f"{stats['out_of_stock']} out of stock"
+            )
+
+    except Exception as e:
+        logger.error(f"Error checking GameStop: {e}")
+        send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
+
+
+def check_bestbuy():
+    """Check Best Buy for restocks and new drops"""
+    logger.info("Checking Best Buy...")
+    scraper = scrapers["bestbuy"]
+
+    try:
+        for url in BESTBUY_URLS:
+            logger.info(f"Scraping: {url}")
+            products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("bestbuy", 1))
+
+            # Always filter to Pokemon products only
+            products = scraper.filter_pokemon_products(products)
+
+            # Apply keyword filter if enabled
+            if KEYWORD_FILTER_ENABLED and KEYWORDS:
+                products = scraper.filter_by_keywords(products, KEYWORDS)
+                logger.info(f"After keyword filter: {len(products)} products")
+
+            # Process and detect changes
+            new_products, restocked_products = tracker.process_products(products)
+
+            # Send notifications for new products (if enabled)
+            if NEW_DROP_NOTIFICATIONS:
+                for product in new_products:
+                    if product.in_stock:
+                        send_stock_alert(
+                            product_name=product.name,
+                            product_url=product.url,
+                            price=product.price or "See link",
+                            site="bestbuy",
+                            alert_type="new_drop",
+                            image_url=product.image_url,
+                        )
+                        logger.info(f"Sent notification for new product: {product.name}")
+
+            # Send notifications for restocks
+            for product in restocked_products:
+                send_stock_alert(
+                    product_name=product.name,
+                    product_url=product.url,
+                    price=product.price or "See link",
+                    site="bestbuy",
+                    alert_type="restock",
+                    image_url=product.image_url,
+                )
+                logger.info(f"Sent notification for restock: {product.name}")
+
+            # Log stats
+            stats = tracker.get_stats()
+            logger.info(
+                f"Stats: {stats['total_products']} total, "
+                f"{stats['in_stock']} in stock, "
+                f"{stats['out_of_stock']} out of stock"
+            )
+
+    except Exception as e:
+        logger.error(f"Error checking Best Buy: {e}")
+        send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
+
+
+def check_walmart():
+    """Check Walmart for restocks and new drops"""
+    logger.info("Checking Walmart...")
+    scraper = scrapers["walmart"]
+
+    try:
+        for url in WALMART_URLS:
+            logger.info(f"Scraping: {url}")
+            products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("walmart", 1))
+
+            # Always filter to Pokemon products only
+            products = scraper.filter_pokemon_products(products)
+
+            # Apply keyword filter if enabled
+            if KEYWORD_FILTER_ENABLED and KEYWORDS:
+                products = scraper.filter_by_keywords(products, KEYWORDS)
+                logger.info(f"After keyword filter: {len(products)} products")
+
+            # Process and detect changes
+            new_products, restocked_products = tracker.process_products(products)
+
+            # Send notifications for new products (if enabled)
+            if NEW_DROP_NOTIFICATIONS:
+                for product in new_products:
+                    if product.in_stock:
+                        send_stock_alert(
+                            product_name=product.name,
+                            product_url=product.url,
+                            price=product.price or "See link",
+                            site="walmart",
+                            alert_type="new_drop",
+                            image_url=product.image_url,
+                        )
+                        logger.info(f"Sent notification for new product: {product.name}")
+
+            # Send notifications for restocks
+            for product in restocked_products:
+                send_stock_alert(
+                    product_name=product.name,
+                    product_url=product.url,
+                    price=product.price or "See link",
+                    site="walmart",
+                    alert_type="restock",
+                    image_url=product.image_url,
+                )
+                logger.info(f"Sent notification for restock: {product.name}")
+
+            # Log stats
+            stats = tracker.get_stats()
+            logger.info(
+                f"Stats: {stats['total_products']} total, "
+                f"{stats['in_stock']} in stock, "
+                f"{stats['out_of_stock']} out of stock"
+            )
+
+    except Exception as e:
+        logger.error(f"Error checking Walmart: {e}")
+        send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
+
+
 def run_checks():
    """Run all enabled site checks"""
    logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
@@ -168,6 +367,15 @@ def run_checks():
    if "target" in SITES_ENABLED:
        check_target()

+    if "gamestop" in SITES_ENABLED:
+        check_gamestop()
+
+    if "bestbuy" in SITES_ENABLED:
+        check_bestbuy()
+
+    if "walmart" in SITES_ENABLED:
+        check_walmart()
+
    logger.info("Check cycle complete")