Files
pokemon-stock-checker/main.py
T
mmcghen 8d382e723f feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
2026-03-27 23:08:09 -04:00

441 lines
15 KiB
Python

"""
Pokemon Stock Monitor - Main entry point
Monitors retail sites for Pokemon card restocks and new drops
"""
import logging
import time
import signal
import sys
from datetime import datetime
import schedule
from config import (
CHECK_INTERVAL_SECONDS,
SITES_ENABLED,
POKEMON_CENTER_URLS,
TARGET_URLS,
GAMESTOP_URLS,
BESTBUY_URLS,
WALMART_URLS,
KEYWORD_FILTER_ENABLED,
KEYWORDS,
LOG_LEVEL,
NEW_DROP_NOTIFICATIONS,
MAX_PAGES,
)
from src.browser import get_browser, shutdown_browser
from src.product_tracker import ProductTracker
from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
from scrapers import (
PokemonCenterScraper,
TargetScraper,
GameStopScraper,
BestBuyScraper,
WalmartScraper,
)
# Setup logging
logging.basicConfig(
level=getattr(logging, LOG_LEVEL),
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
handlers=[
logging.StreamHandler(),
logging.FileHandler("monitor.log", encoding="utf-8"),
],
)
logger = logging.getLogger(__name__)
# Global tracker
tracker = ProductTracker()
# Scrapers
scrapers = {
"pokemoncenter": PokemonCenterScraper(),
"target": TargetScraper(),
"gamestop": GameStopScraper(),
"bestbuy": BestBuyScraper(),
"walmart": WalmartScraper(),
}
def check_pokemoncenter():
"""Check PokemonCenter for restocks and new drops"""
logger.info("Checking PokemonCenter...")
scraper = scrapers["pokemoncenter"]
try:
for url in POKEMON_CENTER_URLS:
logger.info(f"Scraping: {url}")
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1))
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
products = scraper.filter_by_keywords(products, KEYWORDS)
logger.info(f"After keyword filter: {len(products)} products")
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
# Send notifications for new products (if enabled)
if NEW_DROP_NOTIFICATIONS:
for product in new_products:
if product.in_stock:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="pokemoncenter",
alert_type="new_drop",
image_url=product.image_url,
)
logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="pokemoncenter",
alert_type="restock",
image_url=product.image_url,
)
logger.info(f"Sent notification for restock: {product.name}")
# Log stats
stats = tracker.get_stats()
logger.info(
f"Stats: {stats['total_products']} total, "
f"{stats['in_stock']} in stock, "
f"{stats['out_of_stock']} out of stock"
)
except Exception as e:
logger.error(f"Error checking PokemonCenter: {e}")
send_error_notification(f"Error checking PokemonCenter: {str(e)}", "PokemonCenter")
def check_target():
"""Check Target for restocks and new drops"""
logger.info("Checking Target...")
scraper = scrapers["target"]
try:
for url in TARGET_URLS:
logger.info(f"Scraping: {url}")
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("target", 1))
# Always filter to Pokemon products only
products = scraper.filter_pokemon_products(products)
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
products = scraper.filter_by_keywords(products, KEYWORDS)
logger.info(f"After keyword filter: {len(products)} products")
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
# Send notifications for new products (if enabled)
if NEW_DROP_NOTIFICATIONS:
for product in new_products:
if product.in_stock:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="target",
alert_type="new_drop",
image_url=product.image_url,
)
logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="target",
alert_type="restock",
image_url=product.image_url,
)
logger.info(f"Sent notification for restock: {product.name}")
# Log stats
stats = tracker.get_stats()
logger.info(
f"Stats: {stats['total_products']} total, "
f"{stats['in_stock']} in stock, "
f"{stats['out_of_stock']} out of stock"
)
except Exception as e:
logger.error(f"Error checking Target: {e}")
send_error_notification(f"Error checking Target: {str(e)}", "Target")
def check_gamestop():
"""Check GameStop for restocks and new drops"""
logger.info("Checking GameStop...")
scraper = scrapers["gamestop"]
try:
for url in GAMESTOP_URLS:
logger.info(f"Scraping: {url}")
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("gamestop", 1))
# Always filter to Pokemon products only
products = scraper.filter_pokemon_products(products)
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
products = scraper.filter_by_keywords(products, KEYWORDS)
logger.info(f"After keyword filter: {len(products)} products")
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
# Send notifications for new products (if enabled)
if NEW_DROP_NOTIFICATIONS:
for product in new_products:
if product.in_stock:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="gamestop",
alert_type="new_drop",
image_url=product.image_url,
)
logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="gamestop",
alert_type="restock",
image_url=product.image_url,
)
logger.info(f"Sent notification for restock: {product.name}")
# Log stats
stats = tracker.get_stats()
logger.info(
f"Stats: {stats['total_products']} total, "
f"{stats['in_stock']} in stock, "
f"{stats['out_of_stock']} out of stock"
)
except Exception as e:
logger.error(f"Error checking GameStop: {e}")
send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
def check_bestbuy():
"""Check Best Buy for restocks and new drops"""
logger.info("Checking Best Buy...")
scraper = scrapers["bestbuy"]
try:
for url in BESTBUY_URLS:
logger.info(f"Scraping: {url}")
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("bestbuy", 1))
# Always filter to Pokemon products only
products = scraper.filter_pokemon_products(products)
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
products = scraper.filter_by_keywords(products, KEYWORDS)
logger.info(f"After keyword filter: {len(products)} products")
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
# Send notifications for new products (if enabled)
if NEW_DROP_NOTIFICATIONS:
for product in new_products:
if product.in_stock:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="bestbuy",
alert_type="new_drop",
image_url=product.image_url,
)
logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="bestbuy",
alert_type="restock",
image_url=product.image_url,
)
logger.info(f"Sent notification for restock: {product.name}")
# Log stats
stats = tracker.get_stats()
logger.info(
f"Stats: {stats['total_products']} total, "
f"{stats['in_stock']} in stock, "
f"{stats['out_of_stock']} out of stock"
)
except Exception as e:
logger.error(f"Error checking Best Buy: {e}")
send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
def check_walmart():
"""Check Walmart for restocks and new drops"""
logger.info("Checking Walmart...")
scraper = scrapers["walmart"]
try:
for url in WALMART_URLS:
logger.info(f"Scraping: {url}")
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("walmart", 1))
# Always filter to Pokemon products only
products = scraper.filter_pokemon_products(products)
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
products = scraper.filter_by_keywords(products, KEYWORDS)
logger.info(f"After keyword filter: {len(products)} products")
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
# Send notifications for new products (if enabled)
if NEW_DROP_NOTIFICATIONS:
for product in new_products:
if product.in_stock:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="walmart",
alert_type="new_drop",
image_url=product.image_url,
)
logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
send_stock_alert(
product_name=product.name,
product_url=product.url,
price=product.price or "See link",
site="walmart",
alert_type="restock",
image_url=product.image_url,
)
logger.info(f"Sent notification for restock: {product.name}")
# Log stats
stats = tracker.get_stats()
logger.info(
f"Stats: {stats['total_products']} total, "
f"{stats['in_stock']} in stock, "
f"{stats['out_of_stock']} out of stock"
)
except Exception as e:
logger.error(f"Error checking Walmart: {e}")
send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
def run_checks():
"""Run all enabled site checks"""
logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
if "pokemoncenter" in SITES_ENABLED:
check_pokemoncenter()
if "target" in SITES_ENABLED:
check_target()
if "gamestop" in SITES_ENABLED:
check_gamestop()
if "bestbuy" in SITES_ENABLED:
check_bestbuy()
if "walmart" in SITES_ENABLED:
check_walmart()
logger.info("Check cycle complete")
def graceful_shutdown(signum, frame):
"""Handle shutdown gracefully"""
logger.info("Shutting down...")
shutdown_browser()
sys.exit(0)
def main():
"""Main entry point"""
print("=" * 50)
print("Pokemon Stock Monitor")
print("=" * 50)
print(f"Monitoring: {', '.join(SITES_ENABLED)}")
print(f"Check interval: {CHECK_INTERVAL_SECONDS} seconds")
print(f"Keyword filter: {'ON' if KEYWORD_FILTER_ENABLED else 'OFF'}")
print("=" * 50)
print()
# Register signal handlers
signal.signal(signal.SIGINT, graceful_shutdown)
signal.signal(signal.SIGTERM, graceful_shutdown)
# Initialize browser
logger.info("Initializing browser...")
try:
get_browser()
except Exception as e:
logger.error(f"Failed to initialize browser: {e}")
print(f"\nERROR: Failed to start browser. Make sure Playwright is installed:")
print(" pip install playwright")
print(" playwright install chromium")
return
# Send startup notification
send_startup_notification()
# Run initial check
print("Running initial check...")
run_checks()
# Schedule periodic checks
schedule.every(CHECK_INTERVAL_SECONDS).seconds.do(run_checks)
print(f"\nMonitor running! Checking every {CHECK_INTERVAL_SECONDS} seconds.")
print("Press Ctrl+C to stop.\n")
# Main loop
try:
while True:
schedule.run_pending()
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
graceful_shutdown(None, None)
if __name__ == "__main__":
main()