8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
441 lines
15 KiB
Python
441 lines
15 KiB
Python
"""
|
|
Pokemon Stock Monitor - Main entry point
|
|
Monitors retail sites for Pokemon card restocks and new drops
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
import signal
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
import schedule
|
|
|
|
from config import (
|
|
CHECK_INTERVAL_SECONDS,
|
|
SITES_ENABLED,
|
|
POKEMON_CENTER_URLS,
|
|
TARGET_URLS,
|
|
GAMESTOP_URLS,
|
|
BESTBUY_URLS,
|
|
WALMART_URLS,
|
|
KEYWORD_FILTER_ENABLED,
|
|
KEYWORDS,
|
|
LOG_LEVEL,
|
|
NEW_DROP_NOTIFICATIONS,
|
|
MAX_PAGES,
|
|
)
|
|
from src.browser import get_browser, shutdown_browser
|
|
from src.product_tracker import ProductTracker
|
|
from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
|
|
from scrapers import (
|
|
PokemonCenterScraper,
|
|
TargetScraper,
|
|
GameStopScraper,
|
|
BestBuyScraper,
|
|
WalmartScraper,
|
|
)
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=getattr(logging, LOG_LEVEL),
|
|
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
|
handlers=[
|
|
logging.StreamHandler(),
|
|
logging.FileHandler("monitor.log", encoding="utf-8"),
|
|
],
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Global tracker
|
|
tracker = ProductTracker()
|
|
|
|
# Scrapers
|
|
scrapers = {
|
|
"pokemoncenter": PokemonCenterScraper(),
|
|
"target": TargetScraper(),
|
|
"gamestop": GameStopScraper(),
|
|
"bestbuy": BestBuyScraper(),
|
|
"walmart": WalmartScraper(),
|
|
}
|
|
|
|
|
|
def check_pokemoncenter():
|
|
"""Check PokemonCenter for restocks and new drops"""
|
|
logger.info("Checking PokemonCenter...")
|
|
scraper = scrapers["pokemoncenter"]
|
|
|
|
try:
|
|
for url in POKEMON_CENTER_URLS:
|
|
logger.info(f"Scraping: {url}")
|
|
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1))
|
|
|
|
# Apply keyword filter if enabled
|
|
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
logger.info(f"After keyword filter: {len(products)} products")
|
|
|
|
# Process and detect changes
|
|
new_products, restocked_products = tracker.process_products(products)
|
|
|
|
# Send notifications for new products (if enabled)
|
|
if NEW_DROP_NOTIFICATIONS:
|
|
for product in new_products:
|
|
if product.in_stock:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="pokemoncenter",
|
|
alert_type="new_drop",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for new product: {product.name}")
|
|
|
|
# Send notifications for restocks
|
|
for product in restocked_products:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="pokemoncenter",
|
|
alert_type="restock",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for restock: {product.name}")
|
|
|
|
# Log stats
|
|
stats = tracker.get_stats()
|
|
logger.info(
|
|
f"Stats: {stats['total_products']} total, "
|
|
f"{stats['in_stock']} in stock, "
|
|
f"{stats['out_of_stock']} out of stock"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking PokemonCenter: {e}")
|
|
send_error_notification(f"Error checking PokemonCenter: {str(e)}", "PokemonCenter")
|
|
|
|
|
|
def check_target():
|
|
"""Check Target for restocks and new drops"""
|
|
logger.info("Checking Target...")
|
|
scraper = scrapers["target"]
|
|
|
|
try:
|
|
for url in TARGET_URLS:
|
|
logger.info(f"Scraping: {url}")
|
|
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("target", 1))
|
|
|
|
# Always filter to Pokemon products only
|
|
products = scraper.filter_pokemon_products(products)
|
|
|
|
# Apply keyword filter if enabled
|
|
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
logger.info(f"After keyword filter: {len(products)} products")
|
|
|
|
# Process and detect changes
|
|
new_products, restocked_products = tracker.process_products(products)
|
|
|
|
# Send notifications for new products (if enabled)
|
|
if NEW_DROP_NOTIFICATIONS:
|
|
for product in new_products:
|
|
if product.in_stock:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="target",
|
|
alert_type="new_drop",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for new product: {product.name}")
|
|
|
|
# Send notifications for restocks
|
|
for product in restocked_products:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="target",
|
|
alert_type="restock",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for restock: {product.name}")
|
|
|
|
# Log stats
|
|
stats = tracker.get_stats()
|
|
logger.info(
|
|
f"Stats: {stats['total_products']} total, "
|
|
f"{stats['in_stock']} in stock, "
|
|
f"{stats['out_of_stock']} out of stock"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking Target: {e}")
|
|
send_error_notification(f"Error checking Target: {str(e)}", "Target")
|
|
|
|
|
|
def check_gamestop():
|
|
"""Check GameStop for restocks and new drops"""
|
|
logger.info("Checking GameStop...")
|
|
scraper = scrapers["gamestop"]
|
|
|
|
try:
|
|
for url in GAMESTOP_URLS:
|
|
logger.info(f"Scraping: {url}")
|
|
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("gamestop", 1))
|
|
|
|
# Always filter to Pokemon products only
|
|
products = scraper.filter_pokemon_products(products)
|
|
|
|
# Apply keyword filter if enabled
|
|
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
logger.info(f"After keyword filter: {len(products)} products")
|
|
|
|
# Process and detect changes
|
|
new_products, restocked_products = tracker.process_products(products)
|
|
|
|
# Send notifications for new products (if enabled)
|
|
if NEW_DROP_NOTIFICATIONS:
|
|
for product in new_products:
|
|
if product.in_stock:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="gamestop",
|
|
alert_type="new_drop",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for new product: {product.name}")
|
|
|
|
# Send notifications for restocks
|
|
for product in restocked_products:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="gamestop",
|
|
alert_type="restock",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for restock: {product.name}")
|
|
|
|
# Log stats
|
|
stats = tracker.get_stats()
|
|
logger.info(
|
|
f"Stats: {stats['total_products']} total, "
|
|
f"{stats['in_stock']} in stock, "
|
|
f"{stats['out_of_stock']} out of stock"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking GameStop: {e}")
|
|
send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
|
|
|
|
|
|
def check_bestbuy():
|
|
"""Check Best Buy for restocks and new drops"""
|
|
logger.info("Checking Best Buy...")
|
|
scraper = scrapers["bestbuy"]
|
|
|
|
try:
|
|
for url in BESTBUY_URLS:
|
|
logger.info(f"Scraping: {url}")
|
|
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("bestbuy", 1))
|
|
|
|
# Always filter to Pokemon products only
|
|
products = scraper.filter_pokemon_products(products)
|
|
|
|
# Apply keyword filter if enabled
|
|
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
logger.info(f"After keyword filter: {len(products)} products")
|
|
|
|
# Process and detect changes
|
|
new_products, restocked_products = tracker.process_products(products)
|
|
|
|
# Send notifications for new products (if enabled)
|
|
if NEW_DROP_NOTIFICATIONS:
|
|
for product in new_products:
|
|
if product.in_stock:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="bestbuy",
|
|
alert_type="new_drop",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for new product: {product.name}")
|
|
|
|
# Send notifications for restocks
|
|
for product in restocked_products:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="bestbuy",
|
|
alert_type="restock",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for restock: {product.name}")
|
|
|
|
# Log stats
|
|
stats = tracker.get_stats()
|
|
logger.info(
|
|
f"Stats: {stats['total_products']} total, "
|
|
f"{stats['in_stock']} in stock, "
|
|
f"{stats['out_of_stock']} out of stock"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking Best Buy: {e}")
|
|
send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
|
|
|
|
|
|
def check_walmart():
|
|
"""Check Walmart for restocks and new drops"""
|
|
logger.info("Checking Walmart...")
|
|
scraper = scrapers["walmart"]
|
|
|
|
try:
|
|
for url in WALMART_URLS:
|
|
logger.info(f"Scraping: {url}")
|
|
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("walmart", 1))
|
|
|
|
# Always filter to Pokemon products only
|
|
products = scraper.filter_pokemon_products(products)
|
|
|
|
# Apply keyword filter if enabled
|
|
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
logger.info(f"After keyword filter: {len(products)} products")
|
|
|
|
# Process and detect changes
|
|
new_products, restocked_products = tracker.process_products(products)
|
|
|
|
# Send notifications for new products (if enabled)
|
|
if NEW_DROP_NOTIFICATIONS:
|
|
for product in new_products:
|
|
if product.in_stock:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="walmart",
|
|
alert_type="new_drop",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for new product: {product.name}")
|
|
|
|
# Send notifications for restocks
|
|
for product in restocked_products:
|
|
send_stock_alert(
|
|
product_name=product.name,
|
|
product_url=product.url,
|
|
price=product.price or "See link",
|
|
site="walmart",
|
|
alert_type="restock",
|
|
image_url=product.image_url,
|
|
)
|
|
logger.info(f"Sent notification for restock: {product.name}")
|
|
|
|
# Log stats
|
|
stats = tracker.get_stats()
|
|
logger.info(
|
|
f"Stats: {stats['total_products']} total, "
|
|
f"{stats['in_stock']} in stock, "
|
|
f"{stats['out_of_stock']} out of stock"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error checking Walmart: {e}")
|
|
send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
|
|
|
|
|
|
def run_checks():
|
|
"""Run all enabled site checks"""
|
|
logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
|
|
|
|
if "pokemoncenter" in SITES_ENABLED:
|
|
check_pokemoncenter()
|
|
|
|
if "target" in SITES_ENABLED:
|
|
check_target()
|
|
|
|
if "gamestop" in SITES_ENABLED:
|
|
check_gamestop()
|
|
|
|
if "bestbuy" in SITES_ENABLED:
|
|
check_bestbuy()
|
|
|
|
if "walmart" in SITES_ENABLED:
|
|
check_walmart()
|
|
|
|
logger.info("Check cycle complete")
|
|
|
|
|
|
def graceful_shutdown(signum, frame):
|
|
"""Handle shutdown gracefully"""
|
|
logger.info("Shutting down...")
|
|
shutdown_browser()
|
|
sys.exit(0)
|
|
|
|
|
|
def main():
|
|
"""Main entry point"""
|
|
print("=" * 50)
|
|
print("Pokemon Stock Monitor")
|
|
print("=" * 50)
|
|
print(f"Monitoring: {', '.join(SITES_ENABLED)}")
|
|
print(f"Check interval: {CHECK_INTERVAL_SECONDS} seconds")
|
|
print(f"Keyword filter: {'ON' if KEYWORD_FILTER_ENABLED else 'OFF'}")
|
|
print("=" * 50)
|
|
print()
|
|
|
|
# Register signal handlers
|
|
signal.signal(signal.SIGINT, graceful_shutdown)
|
|
signal.signal(signal.SIGTERM, graceful_shutdown)
|
|
|
|
# Initialize browser
|
|
logger.info("Initializing browser...")
|
|
try:
|
|
get_browser()
|
|
except Exception as e:
|
|
logger.error(f"Failed to initialize browser: {e}")
|
|
print(f"\nERROR: Failed to start browser. Make sure Playwright is installed:")
|
|
print(" pip install playwright")
|
|
print(" playwright install chromium")
|
|
return
|
|
|
|
# Send startup notification
|
|
send_startup_notification()
|
|
|
|
# Run initial check
|
|
print("Running initial check...")
|
|
run_checks()
|
|
|
|
# Schedule periodic checks
|
|
schedule.every(CHECK_INTERVAL_SECONDS).seconds.do(run_checks)
|
|
|
|
print(f"\nMonitor running! Checking every {CHECK_INTERVAL_SECONDS} seconds.")
|
|
print("Press Ctrl+C to stop.\n")
|
|
|
|
# Main loop
|
|
try:
|
|
while True:
|
|
schedule.run_pending()
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
pass
|
|
finally:
|
|
graceful_shutdown(None, None)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|