feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
+74
View File
@@ -0,0 +1,74 @@
"""
Warmup tool for sites with CAPTCHA/bot protection.
Run this before the main monitor to solve CAPTCHAs manually.
Usage:
python tools/warmup.py gamestop
python tools/warmup.py all
"""
import sys
import os
# Add parent directory to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import logging
from scrapers.gamestop import warmup_gamestop
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
def warmup_all():
"""Warm up all sites that need it"""
results = {}
logger.info("=" * 50)
logger.info("Starting warmup for GameStop...")
logger.info("=" * 50)
results["gamestop"] = warmup_gamestop()
# Add more sites here as needed
# results["pokemoncenter"] = warmup_pokemoncenter()
logger.info("=" * 50)
logger.info("Warmup Results:")
for site, success in results.items():
status = "OK" if success else "FAILED"
logger.info(f" {site}: {status}")
logger.info("=" * 50)
return all(results.values())
def main():
if len(sys.argv) < 2:
print(__doc__)
print("\nAvailable sites: gamestop, all")
sys.exit(1)
site = sys.argv[1].lower()
if site == "gamestop":
success = warmup_gamestop()
elif site == "all":
success = warmup_all()
else:
print(f"Unknown site: {site}")
print("Available sites: gamestop, all")
sys.exit(1)
if success:
logger.info("Warmup completed successfully!")
sys.exit(0)
else:
logger.error("Warmup failed or timed out")
sys.exit(1)
if __name__ == "__main__":
main()