feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Warmup tool for sites with CAPTCHA/bot protection.
|
||||
Run this before the main monitor to solve CAPTCHAs manually.
|
||||
|
||||
Usage:
|
||||
python tools/warmup.py gamestop
|
||||
python tools/warmup.py all
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import logging
|
||||
from scrapers.gamestop import warmup_gamestop
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def warmup_all():
|
||||
"""Warm up all sites that need it"""
|
||||
results = {}
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info("Starting warmup for GameStop...")
|
||||
logger.info("=" * 50)
|
||||
results["gamestop"] = warmup_gamestop()
|
||||
|
||||
# Add more sites here as needed
|
||||
# results["pokemoncenter"] = warmup_pokemoncenter()
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info("Warmup Results:")
|
||||
for site, success in results.items():
|
||||
status = "OK" if success else "FAILED"
|
||||
logger.info(f" {site}: {status}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
return all(results.values())
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(__doc__)
|
||||
print("\nAvailable sites: gamestop, all")
|
||||
sys.exit(1)
|
||||
|
||||
site = sys.argv[1].lower()
|
||||
|
||||
if site == "gamestop":
|
||||
success = warmup_gamestop()
|
||||
elif site == "all":
|
||||
success = warmup_all()
|
||||
else:
|
||||
print(f"Unknown site: {site}")
|
||||
print("Available sites: gamestop, all")
|
||||
sys.exit(1)
|
||||
|
||||
if success:
|
||||
logger.info("Warmup completed successfully!")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("Warmup failed or timed out")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user