mmcghen
8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
75 lines
1.7 KiB
Python
75 lines
1.7 KiB
Python
"""
|
|
Warmup tool for sites with CAPTCHA/bot protection.
|
|
Run this before the main monitor to solve CAPTCHAs manually.
|
|
|
|
Usage:
|
|
python tools/warmup.py gamestop
|
|
python tools/warmup.py all
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import logging
|
|
from scrapers.gamestop import warmup_gamestop
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def warmup_all():
|
|
"""Warm up all sites that need it"""
|
|
results = {}
|
|
|
|
logger.info("=" * 50)
|
|
logger.info("Starting warmup for GameStop...")
|
|
logger.info("=" * 50)
|
|
results["gamestop"] = warmup_gamestop()
|
|
|
|
# Add more sites here as needed
|
|
# results["pokemoncenter"] = warmup_pokemoncenter()
|
|
|
|
logger.info("=" * 50)
|
|
logger.info("Warmup Results:")
|
|
for site, success in results.items():
|
|
status = "OK" if success else "FAILED"
|
|
logger.info(f" {site}: {status}")
|
|
logger.info("=" * 50)
|
|
|
|
return all(results.values())
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print(__doc__)
|
|
print("\nAvailable sites: gamestop, all")
|
|
sys.exit(1)
|
|
|
|
site = sys.argv[1].lower()
|
|
|
|
if site == "gamestop":
|
|
success = warmup_gamestop()
|
|
elif site == "all":
|
|
success = warmup_all()
|
|
else:
|
|
print(f"Unknown site: {site}")
|
|
print("Available sites: gamestop, all")
|
|
sys.exit(1)
|
|
|
|
if success:
|
|
logger.info("Warmup completed successfully!")
|
|
sys.exit(0)
|
|
else:
|
|
logger.error("Warmup failed or timed out")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|