feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
+111
View File
@@ -0,0 +1,111 @@
"""
Test script for the stealth browser module
Run this to verify undetected-chromedriver is working
"""
import sys
import time
print("=" * 60)
print("Stealth Browser Test")
print("=" * 60)
print()
# Check dependencies
print("1. Checking dependencies...")
try:
import undetected_chromedriver as uc
print(" [OK] undetected-chromedriver installed")
except ImportError:
print(" [ERROR] undetected-chromedriver not installed!")
print(" Run: pip install undetected-chromedriver")
sys.exit(1)
try:
from selenium.webdriver.common.action_chains import ActionChains
print(" [OK] selenium installed")
except ImportError:
print(" [ERROR] selenium not installed!")
print(" Run: pip install selenium")
sys.exit(1)
# Import our stealth browser
print()
print("2. Importing stealth browser module...")
try:
from stealth_browser import StealthBrowser, get_stealth_browser
print(" [OK] stealth_browser module loaded")
except Exception as e:
print(f" [ERROR] Failed to import: {e}")
sys.exit(1)
# Test browser launch
print()
print("3. Launching stealth browser...")
print(" (A Chrome window should open)")
print()
browser = None
try:
browser = StealthBrowser(
headless=False,
session_name="test_session"
)
browser.start()
print(" [OK] Browser started successfully!")
# Test navigation
print()
print("4. Testing navigation to Pokemon Center...")
print(" (Watch the browser window)")
url = "https://www.pokemoncenter.com/category/tcg-cards"
html = browser.get_page(url, wait_time=5)
print(f" [OK] Page loaded - {len(html)} bytes")
# Check for CAPTCHA
print()
print("5. Checking for bot detection...")
if browser.check_for_captcha():
print(" [!] CAPTCHA/Challenge detected!")
print(" The browser window is open - solve it manually if needed")
print(" Waiting up to 60 seconds...")
browser.wait_for_captcha_solve(timeout=60)
else:
print(" [OK] No CAPTCHA detected! Stealth mode working.")
# Show some page info
print()
print("6. Page analysis...")
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, 'html.parser')
title = soup.title.string if soup.title else "No title"
print(f" Page title: {title}")
# Count potential product elements
products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']")
print(f" Potential product elements: {len(products)}")
print()
print("=" * 60)
print("TEST COMPLETE")
print("=" * 60)
print()
print("The browser window will stay open for 10 seconds so you can inspect it.")
print("Press Ctrl+C to close early.")
time.sleep(10)
except KeyboardInterrupt:
print("\n Interrupted by user")
except Exception as e:
print(f" [ERROR] {e}")
import traceback
traceback.print_exc()
finally:
if browser:
print()
print("Closing browser...")
browser.stop()
print("Done!")