feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Test script for the stealth browser module
|
||||
Run this to verify undetected-chromedriver is working
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
print("=" * 60)
|
||||
print("Stealth Browser Test")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Check dependencies
|
||||
print("1. Checking dependencies...")
|
||||
try:
|
||||
import undetected_chromedriver as uc
|
||||
print(" [OK] undetected-chromedriver installed")
|
||||
except ImportError:
|
||||
print(" [ERROR] undetected-chromedriver not installed!")
|
||||
print(" Run: pip install undetected-chromedriver")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
print(" [OK] selenium installed")
|
||||
except ImportError:
|
||||
print(" [ERROR] selenium not installed!")
|
||||
print(" Run: pip install selenium")
|
||||
sys.exit(1)
|
||||
|
||||
# Import our stealth browser
|
||||
print()
|
||||
print("2. Importing stealth browser module...")
|
||||
try:
|
||||
from stealth_browser import StealthBrowser, get_stealth_browser
|
||||
print(" [OK] stealth_browser module loaded")
|
||||
except Exception as e:
|
||||
print(f" [ERROR] Failed to import: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Test browser launch
|
||||
print()
|
||||
print("3. Launching stealth browser...")
|
||||
print(" (A Chrome window should open)")
|
||||
print()
|
||||
|
||||
browser = None
|
||||
try:
|
||||
browser = StealthBrowser(
|
||||
headless=False,
|
||||
session_name="test_session"
|
||||
)
|
||||
browser.start()
|
||||
print(" [OK] Browser started successfully!")
|
||||
|
||||
# Test navigation
|
||||
print()
|
||||
print("4. Testing navigation to Pokemon Center...")
|
||||
print(" (Watch the browser window)")
|
||||
|
||||
url = "https://www.pokemoncenter.com/category/tcg-cards"
|
||||
html = browser.get_page(url, wait_time=5)
|
||||
|
||||
print(f" [OK] Page loaded - {len(html)} bytes")
|
||||
|
||||
# Check for CAPTCHA
|
||||
print()
|
||||
print("5. Checking for bot detection...")
|
||||
if browser.check_for_captcha():
|
||||
print(" [!] CAPTCHA/Challenge detected!")
|
||||
print(" The browser window is open - solve it manually if needed")
|
||||
print(" Waiting up to 60 seconds...")
|
||||
browser.wait_for_captcha_solve(timeout=60)
|
||||
else:
|
||||
print(" [OK] No CAPTCHA detected! Stealth mode working.")
|
||||
|
||||
# Show some page info
|
||||
print()
|
||||
print("6. Page analysis...")
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
title = soup.title.string if soup.title else "No title"
|
||||
print(f" Page title: {title}")
|
||||
|
||||
# Count potential product elements
|
||||
products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']")
|
||||
print(f" Potential product elements: {len(products)}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("TEST COMPLETE")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("The browser window will stay open for 10 seconds so you can inspect it.")
|
||||
print("Press Ctrl+C to close early.")
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n Interrupted by user")
|
||||
except Exception as e:
|
||||
print(f" [ERROR] {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if browser:
|
||||
print()
|
||||
print("Closing browser...")
|
||||
browser.stop()
|
||||
print("Done!")
|
||||
Reference in New Issue
Block a user