mmcghen
8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
112 lines
3.0 KiB
Python
112 lines
3.0 KiB
Python
"""
|
|
Test script for the stealth browser module
|
|
Run this to verify undetected-chromedriver is working
|
|
"""
|
|
|
|
import sys
|
|
import time
|
|
|
|
print("=" * 60)
|
|
print("Stealth Browser Test")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
# Check dependencies
|
|
print("1. Checking dependencies...")
|
|
try:
|
|
import undetected_chromedriver as uc
|
|
print(" [OK] undetected-chromedriver installed")
|
|
except ImportError:
|
|
print(" [ERROR] undetected-chromedriver not installed!")
|
|
print(" Run: pip install undetected-chromedriver")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from selenium.webdriver.common.action_chains import ActionChains
|
|
print(" [OK] selenium installed")
|
|
except ImportError:
|
|
print(" [ERROR] selenium not installed!")
|
|
print(" Run: pip install selenium")
|
|
sys.exit(1)
|
|
|
|
# Import our stealth browser
|
|
print()
|
|
print("2. Importing stealth browser module...")
|
|
try:
|
|
from stealth_browser import StealthBrowser, get_stealth_browser
|
|
print(" [OK] stealth_browser module loaded")
|
|
except Exception as e:
|
|
print(f" [ERROR] Failed to import: {e}")
|
|
sys.exit(1)
|
|
|
|
# Test browser launch
|
|
print()
|
|
print("3. Launching stealth browser...")
|
|
print(" (A Chrome window should open)")
|
|
print()
|
|
|
|
browser = None
|
|
try:
|
|
browser = StealthBrowser(
|
|
headless=False,
|
|
session_name="test_session"
|
|
)
|
|
browser.start()
|
|
print(" [OK] Browser started successfully!")
|
|
|
|
# Test navigation
|
|
print()
|
|
print("4. Testing navigation to Pokemon Center...")
|
|
print(" (Watch the browser window)")
|
|
|
|
url = "https://www.pokemoncenter.com/category/tcg-cards"
|
|
html = browser.get_page(url, wait_time=5)
|
|
|
|
print(f" [OK] Page loaded - {len(html)} bytes")
|
|
|
|
# Check for CAPTCHA
|
|
print()
|
|
print("5. Checking for bot detection...")
|
|
if browser.check_for_captcha():
|
|
print(" [!] CAPTCHA/Challenge detected!")
|
|
print(" The browser window is open - solve it manually if needed")
|
|
print(" Waiting up to 60 seconds...")
|
|
browser.wait_for_captcha_solve(timeout=60)
|
|
else:
|
|
print(" [OK] No CAPTCHA detected! Stealth mode working.")
|
|
|
|
# Show some page info
|
|
print()
|
|
print("6. Page analysis...")
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
title = soup.title.string if soup.title else "No title"
|
|
print(f" Page title: {title}")
|
|
|
|
# Count potential product elements
|
|
products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']")
|
|
print(f" Potential product elements: {len(products)}")
|
|
|
|
print()
|
|
print("=" * 60)
|
|
print("TEST COMPLETE")
|
|
print("=" * 60)
|
|
print()
|
|
print("The browser window will stay open for 10 seconds so you can inspect it.")
|
|
print("Press Ctrl+C to close early.")
|
|
|
|
time.sleep(10)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n Interrupted by user")
|
|
except Exception as e:
|
|
print(f" [ERROR] {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
finally:
|
|
if browser:
|
|
print()
|
|
print("Closing browser...")
|
|
browser.stop()
|
|
print("Done!")
|