Browser session recovery, price tracking, and dashboard improvements
- Add browser session validation and auto-restart for BestBuy/GameStop scrapers - Add price change detection and notifications - Remove PokemonCenter scraper (now uses Chrome Extension) - Dashboard UI improvements Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
+67
-8
@@ -107,15 +107,55 @@ class GameStopScraper(BaseScraper):
|
||||
|
||||
def __init__(self):
|
||||
self._stealth_browser = None
|
||||
self._restart_attempts = 0
|
||||
self._max_restart_attempts = 3
|
||||
|
||||
def _get_stealth_browser(self):
|
||||
"""Get or create stealth browser for GameStop"""
|
||||
"""Get or create stealth browser for GameStop, with auto-restart on session errors"""
|
||||
from tools.stealth_browser import StealthBrowser
|
||||
|
||||
# Check if we need to create a new browser
|
||||
if self._stealth_browser is None:
|
||||
from tools.stealth_browser import StealthBrowser
|
||||
logger.info("Creating new GameStop stealth browser...")
|
||||
self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||
self._stealth_browser.start()
|
||||
self._restart_attempts = 0
|
||||
return self._stealth_browser
|
||||
|
||||
# Check if existing session is still valid
|
||||
if not self._stealth_browser.is_session_valid():
|
||||
if self._restart_attempts >= self._max_restart_attempts:
|
||||
logger.error(f"GameStop browser failed after {self._max_restart_attempts} restart attempts")
|
||||
# Reset counter and try one more time after clearing
|
||||
self._restart_attempts = 0
|
||||
self._stealth_browser = None
|
||||
return self._get_stealth_browser()
|
||||
|
||||
logger.warning("GameStop browser session invalid, restarting...")
|
||||
self._restart_attempts += 1
|
||||
self._stealth_browser.restart()
|
||||
|
||||
return self._stealth_browser
|
||||
|
||||
def _handle_session_error(self, error: Exception) -> bool:
|
||||
"""
|
||||
Check if error is a session error and handle it.
|
||||
Returns True if browser was restarted and operation should be retried.
|
||||
"""
|
||||
error_msg = str(error).lower()
|
||||
session_errors = ["invalid session id", "session deleted", "no such session", "browser has closed"]
|
||||
|
||||
if any(err in error_msg for err in session_errors):
|
||||
logger.warning(f"Session error detected: {error}")
|
||||
if self._stealth_browser and self._restart_attempts < self._max_restart_attempts:
|
||||
self._restart_attempts += 1
|
||||
try:
|
||||
self._stealth_browser.restart()
|
||||
return True # Retry operation
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to restart browser: {e}")
|
||||
return False
|
||||
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a GameStop search/category page for all products
|
||||
@@ -145,14 +185,33 @@ class GameStopScraper(BaseScraper):
|
||||
for page_num in range(1, max_pages + 1):
|
||||
products = []
|
||||
page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}"
|
||||
retry_count = 0
|
||||
max_retries = 2
|
||||
|
||||
while retry_count <= max_retries:
|
||||
try:
|
||||
logger.info(f"Navigating to: {page_url}")
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(5) # Wait for page load
|
||||
|
||||
html = browser.driver.page_source
|
||||
break # Success, exit retry loop
|
||||
|
||||
except Exception as e:
|
||||
if self._handle_session_error(e) and retry_count < max_retries:
|
||||
retry_count += 1
|
||||
logger.info(f"Retrying after browser restart (attempt {retry_count}/{max_retries})")
|
||||
browser = self._get_stealth_browser() # Get restarted browser
|
||||
continue
|
||||
else:
|
||||
logger.error(f"Error navigating to {page_url}: {e}")
|
||||
return all_products
|
||||
else:
|
||||
# while loop completed without break (all retries exhausted)
|
||||
logger.error(f"Failed to load page after {max_retries} retries")
|
||||
return all_products
|
||||
|
||||
try:
|
||||
logger.info(f"Navigating to: {page_url}")
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(5) # Wait for page load
|
||||
|
||||
html = browser.driver.page_source
|
||||
|
||||
# Check for Cloudflare challenge
|
||||
html_lower = html.lower()
|
||||
title = browser.driver.title.lower()
|
||||
|
||||
Reference in New Issue
Block a user