Cleanups for scrapers
This commit is contained in:
+46
-34
@@ -12,7 +12,7 @@ from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from src.browser import get_browser
|
||||
from config import CAPTCHA_WAIT_TIMEOUT
|
||||
from config import CAPTCHA_WAIT_TIMEOUT, GAMESTOP_HEADLESS
|
||||
|
||||
# Add parent directory to path for tools import
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -20,6 +20,20 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_img_url(img_tag) -> Optional[str]:
|
||||
"""Extract real image URL from an img tag, skipping lazy-load placeholders"""
|
||||
if not img_tag:
|
||||
return None
|
||||
for attr in ("src", "data-src", "data-lazy", "data-lazy-src", "data-srcset", "srcset"):
|
||||
val = img_tag.get(attr, "")
|
||||
if not val:
|
||||
continue
|
||||
url = val.split()[0].rstrip(",")
|
||||
if url and not url.startswith("data:") and len(url) > 20:
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def warmup_gamestop():
|
||||
"""
|
||||
Warmup function to solve Cloudflare CAPTCHA manually.
|
||||
@@ -38,7 +52,7 @@ def warmup_gamestop():
|
||||
logger.info("Starting GameStop warmup with undetected-chromedriver...")
|
||||
|
||||
# Use stealth browser instead of Playwright
|
||||
browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||
browser = StealthBrowser(headless=GAMESTOP_HEADLESS, session_name="gamestop")
|
||||
|
||||
try:
|
||||
browser.start()
|
||||
@@ -117,7 +131,7 @@ class GameStopScraper(BaseScraper):
|
||||
# Check if we need to create a new browser
|
||||
if self._stealth_browser is None:
|
||||
logger.info("Creating new GameStop stealth browser...")
|
||||
self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||
self._stealth_browser = StealthBrowser(headless=GAMESTOP_HEADLESS, session_name="gamestop")
|
||||
self._stealth_browser.start()
|
||||
self._restart_attempts = 0
|
||||
return self._stealth_browser
|
||||
@@ -194,6 +208,14 @@ class GameStopScraper(BaseScraper):
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(5) # Wait for page load
|
||||
|
||||
# Scroll to trigger lazy-loaded images
|
||||
browser.driver.execute_script("window.scrollTo(0, document.body.scrollHeight / 2)")
|
||||
time.sleep(1)
|
||||
browser.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
|
||||
time.sleep(1)
|
||||
browser.driver.execute_script("window.scrollTo(0, 0)")
|
||||
time.sleep(1)
|
||||
|
||||
html = browser.driver.page_source
|
||||
break # Success, exit retry loop
|
||||
|
||||
@@ -223,19 +245,15 @@ class GameStopScraper(BaseScraper):
|
||||
)
|
||||
|
||||
if is_cloudflare:
|
||||
logger.warning("Cloudflare challenge detected! Waiting for manual solve...")
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
|
||||
time.sleep(5)
|
||||
html = browser.driver.page_source
|
||||
html_lower = html.lower()
|
||||
title = browser.driver.title.lower()
|
||||
if "just a moment" not in title and "cf-challenge" not in html_lower:
|
||||
logger.info("Cloudflare challenge solved!")
|
||||
break
|
||||
else:
|
||||
logger.warning("Cloudflare challenge detected!")
|
||||
solved = browser.resolve_captcha_interactively(page_url, CAPTCHA_WAIT_TIMEOUT)
|
||||
if not solved:
|
||||
logger.error("Cloudflare challenge not solved in time")
|
||||
return all_products
|
||||
# Re-navigate in headless mode with fresh cookies
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(5)
|
||||
html = browser.driver.page_source
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
@@ -371,12 +389,9 @@ class GameStopScraper(BaseScraper):
|
||||
in_stock = self._check_card_stock_status(card)
|
||||
|
||||
# Get image
|
||||
img = card.select_one("img")
|
||||
image_url = None
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src") or img.get("data-lazy")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"{self.base_url}{image_url}"
|
||||
image_url = _extract_img_url(card.select_one("img"))
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"{self.base_url}{image_url}"
|
||||
|
||||
# Extract product ID from URL
|
||||
product_id = ""
|
||||
@@ -437,10 +452,9 @@ class GameStopScraper(BaseScraper):
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
img = parent.select_one("img")
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = _extract_img_url(parent.select_one("img"))
|
||||
if image_url:
|
||||
if not image_url.startswith("http"):
|
||||
image_url = f"{self.base_url}{image_url}"
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
@@ -474,29 +488,27 @@ class GameStopScraper(BaseScraper):
|
||||
"""Check if a product card indicates in-stock status"""
|
||||
card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
|
||||
|
||||
out_of_stock_phrases = [
|
||||
out_of_stock_phrases = {
|
||||
"sold out",
|
||||
"out of stock",
|
||||
"not available",
|
||||
"unavailable",
|
||||
"currently unavailable",
|
||||
]
|
||||
}
|
||||
|
||||
for phrase in out_of_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return False
|
||||
if any(phrase in card_text for phrase in out_of_stock_phrases):
|
||||
return False
|
||||
|
||||
in_stock_phrases = [
|
||||
in_stock_phrases = {
|
||||
"add to cart",
|
||||
"add to bag",
|
||||
"available",
|
||||
"buy now",
|
||||
"in stock",
|
||||
]
|
||||
}
|
||||
|
||||
for phrase in in_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return True
|
||||
if any(phrase in card_text for phrase in in_stock_phrases):
|
||||
return True
|
||||
|
||||
# Default: assume in stock if listed
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user