feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
+11
-1
@@ -1,5 +1,15 @@
|
||||
# Scrapers package
|
||||
from .pokemoncenter import PokemonCenterScraper
|
||||
from .target import TargetScraper
|
||||
from .gamestop import GameStopScraper, warmup_gamestop
|
||||
from .bestbuy import BestBuyScraper
|
||||
from .walmart import WalmartScraper
|
||||
|
||||
__all__ = ["PokemonCenterScraper", "TargetScraper"]
|
||||
__all__ = [
|
||||
"PokemonCenterScraper",
|
||||
"TargetScraper",
|
||||
"GameStopScraper",
|
||||
"BestBuyScraper",
|
||||
"WalmartScraper",
|
||||
"warmup_gamestop",
|
||||
]
|
||||
|
||||
@@ -35,6 +35,50 @@ class BaseScraper(ABC):
|
||||
|
||||
site_name: str = "unknown"
|
||||
|
||||
# Terms that indicate a Pokemon product
|
||||
POKEMON_TERMS = [
|
||||
"pokemon", "pokémon", "poke", "tcg",
|
||||
"pikachu", "charizard", "mewtwo", "eevee", "snorlax",
|
||||
"booster", "elite trainer", "etb",
|
||||
"scarlet", "violet", "prismatic", "evolutions",
|
||||
]
|
||||
|
||||
# Terms that indicate NOT a Pokemon product (false positives from search)
|
||||
EXCLUDE_TERMS = [
|
||||
"ice cube", "oven", "barbie", "hot wheels", "lego",
|
||||
"furniture", "appliance", "kitchen", "bedding",
|
||||
"glitter girls", "masters of the universe", "transformers",
|
||||
"room essentials", "threshold",
|
||||
]
|
||||
|
||||
def is_pokemon_product(self, product: Product) -> bool:
|
||||
"""
|
||||
Check if a product is actually a Pokemon product.
|
||||
Filters out false positives from search results.
|
||||
"""
|
||||
name_lower = product.name.lower()
|
||||
|
||||
# Check for exclusion terms first
|
||||
for term in self.EXCLUDE_TERMS:
|
||||
if term in name_lower:
|
||||
return False
|
||||
|
||||
# Check for Pokemon terms
|
||||
for term in self.POKEMON_TERMS:
|
||||
if term in name_lower:
|
||||
return True
|
||||
|
||||
# If no Pokemon terms found, reject it
|
||||
return False
|
||||
|
||||
def filter_pokemon_products(self, products: List[Product]) -> List[Product]:
|
||||
"""Filter to only include valid Pokemon products"""
|
||||
filtered = [p for p in products if self.is_pokemon_product(p)]
|
||||
rejected = len(products) - len(filtered)
|
||||
if rejected > 0:
|
||||
logger.info(f"Filtered out {rejected} non-Pokemon products")
|
||||
return filtered
|
||||
|
||||
@abstractmethod
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
"""
|
||||
Best Buy scraper - Uses undetected-chromedriver to bypass bot protection
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
|
||||
# Add parent directory to path for tools import
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BestBuyScraper(BaseScraper):
|
||||
"""Scraper for BestBuy.com - Uses undetected-chromedriver"""
|
||||
|
||||
site_name = "bestbuy"
|
||||
base_url = "https://www.bestbuy.com"
|
||||
|
||||
def __init__(self):
|
||||
self._stealth_browser = None
|
||||
|
||||
def _get_stealth_browser(self):
|
||||
"""Get or create stealth browser for Best Buy"""
|
||||
if self._stealth_browser is None:
|
||||
from tools.stealth_browser import StealthBrowser
|
||||
self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
|
||||
self._stealth_browser.start()
|
||||
return self._stealth_browser
|
||||
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a Best Buy search/category page for all products
|
||||
Uses undetected-chromedriver to bypass bot protection
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
max_pages: Maximum number of pages to scrape (default 1)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
all_products = []
|
||||
|
||||
# Add sort by newest if not already in URL
|
||||
if "sort=" not in url:
|
||||
separator = "&" if "?" in url else "?"
|
||||
url = f"{url}{separator}sp=-releasedate"
|
||||
|
||||
# Use stealth browser
|
||||
try:
|
||||
browser = self._get_stealth_browser()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start stealth browser: {e}")
|
||||
return all_products
|
||||
|
||||
for page_num in range(1, max_pages + 1):
|
||||
products = []
|
||||
page_url = url if page_num == 1 else f"{url}&cp={page_num}"
|
||||
|
||||
try:
|
||||
logger.info(f"Navigating to: {page_url}")
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(8) # Wait longer for initial page load
|
||||
|
||||
# Scroll for lazy loading - wait longer between scrolls
|
||||
browser.driver.execute_script("window.scrollTo(0, 500)")
|
||||
time.sleep(2)
|
||||
browser.driver.execute_script("window.scrollTo(0, 1500)")
|
||||
time.sleep(2)
|
||||
browser.driver.execute_script("window.scrollTo(0, 3000)")
|
||||
time.sleep(2)
|
||||
browser.driver.execute_script("window.scrollTo(0, 0)") # Scroll back to top
|
||||
time.sleep(3)
|
||||
|
||||
# Wait for products to load (check for content)
|
||||
for _ in range(10):
|
||||
html = browser.driver.page_source
|
||||
if "sku-item" in html or "sku-title" in html or "priceView" in html:
|
||||
logger.info("Product content detected")
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
# Get HTML after waiting
|
||||
html = browser.driver.page_source
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
browser.driver.save_screenshot("debug_bestbuy.png")
|
||||
logger.info("Saved debug screenshot to debug_bestbuy.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try to find product data in JSON scripts
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
# Try multiple selectors for product cards - Best Buy updates these frequently
|
||||
product_cards = (
|
||||
soup.select("li.sku-item")
|
||||
or soup.select("[data-sku-id]")
|
||||
or soup.select(".sku-item")
|
||||
or soup.select("[class*='sku-item']")
|
||||
or soup.select("div.shop-sku-list-item")
|
||||
or soup.select("[class*='productCard']")
|
||||
or soup.select("[class*='product-card']")
|
||||
or soup.select(".list-item")
|
||||
)
|
||||
|
||||
logger.info(f"Found {len(product_cards)} product cards on page {page_num}")
|
||||
|
||||
for card in product_cards:
|
||||
product = self._parse_product_card(card)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
# Fallback: parse product links
|
||||
if not product_cards:
|
||||
product_links = soup.select("a[href*='/site/'][href*='.p']")
|
||||
logger.info(f"Fallback: Found {len(product_links)} product links")
|
||||
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href or ".p" not in href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
best_link = links[0]
|
||||
for link in links:
|
||||
if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
|
||||
best_link = link
|
||||
break
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
# Also try parsing from data attributes and script tags
|
||||
if not products:
|
||||
products.extend(self._extract_from_page_data(soup, browser))
|
||||
|
||||
# Don't close - reuse browser for next page
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Best Buy page {page_num}: {e}")
|
||||
break # Don't raise - just return what we have
|
||||
|
||||
all_products.extend(products)
|
||||
|
||||
# Stop if no products found on this page (no more pages)
|
||||
if not products:
|
||||
break
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in all_products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
logger.info(f"Scraped {len(unique_products)} unique products from Best Buy")
|
||||
return unique_products
|
||||
|
||||
def _extract_from_page_data(self, soup, browser) -> List[Product]:
|
||||
"""Extract products from page data attributes and evaluate JS if needed"""
|
||||
products = []
|
||||
|
||||
# Try to get product data from data attributes
|
||||
items_with_data = soup.select("[data-testid][data-sku-id]")
|
||||
for item in items_with_data:
|
||||
sku_id = item.get("data-sku-id", "")
|
||||
if sku_id:
|
||||
# Find name and price within this element
|
||||
name_elem = item.select_one("h4") or item.select_one("[class*='title']") or item.select_one("a")
|
||||
name = name_elem.get_text(strip=True) if name_elem else ""
|
||||
|
||||
if name and len(name) > 5:
|
||||
price_text = item.get_text()
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||
price = price_match.group() if price_match else None
|
||||
|
||||
products.append(Product(
|
||||
name=name,
|
||||
url=f"{self.base_url}/site/{sku_id}.p",
|
||||
price=price,
|
||||
in_stock=True,
|
||||
image_url=None,
|
||||
site=self.site_name,
|
||||
product_id=sku_id,
|
||||
))
|
||||
|
||||
# Try to extract from window.__INITIAL_STATE__ or similar JS objects
|
||||
try:
|
||||
initial_state = browser.driver.execute_script("""
|
||||
if (window.__INITIAL_STATE__) return JSON.stringify(window.__INITIAL_STATE__);
|
||||
if (window.__NEXT_DATA__) return JSON.stringify(window.__NEXT_DATA__);
|
||||
return null;
|
||||
""")
|
||||
if initial_state:
|
||||
data = json.loads(initial_state)
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not extract from JS state: {e}")
|
||||
|
||||
return products
|
||||
|
||||
def _extract_products_from_json(self, data, depth=0) -> List[Product]:
|
||||
"""Recursively search JSON for product data"""
|
||||
products = []
|
||||
if depth > 10:
|
||||
return products
|
||||
|
||||
if isinstance(data, dict):
|
||||
# Check if this looks like a Best Buy product
|
||||
if "skuId" in data or ("name" in data and "regularPrice" in data):
|
||||
product = self._parse_product_json(data)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
for value in data.values():
|
||||
products.extend(self._extract_products_from_json(value, depth + 1))
|
||||
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
products.extend(self._extract_products_from_json(item, depth + 1))
|
||||
|
||||
return products
|
||||
|
||||
def _parse_product_json(self, data: dict) -> Optional[Product]:
|
||||
"""Parse a product from Best Buy's JSON data"""
|
||||
try:
|
||||
name = data.get("name") or data.get("displayName", "")
|
||||
if not name:
|
||||
return None
|
||||
|
||||
sku_id = data.get("skuId") or data.get("sku", "")
|
||||
url_slug = data.get("url") or ""
|
||||
|
||||
if url_slug:
|
||||
url = url_slug if url_slug.startswith("http") else f"{self.base_url}{url_slug}"
|
||||
elif sku_id:
|
||||
url = f"{self.base_url}/site/{sku_id}.p"
|
||||
else:
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
if "regularPrice" in data:
|
||||
price = f"${data['regularPrice']:.2f}"
|
||||
elif "salePrice" in data:
|
||||
price = f"${data['salePrice']:.2f}"
|
||||
|
||||
# Check availability
|
||||
in_stock = True
|
||||
availability = data.get("availability", {})
|
||||
if isinstance(availability, dict):
|
||||
in_stock = availability.get("isAvailable", True)
|
||||
elif data.get("orderable") is False:
|
||||
in_stock = False
|
||||
|
||||
# Get image
|
||||
image_url = data.get("image") or data.get("thumbnailImage")
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=str(sku_id),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Best Buy product JSON: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_card(self, card) -> Optional[Product]:
|
||||
"""Parse a product card element"""
|
||||
try:
|
||||
# Find link
|
||||
link = card.select_one("a[href*='/site/'][href*='.p']") or card.select_one("a.image-link")
|
||||
if not link:
|
||||
link = card.select_one("a")
|
||||
if not link:
|
||||
return None
|
||||
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name
|
||||
name_elem = (
|
||||
card.select_one(".sku-title a")
|
||||
or card.select_one("h4.sku-header a")
|
||||
or card.select_one("[data-testid='product-title']")
|
||||
or card.select_one(".sku-title")
|
||||
or link
|
||||
)
|
||||
name = name_elem.get_text(strip=True) if name_elem else "Unknown"
|
||||
|
||||
# Skip if name too short
|
||||
if len(name) < 5:
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price_elem = (
|
||||
card.select_one(".priceView-customer-price span")
|
||||
or card.select_one("[data-testid='customer-price']")
|
||||
or card.select_one(".pricing-price__regular-price")
|
||||
or card.select_one("[class*='price']")
|
||||
)
|
||||
price = None
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
# Check stock
|
||||
in_stock = self._check_card_stock_status(card)
|
||||
|
||||
# Get image
|
||||
img = card.select_one("img")
|
||||
image_url = None
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}"
|
||||
|
||||
# Extract SKU ID from URL
|
||||
product_id = ""
|
||||
match = re.search(r"/(\d+)\.p", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Best Buy product card: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_link(self, link, soup) -> Optional[Product]:
|
||||
"""Parse a product from a product link element"""
|
||||
try:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
name = link.get("aria-label", "") or link.get_text(strip=True)
|
||||
|
||||
if not name or len(name) < 5:
|
||||
return None
|
||||
|
||||
# Extract SKU ID
|
||||
product_id = ""
|
||||
match = re.search(r"/(\d+)\.p", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
# Find price near link
|
||||
parent = link.find_parent()
|
||||
price = None
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
price_text = parent.get_text()
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Find image
|
||||
image_url = None
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
img = parent.select_one("img")
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Check stock
|
||||
in_stock = True
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
text = parent.get_text().lower()
|
||||
if any(phrase in text for phrase in ["sold out", "out of stock", "unavailable"]):
|
||||
in_stock = False
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Best Buy product link: {e}")
|
||||
return None
|
||||
|
||||
def _check_card_stock_status(self, card) -> bool:
|
||||
"""Check if a product card indicates in-stock status"""
|
||||
card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
|
||||
|
||||
# Check for disabled add to cart button
|
||||
add_btn = card.select_one(".add-to-cart-button")
|
||||
if add_btn and "btn-disabled" in add_btn.get("class", []):
|
||||
return False
|
||||
|
||||
out_of_stock_phrases = [
|
||||
"sold out",
|
||||
"out of stock",
|
||||
"unavailable",
|
||||
"coming soon",
|
||||
"not available",
|
||||
]
|
||||
|
||||
for phrase in out_of_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return False
|
||||
|
||||
in_stock_phrases = [
|
||||
"add to cart",
|
||||
"add to bag",
|
||||
"available",
|
||||
"in stock",
|
||||
]
|
||||
|
||||
for phrase in in_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check if a specific product is in stock"""
|
||||
try:
|
||||
browser = self._get_stealth_browser()
|
||||
browser.driver.get(product_url)
|
||||
time.sleep(3)
|
||||
html = browser.driver.page_source
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
page_text = soup.get_text().lower()
|
||||
|
||||
out_of_stock = any(
|
||||
phrase in page_text
|
||||
for phrase in ["sold out", "out of stock", "unavailable", "coming soon"]
|
||||
)
|
||||
|
||||
in_stock = not out_of_stock
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
price_elem = (
|
||||
soup.select_one(".priceView-customer-price span")
|
||||
or soup.select_one("[data-testid='customer-price']")
|
||||
)
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
return in_stock, price
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Best Buy product stock: {e}")
|
||||
return False, None
|
||||
@@ -0,0 +1,478 @@
|
||||
"""
|
||||
GameStop.com scraper
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from src.browser import get_browser
|
||||
from config import CAPTCHA_WAIT_TIMEOUT
|
||||
|
||||
# Add parent directory to path for tools import
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def warmup_gamestop():
|
||||
"""
|
||||
Warmup function to solve Cloudflare CAPTCHA manually.
|
||||
Uses undetected-chromedriver to bypass bot detection.
|
||||
"""
|
||||
from tools.stealth_browser import StealthBrowser
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("GAMESTOP WARMUP - Using Undetected Chrome")
|
||||
print("=" * 60)
|
||||
print("A browser window should open.")
|
||||
print("If you see a Cloudflare challenge, solve it manually.")
|
||||
print(f"Waiting up to {CAPTCHA_WAIT_TIMEOUT} seconds...")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
logger.info("Starting GameStop warmup with undetected-chromedriver...")
|
||||
|
||||
# Use stealth browser instead of Playwright
|
||||
browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||
|
||||
try:
|
||||
browser.start()
|
||||
logger.info("Stealth browser started")
|
||||
|
||||
# Navigate to GameStop
|
||||
browser.driver.get("https://www.gamestop.com")
|
||||
time.sleep(3)
|
||||
|
||||
# Check for Cloudflare challenge
|
||||
start_time = time.time()
|
||||
challenge_detected = False
|
||||
|
||||
while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
|
||||
try:
|
||||
html = browser.driver.page_source
|
||||
html_lower = html.lower()
|
||||
title = browser.driver.title.lower()
|
||||
|
||||
# Cloudflare challenge indicators
|
||||
is_cloudflare_challenge = (
|
||||
"just a moment" in title or
|
||||
"checking your browser" in html_lower or
|
||||
"cf-challenge" in html_lower or
|
||||
"turnstile" in html_lower or
|
||||
(len(html) < 5000 and "challenge" in html_lower)
|
||||
)
|
||||
|
||||
if is_cloudflare_challenge:
|
||||
if not challenge_detected:
|
||||
challenge_detected = True
|
||||
print(">>> Cloudflare challenge detected! Please solve it in the browser window.")
|
||||
logger.info("Cloudflare challenge detected - waiting for manual solve...")
|
||||
time.sleep(5)
|
||||
else:
|
||||
# Check if we're on actual GameStop content
|
||||
if len(html) > 10000 and "gamestop" in html_lower:
|
||||
print(">>> Challenge solved! GameStop page loaded successfully.")
|
||||
logger.info("GameStop warmup complete!")
|
||||
browser.stop()
|
||||
return True
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking page: {e}")
|
||||
time.sleep(2)
|
||||
|
||||
logger.warning("GameStop warmup timed out - CAPTCHA may not be solved")
|
||||
print(">>> Warmup timed out. You may need to try again.")
|
||||
browser.stop()
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during warmup: {e}")
|
||||
try:
|
||||
browser.stop()
|
||||
except:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
class GameStopScraper(BaseScraper):
|
||||
"""Scraper for GameStop.com - Uses undetected-chromedriver to bypass Cloudflare"""
|
||||
|
||||
site_name = "gamestop"
|
||||
base_url = "https://www.gamestop.com"
|
||||
|
||||
def __init__(self):
|
||||
self._stealth_browser = None
|
||||
|
||||
def _get_stealth_browser(self):
|
||||
"""Get or create stealth browser for GameStop"""
|
||||
if self._stealth_browser is None:
|
||||
from tools.stealth_browser import StealthBrowser
|
||||
self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||
self._stealth_browser.start()
|
||||
return self._stealth_browser
|
||||
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a GameStop search/category page for all products
|
||||
Uses undetected-chromedriver to bypass Cloudflare
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
max_pages: Maximum number of pages to scrape (default 1)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
all_products = []
|
||||
|
||||
# Add sort by newest if not already in URL
|
||||
if "sort=" not in url.lower():
|
||||
separator = "&" if "?" in url else "?"
|
||||
url = f"{url}{separator}sort=date-desc"
|
||||
|
||||
# Use stealth browser for GameStop
|
||||
try:
|
||||
browser = self._get_stealth_browser()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start stealth browser: {e}")
|
||||
return all_products
|
||||
|
||||
for page_num in range(1, max_pages + 1):
|
||||
products = []
|
||||
page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}"
|
||||
|
||||
try:
|
||||
logger.info(f"Navigating to: {page_url}")
|
||||
browser.driver.get(page_url)
|
||||
time.sleep(5) # Wait for page load
|
||||
|
||||
html = browser.driver.page_source
|
||||
|
||||
# Check for Cloudflare challenge
|
||||
html_lower = html.lower()
|
||||
title = browser.driver.title.lower()
|
||||
|
||||
is_cloudflare = (
|
||||
"just a moment" in title or
|
||||
"checking your browser" in html_lower or
|
||||
"cf-challenge" in html_lower
|
||||
)
|
||||
|
||||
if is_cloudflare:
|
||||
logger.warning("Cloudflare challenge detected! Waiting for manual solve...")
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
|
||||
time.sleep(5)
|
||||
html = browser.driver.page_source
|
||||
html_lower = html.lower()
|
||||
title = browser.driver.title.lower()
|
||||
if "just a moment" not in title and "cf-challenge" not in html_lower:
|
||||
logger.info("Cloudflare challenge solved!")
|
||||
break
|
||||
else:
|
||||
logger.error("Cloudflare challenge not solved in time")
|
||||
return all_products
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
browser.driver.save_screenshot("debug_gamestop.png")
|
||||
logger.info("Saved debug screenshot to debug_gamestop.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try multiple selectors for product cards
|
||||
product_cards = (
|
||||
soup.select(".product-tile")
|
||||
or soup.select("[data-testid='product-tile']")
|
||||
or soup.select(".grid-tile")
|
||||
or soup.select(".product-grid-tile")
|
||||
or soup.select("[class*='ProductTile']")
|
||||
)
|
||||
|
||||
logger.info(f"Found {len(product_cards)} product cards")
|
||||
|
||||
# If no cards found, try link-based extraction
|
||||
if not product_cards:
|
||||
product_links = soup.select("a[href*='/products/']")
|
||||
logger.info(f"Fallback: Found {len(product_links)} product links")
|
||||
|
||||
# Group by href
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href or "/products/" not in href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
# Pick best link
|
||||
best_link = links[0]
|
||||
for link in links:
|
||||
if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
|
||||
best_link = link
|
||||
break
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
else:
|
||||
for card in product_cards:
|
||||
product = self._parse_product_card(card)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
# Don't close - reuse browser for next page
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping GameStop page {page_num}: {e}")
|
||||
break # Don't raise - just return what we have
|
||||
|
||||
all_products.extend(products)
|
||||
|
||||
# Stop if no products found on this page
|
||||
if not products:
|
||||
break
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in all_products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
logger.info(f"Scraped {len(unique_products)} unique products from GameStop")
|
||||
return unique_products
|
||||
|
||||
def _parse_product_card(self, card) -> Optional[Product]:
|
||||
"""Parse a product card element"""
|
||||
try:
|
||||
# Find link - try multiple patterns
|
||||
link = (
|
||||
card.select_one("a[href*='/products/']") or
|
||||
card.select_one("a[href*='/video-games/']") or
|
||||
card.select_one("a[href*='/collectibles/']") or
|
||||
card.select_one("a.product-tile-link") or
|
||||
card.select_one("a")
|
||||
)
|
||||
if not link:
|
||||
logger.debug("No link found in card")
|
||||
return None
|
||||
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
logger.debug("Empty href")
|
||||
return None
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name - try multiple patterns
|
||||
name_elem = (
|
||||
card.select_one(".product-tile-title")
|
||||
or card.select_one(".product-name a")
|
||||
or card.select_one(".product-name")
|
||||
or card.select_one(".product-title")
|
||||
or card.select_one("[data-testid='product-name']")
|
||||
or card.select_one("a[aria-label]")
|
||||
or link
|
||||
)
|
||||
|
||||
# Try to get name from aria-label first
|
||||
name = ""
|
||||
if name_elem:
|
||||
name = name_elem.get("aria-label", "") or name_elem.get_text(strip=True)
|
||||
|
||||
# Skip if name too short
|
||||
if len(name) < 5:
|
||||
logger.debug(f"Name too short: '{name}' from {url[:50]}")
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price_elem = (
|
||||
card.select_one(".price-sales")
|
||||
or card.select_one(".product-price")
|
||||
or card.select_one("[data-testid='price']")
|
||||
or card.select_one("[class*='price']")
|
||||
)
|
||||
price = None
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
# Check stock
|
||||
in_stock = self._check_card_stock_status(card)
|
||||
|
||||
# Get image
|
||||
img = card.select_one("img")
|
||||
image_url = None
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src") or img.get("data-lazy")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"{self.base_url}{image_url}"
|
||||
|
||||
# Extract product ID from URL
|
||||
product_id = ""
|
||||
match = re.search(r"/products/[^/]+/(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing GameStop product card: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_link(self, link, soup) -> Optional[Product]:
|
||||
"""Parse a product from a product link element"""
|
||||
try:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name
|
||||
name = link.get("aria-label", "") or link.get_text(strip=True)
|
||||
|
||||
# Skip if name too short
|
||||
if not name or len(name) < 5:
|
||||
return None
|
||||
|
||||
# Extract product ID from URL
|
||||
product_id = ""
|
||||
match = re.search(r"/products/[^/]+/(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
# Try to find price near this link
|
||||
parent = link.find_parent()
|
||||
price = None
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
price_text = parent.get_text()
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Try to find image
|
||||
image_url = None
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
img = parent.select_one("img")
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"{self.base_url}{image_url}"
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Check stock
|
||||
in_stock = True
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
text = parent.get_text().lower()
|
||||
if any(phrase in text for phrase in ["sold out", "out of stock", "not available"]):
|
||||
in_stock = False
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing GameStop product link: {e}")
|
||||
return None
|
||||
|
||||
def _check_card_stock_status(self, card) -> bool:
|
||||
"""Check if a product card indicates in-stock status"""
|
||||
card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
|
||||
|
||||
out_of_stock_phrases = [
|
||||
"sold out",
|
||||
"out of stock",
|
||||
"not available",
|
||||
"unavailable",
|
||||
"currently unavailable",
|
||||
]
|
||||
|
||||
for phrase in out_of_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return False
|
||||
|
||||
in_stock_phrases = [
|
||||
"add to cart",
|
||||
"add to bag",
|
||||
"available",
|
||||
"buy now",
|
||||
"in stock",
|
||||
]
|
||||
|
||||
for phrase in in_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return True
|
||||
|
||||
# Default: assume in stock if listed
|
||||
return True
|
||||
|
||||
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check if a specific product is in stock"""
|
||||
try:
|
||||
browser = self._get_stealth_browser()
|
||||
browser.driver.get(product_url)
|
||||
time.sleep(3)
|
||||
html = browser.driver.page_source
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
page_text = soup.get_text().lower()
|
||||
|
||||
out_of_stock = any(
|
||||
phrase in page_text
|
||||
for phrase in ["sold out", "out of stock", "not available", "unavailable"]
|
||||
)
|
||||
|
||||
in_stock = not out_of_stock
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
price_elem = (
|
||||
soup.select_one(".product-price")
|
||||
or soup.select_one(".price-sales")
|
||||
or soup.select_one("[data-testid='price']")
|
||||
)
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
return in_stock, price
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking GameStop product stock: {e}")
|
||||
return False, None
|
||||
+82
-24
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
PokemonCenter.com scraper
|
||||
Handles bot protection with Playwright stealth
|
||||
Handles bot protection with undetected-chromedriver stealth browser
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -9,44 +9,83 @@ from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from browser import get_browser
|
||||
import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_stealth_or_regular_browser():
|
||||
"""Get appropriate browser based on config"""
|
||||
if config.USE_STEALTH_BROWSER:
|
||||
from tools.stealth_browser import get_stealth_browser
|
||||
return get_stealth_browser(
|
||||
headless=config.STEALTH_HEADLESS,
|
||||
session_name=config.STEALTH_SESSION_NAME
|
||||
), True
|
||||
else:
|
||||
from src.browser import get_browser
|
||||
return get_browser(), False
|
||||
|
||||
|
||||
class PokemonCenterScraper(BaseScraper):
|
||||
"""Scraper for PokemonCenter.com"""
|
||||
|
||||
site_name = "pokemoncenter"
|
||||
base_url = "https://www.pokemoncenter.com"
|
||||
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a PokemonCenter category page for all products
|
||||
|
||||
Args:
|
||||
url: Category page URL
|
||||
max_pages: Maximum pages to scrape (not yet implemented for PokemonCenter)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
browser = get_browser()
|
||||
# TODO: Implement pagination for PokemonCenter when needed
|
||||
browser, is_stealth = get_stealth_or_regular_browser()
|
||||
products = []
|
||||
|
||||
try:
|
||||
# Navigate and wait for page to load (don't wait for specific selector)
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None, # Let it use networkidle instead
|
||||
timeout=60000,
|
||||
)
|
||||
if is_stealth:
|
||||
# Use stealth browser (undetected-chromedriver)
|
||||
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
|
||||
|
||||
# Save screenshot for debugging if needed
|
||||
try:
|
||||
page.screenshot(path="debug_screenshot.png")
|
||||
logger.info("Saved debug screenshot to debug_screenshot.png")
|
||||
except:
|
||||
pass
|
||||
# Check for CAPTCHA
|
||||
if browser.check_for_captcha():
|
||||
logger.warning("CAPTCHA detected on PokemonCenter!")
|
||||
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
|
||||
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
|
||||
# Re-fetch page after CAPTCHA solve
|
||||
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
|
||||
else:
|
||||
logger.error("Cannot solve CAPTCHA in headless mode")
|
||||
return []
|
||||
|
||||
# Save screenshot for debugging
|
||||
try:
|
||||
browser.screenshot("debug_screenshot.png")
|
||||
logger.info("Saved debug screenshot to debug_screenshot.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
page = None # Stealth browser doesn't return page object
|
||||
else:
|
||||
# Use regular Playwright browser
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None, # Let it use networkidle instead
|
||||
timeout=60000,
|
||||
)
|
||||
|
||||
# Save screenshot for debugging if needed
|
||||
try:
|
||||
page.screenshot(path="debug_screenshot.png")
|
||||
logger.info("Saved debug screenshot to debug_screenshot.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
@@ -96,7 +135,9 @@ class PokemonCenterScraper(BaseScraper):
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
page.close()
|
||||
# Close page only for Playwright browser
|
||||
if page is not None:
|
||||
page.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping PokemonCenter category page: {e}")
|
||||
@@ -224,14 +265,30 @@ class PokemonCenterScraper(BaseScraper):
|
||||
Returns:
|
||||
Tuple of (is_in_stock, price)
|
||||
"""
|
||||
browser = get_browser()
|
||||
browser, is_stealth = get_stealth_or_regular_browser()
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(
|
||||
product_url,
|
||||
wait_for_selector="button, [data-testid]",
|
||||
timeout=30000,
|
||||
)
|
||||
if is_stealth:
|
||||
# Use stealth browser
|
||||
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
|
||||
|
||||
# Check for CAPTCHA
|
||||
if browser.check_for_captcha():
|
||||
logger.warning("CAPTCHA detected on product page!")
|
||||
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
|
||||
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
|
||||
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
|
||||
else:
|
||||
return False, None
|
||||
|
||||
page = None
|
||||
else:
|
||||
# Use regular Playwright browser
|
||||
page, html = browser.get_page_content(
|
||||
product_url,
|
||||
wait_for_selector="button, [data-testid]",
|
||||
timeout=30000,
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
page_text = soup.get_text().lower()
|
||||
@@ -257,7 +314,8 @@ class PokemonCenterScraper(BaseScraper):
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
page.close()
|
||||
if page is not None:
|
||||
page.close()
|
||||
return in_stock, price
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+74
-56
@@ -9,7 +9,7 @@ from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from browser import get_browser
|
||||
from src.browser import get_browser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,87 +20,105 @@ class TargetScraper(BaseScraper):
|
||||
site_name = "target"
|
||||
base_url = "https://www.target.com"
|
||||
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a Target search/category page for all products
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
max_pages: Maximum number of pages to scrape (default 1)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
browser = get_browser()
|
||||
products = []
|
||||
all_products = []
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
# Add sort by newest if not already in URL
|
||||
if "sortBy=" not in url:
|
||||
separator = "&" if "?" in url else "?"
|
||||
url = f"{url}{separator}sortBy=newest"
|
||||
|
||||
for page_num in range(1, max_pages + 1):
|
||||
products = []
|
||||
page_url = url if page_num == 1 else f"{url}&Nao={24 * (page_num - 1)}"
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
page.screenshot(path="debug_target.png")
|
||||
logger.info("Saved debug screenshot to debug_target.png")
|
||||
except:
|
||||
pass
|
||||
page, html = browser.get_page_content(
|
||||
page_url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try to find product data in page scripts (Target uses React/hydration)
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
# Save debug screenshot
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
# Look for product data in the JSON
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
page.screenshot(path="debug_target.png")
|
||||
logger.info("Saved debug screenshot to debug_target.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Parse product links from the page
|
||||
product_links = soup.select("a[href*='/p/']")
|
||||
logger.info(f"Found {len(product_links)} product links")
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Group links by href and pick the best one (with aria-label or text)
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
# Try to find product data in page scripts (Target uses React/hydration)
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
# Look for product data in the JSON
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
logger.info(f"Found {len(href_to_links)} unique hrefs")
|
||||
# Parse product links from the page
|
||||
product_links = soup.select("a[href*='/p/']")
|
||||
logger.info(f"Found {len(product_links)} product links")
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
# Find the best link (one with aria-label or text content)
|
||||
best_link = None
|
||||
for link in links:
|
||||
aria = link.get("aria-label", "")
|
||||
text = link.get_text(strip=True)
|
||||
if aria or (text and len(text) > 10):
|
||||
best_link = link
|
||||
break
|
||||
if not best_link:
|
||||
best_link = links[0] # Fallback to first link
|
||||
# Group links by href and pick the best one (with aria-label or text)
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
logger.info(f"Found {len(href_to_links)} unique hrefs")
|
||||
|
||||
page.close()
|
||||
for href, links in href_to_links.items():
|
||||
# Find the best link (one with aria-label or text content)
|
||||
best_link = None
|
||||
for link in links:
|
||||
aria = link.get("aria-label", "")
|
||||
text = link.get_text(strip=True)
|
||||
if aria or (text and len(text) > 10):
|
||||
best_link = link
|
||||
break
|
||||
if not best_link:
|
||||
best_link = links[0] # Fallback to first link
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Target category page: {e}")
|
||||
raise
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
page.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Target page {page_num}: {e}")
|
||||
if page_num == 1:
|
||||
raise
|
||||
break
|
||||
|
||||
all_products.extend(products)
|
||||
|
||||
# Stop if no products found on this page
|
||||
if not products:
|
||||
break
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in products:
|
||||
for p in all_products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
"""
|
||||
Walmart.com scraper
|
||||
|
||||
Note: Walmart has aggressive bot protection (PerimeterX).
|
||||
May need stealth browser mode for sustained use.
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from src.browser import get_browser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WalmartScraper(BaseScraper):
|
||||
"""Scraper for Walmart.com"""
|
||||
|
||||
site_name = "walmart"
|
||||
base_url = "https://www.walmart.com"
|
||||
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a Walmart search/category page for all products
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
max_pages: Maximum pages to scrape (not yet implemented for Walmart)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
# TODO: Implement pagination for Walmart when needed
|
||||
browser = get_browser()
|
||||
products = []
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
page.screenshot(path="debug_walmart.png")
|
||||
logger.info("Saved debug screenshot to debug_walmart.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try to extract from __NEXT_DATA__ JSON (Walmart uses Next.js)
|
||||
next_data = soup.select_one("script#__NEXT_DATA__")
|
||||
if next_data:
|
||||
try:
|
||||
data = json.loads(next_data.string)
|
||||
products.extend(self._extract_products_from_next_data(data))
|
||||
except (json.JSONDecodeError, TypeError) as e:
|
||||
logger.debug(f"Error parsing __NEXT_DATA__: {e}")
|
||||
|
||||
# Also try application/json scripts
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
# Try multiple selectors for product cards
|
||||
product_cards = (
|
||||
soup.select("[data-item-id]")
|
||||
or soup.select(".search-result-gridview-item")
|
||||
or soup.select("[data-testid='list-view']")
|
||||
or soup.select("[class*='product-item']")
|
||||
or soup.select("[class*='ProductCard']")
|
||||
)
|
||||
|
||||
logger.info(f"Found {len(product_cards)} product cards")
|
||||
|
||||
for card in product_cards:
|
||||
product = self._parse_product_card(card)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
# Fallback: parse product links
|
||||
if not product_cards and not products:
|
||||
product_links = soup.select("a[href*='/ip/']")
|
||||
logger.info(f"Fallback: Found {len(product_links)} product links")
|
||||
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href or "/ip/" not in href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
best_link = links[0]
|
||||
for link in links:
|
||||
if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
|
||||
best_link = link
|
||||
break
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
page.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Walmart category page: {e}")
|
||||
raise
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
logger.info(f"Scraped {len(unique_products)} unique products from Walmart")
|
||||
return unique_products
|
||||
|
||||
def _extract_products_from_next_data(self, data: dict) -> List[Product]:
|
||||
"""Extract products from Next.js __NEXT_DATA__ JSON"""
|
||||
products = []
|
||||
|
||||
try:
|
||||
# Navigate to search results in Next.js data structure
|
||||
props = data.get("props", {})
|
||||
page_props = props.get("pageProps", {})
|
||||
initial_data = page_props.get("initialData", {})
|
||||
search_result = initial_data.get("searchResult", {})
|
||||
item_stacks = search_result.get("itemStacks", [])
|
||||
|
||||
for stack in item_stacks:
|
||||
items = stack.get("items", [])
|
||||
for item in items:
|
||||
product = self._parse_walmart_item(item)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error extracting from __NEXT_DATA__: {e}")
|
||||
|
||||
return products
|
||||
|
||||
def _extract_products_from_json(self, data, depth=0) -> List[Product]:
|
||||
"""Recursively search JSON for product data"""
|
||||
products = []
|
||||
if depth > 10:
|
||||
return products
|
||||
|
||||
if isinstance(data, dict):
|
||||
# Check if this looks like a Walmart product
|
||||
if "usItemId" in data or ("name" in data and "priceInfo" in data):
|
||||
product = self._parse_walmart_item(data)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
for value in data.values():
|
||||
products.extend(self._extract_products_from_json(value, depth + 1))
|
||||
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
products.extend(self._extract_products_from_json(item, depth + 1))
|
||||
|
||||
return products
|
||||
|
||||
def _parse_walmart_item(self, data: dict) -> Optional[Product]:
|
||||
"""Parse a product from Walmart's JSON data"""
|
||||
try:
|
||||
name = data.get("name") or data.get("title", "")
|
||||
if not name:
|
||||
return None
|
||||
|
||||
item_id = data.get("usItemId") or data.get("id", "")
|
||||
canonical_url = data.get("canonicalUrl") or data.get("productPageUrl", "")
|
||||
|
||||
if canonical_url:
|
||||
url = canonical_url if canonical_url.startswith("http") else f"{self.base_url}{canonical_url}"
|
||||
elif item_id:
|
||||
url = f"{self.base_url}/ip/{item_id}"
|
||||
else:
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
price_info = data.get("priceInfo", {})
|
||||
if isinstance(price_info, dict):
|
||||
current_price = price_info.get("currentPrice", {})
|
||||
if isinstance(current_price, dict):
|
||||
price = current_price.get("priceString")
|
||||
elif price_info.get("priceString"):
|
||||
price = price_info.get("priceString")
|
||||
|
||||
if not price and "price" in data:
|
||||
price_val = data.get("price")
|
||||
if isinstance(price_val, (int, float)):
|
||||
price = f"${price_val:.2f}"
|
||||
|
||||
# Check availability
|
||||
in_stock = True
|
||||
availability = data.get("availabilityStatusV2", {})
|
||||
if isinstance(availability, dict):
|
||||
status = availability.get("value", "").upper()
|
||||
in_stock = status not in ["OUT_OF_STOCK", "NOT_AVAILABLE"]
|
||||
elif data.get("availabilityStatus"):
|
||||
in_stock = data.get("availabilityStatus") != "OUT_OF_STOCK"
|
||||
|
||||
# Get image
|
||||
image_url = None
|
||||
image_info = data.get("imageInfo", {})
|
||||
if isinstance(image_info, dict):
|
||||
image_url = image_info.get("thumbnailUrl") or image_info.get("url")
|
||||
elif data.get("image"):
|
||||
image_url = data.get("image")
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=str(item_id),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Walmart product JSON: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_card(self, card) -> Optional[Product]:
|
||||
"""Parse a product card element"""
|
||||
try:
|
||||
# Find link
|
||||
link = card.select_one("a[href*='/ip/']") or card.select_one("a")
|
||||
if not link:
|
||||
return None
|
||||
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name
|
||||
name_elem = (
|
||||
card.select_one("[data-automation-id='product-title']")
|
||||
or card.select_one(".product-title-link span")
|
||||
or card.select_one("[class*='ProductTitle']")
|
||||
or card.select_one("[class*='product-title']")
|
||||
or link
|
||||
)
|
||||
name = name_elem.get_text(strip=True) if name_elem else "Unknown"
|
||||
|
||||
# Skip if name too short
|
||||
if len(name) < 5:
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price_elem = (
|
||||
card.select_one("[data-automation-id='product-price']")
|
||||
or card.select_one(".price-current")
|
||||
or card.select_one("[class*='ProductPrice']")
|
||||
or card.select_one("[class*='price']")
|
||||
)
|
||||
price = None
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
# Check stock
|
||||
in_stock = self._check_card_stock_status(card)
|
||||
|
||||
# Get image
|
||||
img = card.select_one("img")
|
||||
image_url = None
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
if image_url and not image_url.startswith("http"):
|
||||
image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}"
|
||||
|
||||
# Extract item ID from URL
|
||||
product_id = ""
|
||||
match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Walmart product card: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_link(self, link, soup) -> Optional[Product]:
|
||||
"""Parse a product from a product link element"""
|
||||
try:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
name = link.get("aria-label", "") or link.get_text(strip=True)
|
||||
|
||||
if not name or len(name) < 5:
|
||||
return None
|
||||
|
||||
# Extract item ID
|
||||
product_id = ""
|
||||
match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
# Find price near link
|
||||
parent = link.find_parent()
|
||||
price = None
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
price_text = parent.get_text()
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Find image
|
||||
image_url = None
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
img = parent.select_one("img")
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Check stock
|
||||
in_stock = True
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
text = parent.get_text().lower()
|
||||
if any(phrase in text for phrase in ["out of stock", "unavailable", "not available"]):
|
||||
in_stock = False
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Walmart product link: {e}")
|
||||
return None
|
||||
|
||||
def _check_card_stock_status(self, card) -> bool:
|
||||
"""Check if a product card indicates in-stock status"""
|
||||
card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
|
||||
|
||||
out_of_stock_phrases = [
|
||||
"out of stock",
|
||||
"unavailable",
|
||||
"not available",
|
||||
"sold out",
|
||||
"pickup not available",
|
||||
]
|
||||
|
||||
for phrase in out_of_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return False
|
||||
|
||||
in_stock_phrases = [
|
||||
"add to cart",
|
||||
"available",
|
||||
"in stock",
|
||||
"pickup available",
|
||||
"delivery available",
|
||||
]
|
||||
|
||||
for phrase in in_stock_phrases:
|
||||
if phrase in card_text:
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check if a specific product is in stock"""
|
||||
browser = get_browser()
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(product_url, timeout=30000)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
page_text = soup.get_text().lower()
|
||||
|
||||
out_of_stock = any(
|
||||
phrase in page_text
|
||||
for phrase in ["out of stock", "unavailable", "not available", "sold out"]
|
||||
)
|
||||
|
||||
in_stock = not out_of_stock
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
price_elem = (
|
||||
soup.select_one("[data-automation-id='product-price']")
|
||||
or soup.select_one("[itemprop='price']")
|
||||
or soup.select_one("[class*='price']")
|
||||
)
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
page.close()
|
||||
return in_stock, price
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Walmart product stock: {e}")
|
||||
return False, None
|
||||
Reference in New Issue
Block a user