mmcghen/pokemon-stock-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanups for scrapers

Browse Source
This commit is contained in:
Michael McGhen
2026-04-10 10:04:20 -04:00
parent 0d5cc08bc0
commit 052a762182
15 changed files with 1411 additions and 567 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/bestbuy.py
+138 -47
View File
@@ -12,6 +12,7 @@ from typing import List, Optional
from bs4 import BeautifulSoup
from .base import BaseScraper, Product
from config import BESTBUY_HEADLESS
# Add parent directory to path for tools import
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -37,7 +38,7 @@ class BestBuyScraper(BaseScraper):
# Check if we need to create a new browser
if self._stealth_browser is None:
logger.info("Creating new Best Buy stealth browser...")
self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
self._stealth_browser = StealthBrowser(headless=BESTBUY_HEADLESS, session_name="bestbuy")
self._stealth_browser.start()
self._restart_attempts = 0
return self._stealth_browser
@@ -63,7 +64,11 @@ class BestBuyScraper(BaseScraper):
Returns True if browser was restarted and operation should be retried.
"""
error_msg = str(error).lower()
session_errors = ["invalid session id", "session deleted", "no such session", "browser has closed"]
session_errors = [
"invalid session id", "session deleted", "no such session", "browser has closed",
"newconnectionerror", "connection refused", "winerror 10061",
"failed to establish a new connection", "max retries exceeded",
]
if any(err in error_msg for err in session_errors):
logger.warning(f"Session error detected: {error}")
@@ -90,11 +95,6 @@ class BestBuyScraper(BaseScraper):
"""
all_products = []
# Add sort by newest if not already in URL
if "sort=" not in url:
separator = "&" if "?" in url else "?"
url = f"{url}{separator}sp=-releasedate"
# Use stealth browser
try:
browser = self._get_stealth_browser()
@@ -113,22 +113,26 @@ class BestBuyScraper(BaseScraper):
try:
logger.info(f"Navigating to: {page_url}")
browser.driver.get(page_url)
time.sleep(8) # Wait longer for initial page load
time.sleep(15) # Wait for React hydration and GraphQL fetches to complete
# Scroll for lazy loading - wait longer between scrolls
browser.driver.execute_script("window.scrollTo(0, 500)")
# Scroll to trigger any remaining lazy-loaded products
scroll_pause = 1.5
scroll_step = 800
current_pos = 0
for _ in range(20):
current_pos += scroll_step
browser.driver.execute_script(f"window.scrollTo(0, {current_pos})")
time.sleep(scroll_pause)
page_height = browser.driver.execute_script("return document.body.scrollHeight")
if current_pos >= page_height:
break
browser.driver.execute_script("window.scrollTo(0, 0)")
time.sleep(2)
browser.driver.execute_script("window.scrollTo(0, 1500)")
time.sleep(2)
browser.driver.execute_script("window.scrollTo(0, 3000)")
time.sleep(2)
browser.driver.execute_script("window.scrollTo(0, 0)") # Scroll back to top
time.sleep(3)
# Wait for products to load (check for content)
for _ in range(10):
html = browser.driver.page_source
if "sku-item" in html or "sku-title" in html or "priceView" in html:
if any(kw in html for kw in ["sku-item", "sku-title", "priceView", "product-list-item", "ApolloSSRDataTransport", '"pdp":']):
logger.info("Product content detected")
break
time.sleep(1)
@@ -136,10 +140,12 @@ class BestBuyScraper(BaseScraper):
# Get HTML after waiting
html = browser.driver.page_source
# Save debug screenshot
# Save debug screenshot and HTML for analysis
try:
browser.driver.save_screenshot("debug_bestbuy.png")
logger.info("Saved debug screenshot to debug_bestbuy.png")
with open("debug_bestbuy.html", "w", encoding="utf-8") as f:
f.write(html)
logger.info("Saved debug screenshot and HTML to debug_bestbuy.*")
except:
pass
@@ -274,6 +280,45 @@ class BestBuyScraper(BaseScraper):
logger.info(f"Scraped {len(unique_products)} unique products from Best Buy")
return unique_products
def _extract_from_graphql_response(self, data: dict) -> List[Product]:
"""Extract products from BestBuy's intercepted GraphQL search response"""
products = []
try:
search = data.get('searchResult') or data.get('searchResults') or {}
items = search.get('products') or search.get('items') or []
logger.info(f"GraphQL response has {len(items)} products")
for item in items:
product = item.get('product') or item
name = (product.get('name') or {}).get('short') or (product.get('name') or {}).get('title') or product.get('displayName') or ''
if not name or len(name) < 5:
continue
sku_id = str(product.get('skuId') or '')
url_obj = product.get('url') or {}
url = url_obj.get('skuSpecificUrl') or url_obj.get('pdp') or (f"{self.base_url}/site/{sku_id}.p" if sku_id else None)
if not url:
continue
image = (product.get('primaryImage') or {}).get('piscesHref')
price = None
price_info = product.get('priceInfo') or product.get('price') or {}
if isinstance(price_info, dict):
raw = price_info.get('currentPrice') or price_info.get('regularPrice')
if isinstance(raw, (int, float)):
price = f"${raw:.2f}"
# Check stock via button state
in_stock = True
fulfillment = product.get('fulfillmentOptions') or {}
btn_states = fulfillment.get('buttonStates') or []
if btn_states:
state = btn_states[0].get('buttonState', '')
in_stock = state in ('ADD_TO_CART', 'PRE_ORDER', 'CHECK_STORES')
products.append(Product(
name=name, url=url, price=price, in_stock=in_stock,
image_url=image, site=self.site_name, product_id=sku_id,
))
except Exception as e:
logger.error(f"Error parsing GraphQL response: {e}")
return products
def _extract_from_page_data(self, soup, browser) -> List[Product]:
"""Extract products from page data attributes and evaluate JS if needed"""
products = []
@@ -298,7 +343,58 @@ class BestBuyScraper(BaseScraper):
except (json.JSONDecodeError, TypeError) as e:
logger.debug(f"Could not parse __NEXT_DATA__ from HTML: {e}")
# Method 3: Try multiple JS state sources via browser execution
# Method 3: Extract from live Apollo client cache (captures client-side loaded products)
# BestBuy SSRs only ~4 featured products; the rest load via client-side GraphQL.
# The Apollo client cache holds all of them after hydration.
apollo_cache_script = """
try {
var client = window.__APOLLO_CLIENT__;
if (!client) return null;
var cache = client.cache.extract();
var products = [];
for (var key in cache) {
var obj = cache[key];
if (obj && obj.__typename === 'Product' && obj.skuId && obj.name && obj.name.short) {
products.push({
skuId: obj.skuId,
name: obj.name.short || obj.name.title || '',
pdp: obj.url ? obj.url.pdp || obj.url.skuSpecificUrl : null,
price: obj.priceAndAvailabilityInfo ? obj.priceAndAvailabilityInfo.currentPrice : null,
inStock: obj.fulfillmentOptions ? !!(obj.fulfillmentOptions.buttonStates || []).find(function(b){ return b.buttonState === 'ADD_TO_CART'; }) : true,
image: obj.primaryImage ? obj.primaryImage.piscesHref : null
});
}
}
return products.length > 0 ? JSON.stringify(products) : null;
} catch(e) { return null; }
"""
try:
apollo_json = browser.driver.execute_script(apollo_cache_script)
if apollo_json:
apollo_products = json.loads(apollo_json)
logger.info(f"Extracted {len(apollo_products)} products from Apollo client cache")
for item in apollo_products:
if not item.get('name') or len(item.get('name', '')) < 5:
continue
sku_id = str(item.get('skuId', ''))
pdp = item.get('pdp')
url = pdp if pdp else (f"{self.base_url}/site/{sku_id}.p" if sku_id else None)
if not url:
continue
price = item.get('price')
products.append(Product(
name=item['name'],
url=url,
price=f"${price:.2f}" if isinstance(price, (int, float)) else str(price) if price else None,
in_stock=item.get('inStock', True),
image_url=item.get('image'),
site=self.site_name,
product_id=sku_id,
))
except Exception as e:
logger.debug(f"Could not extract from Apollo client cache: {e}")
# Method 4: Try multiple JS state sources via browser execution
if not products:
state_scripts = [
"return window.__NEXT_DATA__ ? JSON.stringify(window.__NEXT_DATA__) : null",
@@ -428,21 +524,18 @@ class BestBuyScraper(BaseScraper):
if image_matches:
image_url = image_matches[0]
# Try to determine stock status from Apollo data context
in_stock = True # Default to True if no stock info found
if url_pos > 0:
# Look for availability info near this product's URL
context_start = max(0, url_pos - 3000)
context_end = min(len(script_content), url_pos + 500)
product_context = script_content[context_start:context_end]
# Check for explicit out of stock indicators
if '"isAvailable":false' in product_context or '"available":false' in product_context:
in_stock = False
elif '"soldOut":true' in product_context or '"outOfStock":true' in product_context:
in_stock = False
elif 'sold out' in product_context.lower() or 'out of stock' in product_context.lower():
in_stock = False
# Determine stock status by anchoring to the SKU ID in the Apollo data,
# then searching for buttonState within 500 chars after it.
# This is more reliable than looking near the URL (which bleeds across products).
in_stock = True
if sku_id:
sku_anchor = f'"skuId":"{sku_id}"'
sku_pos = script_content.find(sku_anchor)
if sku_pos >= 0:
sku_context = script_content[sku_pos:sku_pos + 500]
btn_match = re.search(r'"buttonState":"([^"]+)"', sku_context)
if btn_match:
in_stock = btn_match.group(1) in ("ADD_TO_CART", "PRE_ORDER", "CHECK_STORES")
if name and len(name) > 5:
products.append(Product(
@@ -789,12 +882,14 @@ class BestBuyScraper(BaseScraper):
break
parent = parent.find_parent()
# Check stock
# Check stock — "add to cart" takes priority over "unavailable"
in_stock = True
parent = link.find_parent()
for _ in range(5):
if parent:
text = parent.get_text().lower()
if any(phrase in text for phrase in ["add to cart", "add to bag", "pre-order"]):
break # purchasable — in_stock stays True
if any(phrase in text for phrase in ["sold out", "out of stock", "unavailable"]):
in_stock = False
break
@@ -818,6 +913,13 @@ class BestBuyScraper(BaseScraper):
"""Check if a product card indicates in-stock status"""
card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
# "Add to Cart" takes priority — a card can show "Unavailable" for in-store/delivery
# while still being purchasable online, so check purchasable state first.
if "add to cart" in card_text or "add to bag" in card_text:
return True
if "pre-order" in card_text:
return True
# Check for disabled add to cart button
add_btn = card.select_one(".add-to-cart-button")
if add_btn and "btn-disabled" in add_btn.get("class", []):
@@ -835,17 +937,6 @@ class BestBuyScraper(BaseScraper):
if phrase in card_text:
return False
in_stock_phrases = [
"add to cart",
"add to bag",
"available",
"in stock",
]
for phrase in in_stock_phrases:
if phrase in card_text:
return True
return True
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]: