Pokemon Stock Monitor - Initial commit
Chrome extension for PokemonCenter monitoring with Discord notifications. Includes Python scripts for Target monitoring. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
Target.com scraper
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from browser import get_browser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TargetScraper(BaseScraper):
|
||||
"""Scraper for Target.com"""
|
||||
|
||||
site_name = "target"
|
||||
base_url = "https://www.target.com"
|
||||
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
"""
|
||||
Scrape a Target search/category page for all products
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
browser = get_browser()
|
||||
products = []
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
page.screenshot(path="debug_target.png")
|
||||
logger.info("Saved debug screenshot to debug_target.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try to find product data in page scripts (Target uses React/hydration)
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
# Look for product data in the JSON
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
# Parse product links from the page
|
||||
product_links = soup.select("a[href*='/p/']")
|
||||
logger.info(f"Found {len(product_links)} product links")
|
||||
|
||||
# Group links by href and pick the best one (with aria-label or text)
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
logger.info(f"Found {len(href_to_links)} unique hrefs")
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
# Find the best link (one with aria-label or text content)
|
||||
best_link = None
|
||||
for link in links:
|
||||
aria = link.get("aria-label", "")
|
||||
text = link.get_text(strip=True)
|
||||
if aria or (text and len(text) > 10):
|
||||
best_link = link
|
||||
break
|
||||
if not best_link:
|
||||
best_link = links[0] # Fallback to first link
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
page.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Target category page: {e}")
|
||||
raise
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
logger.info(f"Scraped {len(unique_products)} unique products from Target")
|
||||
return unique_products
|
||||
|
||||
def _extract_products_from_json(self, data, depth=0) -> List[Product]:
|
||||
"""Recursively search JSON for product data"""
|
||||
products = []
|
||||
if depth > 10: # Prevent infinite recursion
|
||||
return products
|
||||
|
||||
if isinstance(data, dict):
|
||||
# Check if this looks like a product
|
||||
if "tcin" in data or ("title" in data and "price" in data):
|
||||
product = self._parse_product_json(data)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
# Recurse into nested objects
|
||||
for value in data.values():
|
||||
products.extend(self._extract_products_from_json(value, depth + 1))
|
||||
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
products.extend(self._extract_products_from_json(item, depth + 1))
|
||||
|
||||
return products
|
||||
|
||||
def _parse_product_json(self, data: dict) -> Optional[Product]:
|
||||
"""Parse a product from Target's JSON data"""
|
||||
try:
|
||||
name = data.get("title") or data.get("product_description", {}).get("title", "")
|
||||
if not name:
|
||||
return None
|
||||
|
||||
# Build URL
|
||||
tcin = data.get("tcin", "")
|
||||
slug = data.get("url_slug", name.lower().replace(" ", "-"))
|
||||
url = f"{self.base_url}/p/{slug}/-/A-{tcin}" if tcin else ""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
# Get price
|
||||
price_data = data.get("price", {})
|
||||
price = None
|
||||
if isinstance(price_data, dict):
|
||||
price = price_data.get("formatted_current_price") or price_data.get("current_retail")
|
||||
elif isinstance(price_data, (int, float)):
|
||||
price = f"${price_data:.2f}"
|
||||
|
||||
# Check availability
|
||||
availability = data.get("availability_status", "")
|
||||
fulfillment = data.get("fulfillment", {})
|
||||
in_stock = availability not in ["OUT_OF_STOCK", "UNAVAILABLE"]
|
||||
if fulfillment:
|
||||
in_stock = fulfillment.get("is_out_of_stock_in_all_store_locations", True) is False
|
||||
|
||||
# Get image
|
||||
images = data.get("images", [])
|
||||
image_url = images[0].get("base_url") if images else None
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=str(price) if price else None,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=tcin,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Target product JSON: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_link(self, link, soup) -> Optional[Product]:
|
||||
"""Parse a product from a product link element"""
|
||||
try:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
return None
|
||||
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name - prefer aria-label as it's usually clean
|
||||
name = link.get("aria-label", "")
|
||||
|
||||
# If no aria-label, try link text
|
||||
if not name or len(name) < 5:
|
||||
name = link.get_text(strip=True)
|
||||
|
||||
# Clean up the name
|
||||
if name:
|
||||
# Remove rating text patterns
|
||||
name = re.sub(r'\d+\.?\d*\s*out of \d+ stars.*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*\d+ ratings?.*$', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r'\s*\d+ reviews?.*$', '', name, flags=re.IGNORECASE)
|
||||
name = name.strip()
|
||||
|
||||
# Skip empty/short names
|
||||
if not name or len(name) < 10:
|
||||
return None
|
||||
|
||||
# Skip if name looks like navigation/rating text only
|
||||
lower_name = name.lower()
|
||||
if any(lower_name.startswith(skip) for skip in ['rating', 'stars', 'review', 'filter', 'sort']):
|
||||
return None
|
||||
|
||||
# Extract product ID from URL (A-12345678)
|
||||
product_id = ""
|
||||
match = re.search(r"/A-(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
# Try to find price near this link
|
||||
parent = link.find_parent()
|
||||
price = None
|
||||
for _ in range(5): # Go up 5 levels max
|
||||
if parent:
|
||||
price_text = parent.get_text()
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
# Try to find image
|
||||
image_url = None
|
||||
img = link.select_one("img")
|
||||
if img:
|
||||
image_url = img.get("src") or img.get("data-src")
|
||||
|
||||
# Check stock (assume in stock unless we see otherwise)
|
||||
in_stock = True
|
||||
parent = link.find_parent()
|
||||
for _ in range(5):
|
||||
if parent:
|
||||
text = parent.get_text().lower()
|
||||
if "out of stock" in text or "unavailable" in text:
|
||||
in_stock = False
|
||||
break
|
||||
parent = parent.find_parent()
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Target product link: {e}")
|
||||
return None
|
||||
|
||||
def _parse_product_card(self, card) -> Optional[Product]:
|
||||
"""Parse a product card element"""
|
||||
try:
|
||||
# Find link
|
||||
link = card.select_one("a[href*='/p/']")
|
||||
if not link:
|
||||
return None
|
||||
|
||||
href = link.get("href", "")
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Get name
|
||||
name_elem = card.select_one("[data-test='product-title']") or card.select_one("a")
|
||||
name = name_elem.get_text(strip=True) if name_elem else "Unknown"
|
||||
|
||||
# Get price
|
||||
price_elem = card.select_one("[data-test='current-price']") or card.select_one("[class*='price']")
|
||||
price = None
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
# Check stock
|
||||
card_text = card.get_text().lower()
|
||||
out_of_stock = "out of stock" in card_text or "unavailable" in card_text
|
||||
in_stock = not out_of_stock
|
||||
|
||||
# Get image
|
||||
img = card.select_one("img")
|
||||
image_url = img.get("src") if img else None
|
||||
|
||||
# Extract product ID from URL
|
||||
product_id = ""
|
||||
match = re.search(r"/A-(\d+)", url)
|
||||
if match:
|
||||
product_id = match.group(1)
|
||||
|
||||
return Product(
|
||||
name=name,
|
||||
url=url,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
site=self.site_name,
|
||||
product_id=product_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing Target product card: {e}")
|
||||
return None
|
||||
|
||||
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
||||
"""Check if a specific product is in stock"""
|
||||
browser = get_browser()
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(product_url, timeout=30000)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
page_text = soup.get_text().lower()
|
||||
|
||||
out_of_stock = any(
|
||||
phrase in page_text
|
||||
for phrase in ["out of stock", "unavailable", "not available"]
|
||||
)
|
||||
|
||||
in_stock = not out_of_stock
|
||||
|
||||
# Get price
|
||||
price = None
|
||||
price_elem = soup.select_one("[data-test='product-price']")
|
||||
if price_elem:
|
||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
||||
if price_match:
|
||||
price = price_match.group()
|
||||
|
||||
page.close()
|
||||
return in_stock, price
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Target product stock: {e}")
|
||||
return False, None
|
||||
Reference in New Issue
Block a user