\ No newline at end of file
diff --git a/debug_gamestop.html b/debug_gamestop.html
new file mode 100644
index 0000000..7eaae7b
--- /dev/null
+++ b/debug_gamestop.html
@@ -0,0 +1 @@
+Just a moment...
www.gamestop.com
Performing security verification
This website uses a security service to protect against malicious bots. This page is displayed while the website verifies you are not a bot.
Verification successful. Waiting for www.gamestop.com to respond
\ No newline at end of file
diff --git a/discord_notifier.py b/discord_notifier.py
deleted file mode 100644
index 8f30c19..0000000
--- a/discord_notifier.py
+++ /dev/null
@@ -1,183 +0,0 @@
-"""
-Discord webhook notifications for stock alerts
-Sends rich embeds with product info and direct links
-"""
-
-import logging
-import requests
-from datetime import datetime
-from typing import Optional
-from config import DISCORD_WEBHOOK_URL
-
-logger = logging.getLogger(__name__)
-
-# Colors for different notification types
-COLOR_RESTOCK = 0x00FF00 # Green - item back in stock
-COLOR_NEW_DROP = 0x0099FF # Blue - new product listing
-COLOR_PREORDER = 0xFFAA00 # Orange - pre-order available
-COLOR_ERROR = 0xFF0000 # Red - error notification
-
-# Site icons/emojis
-SITE_EMOJIS = {
- "pokemoncenter": "\U0001F7E1", # Yellow circle
- "target": "\U0001F534", # Red circle
- "walmart": "\U0001F535", # Blue circle
- "bestbuy": "\U0001F7E1", # Yellow circle
-}
-
-
-def send_stock_alert(
- product_name: str,
- product_url: str,
- price: str,
- site: str,
- alert_type: str = "restock",
- image_url: Optional[str] = None,
-):
- """
- Send a Discord notification for a stock alert
-
- Args:
- product_name: Name of the product
- product_url: Direct link to the product
- price: Price string (e.g., "$49.99")
- site: Site name (pokemoncenter, target, etc.)
- alert_type: "restock", "new_drop", or "preorder"
- image_url: Optional product image URL
- """
- if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
- logger.error("Discord webhook URL not configured! Update config.py")
- return False
-
- # Choose color based on alert type
- if alert_type == "restock":
- color = COLOR_RESTOCK
- title = f"\U0001F6A8 RESTOCK ALERT"
- elif alert_type == "new_drop":
- color = COLOR_NEW_DROP
- title = f"\U0001F195 NEW DROP"
- elif alert_type == "preorder":
- color = COLOR_PREORDER
- title = f"\u23F0 PRE-ORDER AVAILABLE"
- else:
- color = COLOR_RESTOCK
- title = f"\U0001F514 STOCK ALERT"
-
- site_emoji = SITE_EMOJIS.get(site.lower(), "\U0001F6D2")
- site_display = site.replace("pokemoncenter", "Pokemon Center").title()
-
- # Build the embed
- embed = {
- "title": title,
- "description": f"**{product_name}**",
- "url": product_url,
- "color": color,
- "fields": [
- {"name": "Price", "value": price or "See link", "inline": True},
- {"name": "Store", "value": f"{site_emoji} {site_display}", "inline": True},
- {"name": "Link", "value": f"[\U0001F6D2 BUY NOW]({product_url})", "inline": False},
- ],
- "footer": {"text": "Pokemon Stock Monitor"},
- "timestamp": datetime.utcnow().isoformat(),
- }
-
- if image_url:
- embed["thumbnail"] = {"url": image_url}
-
- payload = {
- "content": "@everyone", # Ping everyone
- "embeds": [embed],
- }
-
- try:
- response = requests.post(
- DISCORD_WEBHOOK_URL,
- json=payload,
- timeout=10,
- )
- response.raise_for_status()
- logger.info(f"Discord notification sent for: {product_name}")
- return True
- except requests.exceptions.RequestException as e:
- logger.error(f"Failed to send Discord notification: {e}")
- return False
-
-
-def send_error_notification(error_message: str, site: str = "Unknown"):
- """Send an error notification to Discord"""
- if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
- return False
-
- embed = {
- "title": "\u26A0\uFE0F Monitor Error",
- "description": error_message,
- "color": COLOR_ERROR,
- "fields": [{"name": "Site", "value": site, "inline": True}],
- "footer": {"text": "Pokemon Stock Monitor"},
- "timestamp": datetime.utcnow().isoformat(),
- }
-
- payload = {"embeds": [embed]}
-
- try:
- response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
- response.raise_for_status()
- return True
- except requests.exceptions.RequestException as e:
- logger.error(f"Failed to send error notification: {e}")
- return False
-
-
-def send_startup_notification():
- """Send a notification that the monitor has started"""
- if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
- return False
-
- embed = {
- "title": "\u2705 Monitor Started",
- "description": "Pokemon Stock Monitor is now running and watching for restocks!",
- "color": 0x00FF00,
- "footer": {"text": "Pokemon Stock Monitor"},
- "timestamp": datetime.utcnow().isoformat(),
- }
-
- payload = {"embeds": [embed]}
-
- try:
- response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
- response.raise_for_status()
- return True
- except requests.exceptions.RequestException as e:
- logger.error(f"Failed to send startup notification: {e}")
- return False
-
-
-def test_webhook():
- """Test the Discord webhook connection"""
- print("Testing Discord webhook...")
-
- if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
- print("ERROR: Webhook URL not configured!")
- print("Edit config.py and set DISCORD_WEBHOOK_URL")
- return False
-
- # Send a test notification
- success = send_stock_alert(
- product_name="Test Product - Pokemon TCG Booster",
- product_url="https://www.pokemoncenter.com/test",
- price="$4.99",
- site="pokemoncenter",
- alert_type="restock",
- )
-
- if success:
- print("SUCCESS! Check your Discord channel for the test message.")
- else:
- print("FAILED! Check the webhook URL and try again.")
-
- return success
-
-
-if __name__ == "__main__":
- # Run webhook test
- test_webhook()
diff --git a/docs/BOT_DETECTION_STRATEGIES.md b/docs/BOT_DETECTION_STRATEGIES.md
new file mode 100644
index 0000000..bb32791
--- /dev/null
+++ b/docs/BOT_DETECTION_STRATEGIES.md
@@ -0,0 +1,189 @@
+# Pokemon Center Bot Detection - Strategies & Solutions
+
+Pokemon Center uses **Imperva (Incapsula)** bot protection, which is one of the most aggressive anti-bot systems. Here are practical approaches to work around it.
+
+## Current Problem
+- Direct API calls: **403 Forbidden**
+- Sitemap.xml: **Blocked by Imperva**
+- Browser automation: **CAPTCHA after a few requests**
+
+---
+
+## Strategy 1: Third-Party Stock Trackers (Easiest)
+
+Instead of scraping Pokemon Center directly, integrate with existing stock tracking services:
+
+### Discord Bots/Webhooks
+Several Discord servers track Pokemon Center in real-time:
+- **Pokemon TCG Drops** - Dedicated Pokemon TCG stock alerts
+- **Stock Informer** - General retail stock tracking
+- Search Discord for "Pokemon Center stock alerts"
+
+You can join these servers and set up webhook forwarding to your own Discord.
+
+### Stock Alert Services
+- **NowInStock.net** - Has Pokemon Center tracking
+- **Distill.io** - Browser extension for change detection
+- **Visualping** - Monitors page changes
+
+### Implementation
+```python
+# Forward alerts from a public tracker to your system
+# Set up a Discord bot to listen to stock alert channels
+# Then forward to your own notification system
+```
+
+---
+
+## Strategy 2: Undetected Chrome Driver
+
+Use `undetected-chromedriver` which patches Chrome to avoid detection:
+
+```bash
+pip install undetected-chromedriver
+```
+
+```python
+import undetected_chromedriver as uc
+
+driver = uc.Chrome(headless=False) # Headless often gets detected
+driver.get("https://www.pokemoncenter.com/category/tcg-cards")
+
+# Add human-like delays
+import time
+import random
+time.sleep(random.uniform(3, 7))
+
+# Scroll like a human
+driver.execute_script("window.scrollBy(0, 500)")
+time.sleep(random.uniform(1, 3))
+```
+
+**Pros:** Often bypasses Imperva
+**Cons:** Slower, still may trigger CAPTCHA eventually
+
+---
+
+## Strategy 3: Session Persistence
+
+Keep a browser session alive and logged in:
+
+1. **Manual login once** - Complete any CAPTCHA manually
+2. **Save cookies** - Export session cookies
+3. **Reuse session** - Load cookies on each check
+4. **Longer intervals** - Check every 3-5 minutes instead of 60-90 seconds
+
+```python
+# Save cookies after manual login
+import pickle
+pickle.dump(driver.get_cookies(), open("cookies.pkl", "wb"))
+
+# Load cookies on subsequent runs
+cookies = pickle.load(open("cookies.pkl", "rb"))
+for cookie in cookies:
+ driver.add_cookie(cookie)
+```
+
+---
+
+## Strategy 4: Residential Proxy Rotation
+
+Use residential proxies that look like real home internet connections:
+
+### Providers
+- **Bright Data** (formerly Luminati) - Best but expensive
+- **Oxylabs** - Good quality
+- **Smartproxy** - Budget option
+- **IPRoyal** - Pay-per-GB
+
+### Cost
+- ~$10-15/GB for residential proxies
+- ~50-100 requests per MB depending on page size
+
+```python
+proxies = {
+ "http": "http://user:pass@proxy.provider.com:port",
+ "https": "http://user:pass@proxy.provider.com:port"
+}
+```
+
+---
+
+## Strategy 5: Human-Like Behavior Pattern
+
+If continuing with browser automation:
+
+```python
+import random
+import time
+
+def human_like_check():
+ # Random delay between checks (2-5 minutes)
+ delay = random.uniform(120, 300)
+
+ # Random user agents
+ user_agents = [
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...",
+ ]
+
+ # Vary timing throughout the day
+ # Check more during business hours, less at night
+ hour = datetime.now().hour
+ if 2 <= hour <= 6: # 2am-6am
+ delay *= 2 # Slower at night
+
+ # Add random mouse movements
+ # Add random scrolling
+ # Occasionally visit other pages (home, about)
+
+ return delay
+```
+
+---
+
+## Strategy 6: Webhook from Manual Monitoring
+
+The most reliable approach for Pokemon Center:
+
+1. **Open Pokemon Center in your actual browser**
+2. **Use Distill.io browser extension** to monitor changes
+3. **Set up webhook** to forward alerts to your system
+
+This way:
+- You're using a real browser with real cookies
+- Imperva sees normal human behavior
+- Change detection triggers your notification system
+
+---
+
+## Recommended Approach
+
+Given Pokemon Center's aggressive protection, I recommend a **hybrid approach**:
+
+1. **Primary: Join existing stock alert Discord servers**
+ - Let others deal with the bot detection
+ - Forward their alerts to your system
+
+2. **Secondary: Use undetected-chromedriver with long intervals**
+ - Check every 3-5 minutes
+ - Use session persistence
+ - Add human-like behavior
+
+3. **Backup: Manual Distill.io monitoring**
+ - For specific products you really care about
+ - Most reliable but requires keeping browser open
+
+---
+
+## Code Changes Needed
+
+To implement these strategies, we would need to:
+
+1. Add `undetected-chromedriver` as an option
+2. Implement cookie/session persistence
+3. Add configurable random delays
+4. Add proxy support
+5. Consider adding Discord bot listener for third-party alerts
+
+Would you like me to implement any of these strategies?
diff --git a/docs/NOTES.md b/docs/NOTES.md
new file mode 100644
index 0000000..0a98e7a
--- /dev/null
+++ b/docs/NOTES.md
@@ -0,0 +1,13 @@
+# Pokemon Stock Monitor - Notes
+
+## Bot Detection / CAPTCHA Log
+
+| Date | Time | Site | Notes |
+|------|------|------|-------|
+| 2026-03-25 | ~10:30 AM | PokemonCenter | CAPTCHAs appeared, bot protection triggered |
+
+## Tips
+- Pokemon Center uses Imperva bot protection
+- CAPTCHAs more likely during high traffic (product drops)
+- Using Chrome extension with real browser avoids most detection
+- If blocked, clear cookies and wait before retrying
diff --git a/docs/POKEMON_CENTER_DROPS_NOTES.md b/docs/POKEMON_CENTER_DROPS_NOTES.md
new file mode 100644
index 0000000..77f8948
--- /dev/null
+++ b/docs/POKEMON_CENTER_DROPS_NOTES.md
@@ -0,0 +1,46 @@
+# Pokemon Center Drop Strategy Notes
+
+Source: https://www.pokepullz.ca/blog/pokemon-center-drops-guide.html
+
+## Queue System
+- Pokemon Center uses a **randomized virtual queue** that activates before major drops
+- Position is NOT based on when you arrived - it's assigned randomly once queue activates
+- This prevents site crashes and distributes access fairly
+
+## Timing Signals
+- Security changes on the Pokemon Center site serve as early warning signals that drops are imminent
+- Products sometimes load in waves rather than all at once
+
+## Multi-Device Strategy
+1. Join queue on primary device (desktop on WiFi) first
+2. Wait a few minutes, then join on second device (phone on mobile data)
+3. This gives you two separate queue positions and accounts for staggered product loading
+
+## Technical Tips
+
+### The Cart Trick
+When Add to Cart button isn't displaying during glitchy drops:
+1. Click the cart icon to go to your cart page
+2. Hit browser back button to return to product page
+3. This forces a page reload that may show the button
+
+## Order Flagging Risks (Avoid These!)
+- Typos in billing/shipping information
+- Mismatched address data
+- VPNs or browser extensions
+- Multiple rapid orders
+- Overloaded carts
+- Queue-checking tools or auto-refreshers
+
+## Pre-Drop Prep Checklist
+- [ ] Log into account days ahead
+- [ ] Have devices ready
+- [ ] Monitor Discord alerts
+- [ ] Disable browser extensions
+- [ ] Turn off VPN
+
+## Implications for Our Monitor
+1. **Don't use VPN/proxies during actual purchases** - only for monitoring
+2. **Queue position is random** - early detection still helps you GET in queue early
+3. **Watch for security changes** - could indicate imminent drop
+4. **Multi-wave loading** - may need multiple checks to catch all products
diff --git a/docs/SMART_MONITOR_DESIGN.md b/docs/SMART_MONITOR_DESIGN.md
new file mode 100644
index 0000000..a85b44c
--- /dev/null
+++ b/docs/SMART_MONITOR_DESIGN.md
@@ -0,0 +1,199 @@
+# Smart Monitor Design Document
+
+## Overview
+Design for an adaptive monitoring system that mimics human behavior and automatically switches modes based on detected activity.
+
+---
+
+## Monitoring Modes
+
+### 1. STEALTH MODE (Default)
+- **Check interval**: 45-90 seconds (randomized around 1 minute)
+- **Behavior**: Human-like patterns, decoy requests
+- **Purpose**: Avoid detection during normal monitoring
+
+### 2. ALERT MODE (Triggered by new SKU)
+- **Check interval**: 15-30 seconds (faster, still randomized)
+- **Duration**: 30-60 minutes after detection
+- **Purpose**: Catch the drop going live quickly
+
+### 3. COOLDOWN MODE (After alert period)
+- **Gradually slow back down** to stealth mode
+- **Prevents sudden behavior change** which could trigger detection
+
+### 4. SLEEP MODE (Optional - night hours)
+- **Check interval**: 5-10 minutes
+- **Active hours**: e.g., 2am-6am local time
+- **Purpose**: Real humans sleep, bots don't
+
+---
+
+## Human-Like Obfuscation Techniques
+
+### Timing Randomization
+```python
+# Instead of exact intervals:
+time.sleep(60) # BAD - robotic
+
+# Use gaussian distribution around target:
+import random
+base_interval = 60
+jitter = random.gauss(0, 10) # +/- 10 seconds standard deviation
+time.sleep(max(30, base_interval + jitter)) # GOOD - human-like
+```
+
+### Decoy Requests
+Mix in non-API requests to look like a real browser:
+- Occasionally fetch an image from the site
+- Load the homepage or a random category page
+- Request favicon, robots.txt, etc.
+
+```python
+DECOY_URLS = [
+ "/favicon.ico",
+ "/",
+ "/category/plush",
+ "/category/accessories",
+]
+
+def make_decoy_request():
+ """Occasionally make a non-API request"""
+ if random.random() < 0.2: # 20% of the time
+ url = random.choice(DECOY_URLS)
+ session.get(API_BASE + url)
+```
+
+### Referrer Rotation
+Change the Referer header to look like you're browsing:
+```python
+REFERRERS = [
+ "https://www.pokemoncenter.com/",
+ "https://www.pokemoncenter.com/category/tcg-cards",
+ "https://www.pokemoncenter.com/category/new-releases",
+ "https://www.google.com/",
+]
+
+headers["Referer"] = random.choice(REFERRERS)
+```
+
+### User-Agent Variation
+Slightly vary the User-Agent (within reason):
+```python
+# Base UA with minor variations
+chrome_versions = ["146.0.0.0", "145.0.0.0", "146.0.7680.154"]
+version = random.choice(chrome_versions)
+headers["User-Agent"] = f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{version} Safari/537.36"
+```
+
+### Session Breaks
+Occasionally "take a break" like a human would:
+```python
+def maybe_take_break():
+ """Occasionally pause for a longer period"""
+ if random.random() < 0.05: # 5% chance
+ break_duration = random.randint(120, 300) # 2-5 minutes
+ logger.info(f"Taking a break for {break_duration}s...")
+ time.sleep(break_duration)
+```
+
+### Time-of-Day Awareness
+Adjust behavior based on time:
+```python
+from datetime import datetime
+
+def get_interval_for_time():
+ hour = datetime.now().hour
+
+ if 2 <= hour < 6:
+ # Late night - slower checks
+ return random.randint(300, 600) # 5-10 min
+ elif 9 <= hour < 17:
+ # Business hours - prime drop time
+ return random.randint(45, 90) # ~1 min
+ else:
+ # Evening/early morning
+ return random.randint(60, 120) # 1-2 min
+```
+
+---
+
+## Mode Transition Logic
+
+```
+ ┌─────────────────┐
+ │ STEALTH MODE │
+ │ (45-90 sec) │
+ └────────┬────────┘
+ │
+ New SKU detected?
+ │
+ YES ───────────┴─────────── NO
+ │ │
+ ▼ │
+ ┌─────────────────┐ │
+ │ ALERT MODE │ │
+ │ (15-30 sec) │◄──────────────────┘
+ └────────┬────────┘
+ │
+ 30-60 min elapsed?
+ │
+ YES │
+ ▼
+ ┌─────────────────┐
+ │ COOLDOWN MODE │
+ │ (gradually │
+ │ slow down) │
+ └────────┬────────┘
+ │
+ ▼
+ Back to STEALTH MODE
+```
+
+---
+
+## Detection Tracking
+
+When a new product is detected:
+
+1. **Log detection** with timestamp
+2. **Fetch full product details**
+3. **Send Discord alert**
+4. **Enter ALERT MODE**
+5. **Track if product becomes "announced"**:
+ - Visible on homepage
+ - Linked from category pages
+ - Social media announcement
+
+When product is announced, return to STEALTH MODE.
+
+---
+
+## Implementation Priority
+
+1. [x] Basic backend monitor (done)
+2. [ ] Add timing randomization
+3. [ ] Implement mode switching
+4. [ ] Add decoy requests
+5. [ ] Add time-of-day awareness
+6. [ ] Add session breaks
+7. [ ] Integrate with Discord notifier
+
+---
+
+## Testing Plan
+
+1. **Test with cookies**: Run warmup, verify API access works
+2. **Test SKU detection**: Manually add a fake SKU to known list, verify detection
+3. **Test mode switching**: Simulate detection, verify faster checks activate
+4. **Monitor for blocks**: Run for extended period, note when cookies expire
+
+---
+
+## Risk Mitigation
+
+| Risk | Mitigation |
+|------|------------|
+| Cookies expire | Auto-detect 403, prompt for re-warmup |
+| IP blocked | Support proxy rotation (already built) |
+| Pattern detected | Randomization + decoys + breaks |
+| Rate limited | Back off on 429 responses |
diff --git a/docs/backend-api-monitor.md b/docs/backend-api-monitor.md
new file mode 100644
index 0000000..373708a
--- /dev/null
+++ b/docs/backend-api-monitor.md
@@ -0,0 +1,285 @@
+# PokemonCenter Backend API Monitor
+
+## Overview
+
+This document outlines how to build a backend API monitor that detects new products before they appear on the website - similar to what accounts like @pokepullzhq do.
+
+## Why Backend Monitoring?
+
+- **Faster detection**: Products are loaded into the backend/API before the frontend displays them
+- **Less aggressive**: API calls are lighter than full page loads
+- **Avoids bot detection**: Direct API calls look different than browser automation
+- **More reliable**: JSON responses are easier to parse than HTML
+
+---
+
+## Step 1: Find the API Endpoints
+
+### How to Investigate
+
+1. Open https://www.pokemoncenter.com/category/tcg-cards in Chrome
+2. Open DevTools (F12)
+3. Go to **Network** tab
+4. Check **Preserve log**
+5. Filter by **Fetch/XHR**
+6. Refresh the page
+7. Look for API calls that return product data
+
+### What to Look For
+
+**Common patterns:**
+- GraphQL endpoints: `/graphql` or `/api/graphql`
+- REST APIs: `/api/products`, `/api/catalog`, `/api/search`
+- Third-party services:
+ - **Algolia** (search): `*.algolia.net` or `*.algolianet.com`
+ - **Contentful** (CMS): `cdn.contentful.com`
+ - **Commercetools** (e-commerce): `*.commercetools.com`
+ - **Salesforce Commerce**: `*.demandware.net`
+
+**Signs you found the right endpoint:**
+- Response contains product names, prices, SKUs
+- Response has `inStock`, `availability`, or similar fields
+- Response includes product URLs or IDs
+
+### Example Findings to Document
+
+For each endpoint found, note:
+```
+URL: https://api.pokemoncenter.com/products?category=tcg
+Method: GET
+Headers:
+ - Authorization: Bearer xxx (if any)
+ - x-api-key: xxx (if any)
+Response format: JSON
+Contains: productId, name, price, availability, url
+```
+
+---
+
+## Step 2: API Response Analysis
+
+Once you find the product API, analyze the response structure:
+
+### Key Fields to Track
+
+```json
+{
+ "products": [
+ {
+ "id": "12345", // Unique product ID
+ "sku": "PKM-CR-ETB", // SKU code
+ "name": "Chaos Rising ETB",
+ "url": "/product/chaos-rising-etb",
+ "price": 54.99,
+ "availability": {
+ "inStock": false, // Current stock status
+ "preorder": true, // Pre-order available?
+ "releaseDate": "2026-04-15"
+ },
+ "status": "ACTIVE", // May be "HIDDEN", "DRAFT" before launch
+ "publishedAt": null // null = not yet visible on site
+ }
+ ]
+}
+```
+
+### Detection Strategies
+
+1. **New Product Detection**
+ - Compare product IDs against known list
+ - New ID = new product added to backend
+
+2. **Pre-Launch Detection**
+ - Product exists but `publishedAt` is null
+ - Product has `status: "DRAFT"` or similar
+ - Product `availability.inStock` changes from false to true
+
+3. **Stock Change Detection**
+ - Track `inStock` or `availability` field changes
+
+---
+
+## Step 3: Implementation Plan
+
+### Option A: Add to Chrome Extension
+
+```javascript
+// In background.js - add API monitoring alongside page monitoring
+
+async function checkBackendAPI() {
+ const API_URL = "https://api.pokemoncenter.com/products?category=tcg";
+
+ try {
+ const response = await fetch(API_URL, {
+ headers: {
+ // Add any required headers discovered during investigation
+ 'Accept': 'application/json',
+ }
+ });
+
+ const data = await response.json();
+ const products = data.products || [];
+
+ // Compare against known products
+ for (const product of products) {
+ if (!knownBackendProducts[product.id]) {
+ // NEW PRODUCT DETECTED!
+ await sendDiscordNotification({
+ name: product.name,
+ url: `https://www.pokemoncenter.com${product.url}`,
+ price: `$${product.price}`,
+ inStock: product.availability?.inStock
+ }, "backend_detect");
+
+ knownBackendProducts[product.id] = product;
+ }
+ }
+ } catch (error) {
+ console.error("Backend API check failed:", error);
+ }
+}
+```
+
+### Option B: Standalone Python Script
+
+```python
+# backend_monitor.py
+import requests
+import time
+import json
+from discord_webhook import DiscordWebhook, DiscordEmbed
+
+API_URL = "https://api.pokemoncenter.com/products"
+DISCORD_WEBHOOK = "your-webhook-url"
+KNOWN_PRODUCTS_FILE = "known_backend_products.json"
+CHECK_INTERVAL = 30 # seconds - can be faster for API
+
+def load_known_products():
+ try:
+ with open(KNOWN_PRODUCTS_FILE, 'r') as f:
+ return json.load(f)
+ except FileNotFoundError:
+ return {}
+
+def save_known_products(products):
+ with open(KNOWN_PRODUCTS_FILE, 'w') as f:
+ json.dump(products, f)
+
+def check_api():
+ headers = {
+ 'User-Agent': 'Mozilla/5.0...',
+ # Add discovered headers
+ }
+
+ response = requests.get(API_URL, headers=headers)
+ return response.json()
+
+def send_alert(product, alert_type):
+ webhook = DiscordWebhook(url=DISCORD_WEBHOOK, content="@everyone")
+
+ embed = DiscordEmbed(
+ title=f"🚨 {alert_type.upper()}",
+ description=f"**{product['name']}**",
+ color=0xFF0000 if alert_type == "BACKEND DETECT" else 0x00FF00
+ )
+ embed.add_embed_field(name="Price", value=f"${product.get('price', 'TBD')}")
+ embed.add_embed_field(name="Status", value=product.get('status', 'Unknown'))
+ embed.add_embed_field(name="Link", value=f"[VIEW]({product['url']})", inline=False)
+
+ webhook.add_embed(embed)
+ webhook.execute()
+
+def main():
+ known = load_known_products()
+ print(f"Loaded {len(known)} known products")
+
+ while True:
+ try:
+ data = check_api()
+ products = data.get('products', [])
+
+ for product in products:
+ pid = product['id']
+
+ if pid not in known:
+ print(f"NEW BACKEND PRODUCT: {product['name']}")
+ send_alert(product, "BACKEND DETECT")
+ known[pid] = product
+ save_known_products(known)
+
+ # Check for status changes
+ elif known[pid].get('status') != product.get('status'):
+ print(f"STATUS CHANGE: {product['name']}")
+ send_alert(product, "STATUS CHANGE")
+ known[pid] = product
+ save_known_products(known)
+
+ print(f"Checked {len(products)} products")
+
+ except Exception as e:
+ print(f"Error: {e}")
+
+ time.sleep(CHECK_INTERVAL)
+
+if __name__ == "__main__":
+ main()
+```
+
+---
+
+## Step 4: Rate Limiting Considerations
+
+### API vs Page Scraping
+
+| Approach | Safe Interval | Notes |
+|----------|---------------|-------|
+| Full page load | 60-90 sec | Heavy, triggers bot detection |
+| API call | 15-30 sec | Lighter, more tolerant |
+| GraphQL query | 15-30 sec | Depends on complexity |
+
+### Best Practices
+
+1. **Use proper headers**: Include realistic User-Agent, Accept, etc.
+2. **Don't hammer**: Even APIs have rate limits
+3. **Cache responses**: Don't re-process unchanged data
+4. **Handle 429s gracefully**: Back off when rate limited
+
+---
+
+## Step 5: Advanced - Multiple Detection Layers
+
+For maximum coverage, run both:
+
+1. **Backend API Monitor** (every 30 sec)
+ - Fast detection of new products in system
+ - Lighter on resources
+
+2. **Frontend Page Monitor** (every 90 sec)
+ - Confirms products are live on website
+ - Catches anything API might miss
+
+### Alert Priority
+
+```
+BACKEND DETECT (API) = "Product loaded, drop imminent!"
+FRONTEND DETECT (Page) = "Product is LIVE, buy now!"
+RESTOCK = "Back in stock!"
+```
+
+---
+
+## Next Steps
+
+1. **Wait for rate limit to clear** (~30 min)
+2. **Investigate the API** using DevTools Network tab
+3. **Document the endpoints** you find
+4. **Share the findings** so we can build the monitor
+
+### Questions to Answer During Investigation
+
+- [ ] What URL serves product data?
+- [ ] What headers are required?
+- [ ] Is authentication needed?
+- [ ] What does the response structure look like?
+- [ ] Are there pagination parameters?
+- [ ] Is there a "hidden" or "draft" status visible in API?
diff --git a/main.py b/main.py
index f61e738..d9e0c6c 100644
--- a/main.py
+++ b/main.py
@@ -16,14 +16,25 @@ from config import (
SITES_ENABLED,
POKEMON_CENTER_URLS,
TARGET_URLS,
+ GAMESTOP_URLS,
+ BESTBUY_URLS,
+ WALMART_URLS,
KEYWORD_FILTER_ENABLED,
KEYWORDS,
LOG_LEVEL,
+ NEW_DROP_NOTIFICATIONS,
+ MAX_PAGES,
+)
+from src.browser import get_browser, shutdown_browser
+from src.product_tracker import ProductTracker
+from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
+from scrapers import (
+ PokemonCenterScraper,
+ TargetScraper,
+ GameStopScraper,
+ BestBuyScraper,
+ WalmartScraper,
)
-from browser import get_browser, shutdown_browser
-from product_tracker import ProductTracker
-from discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
-from scrapers import PokemonCenterScraper, TargetScraper
# Setup logging
logging.basicConfig(
@@ -43,6 +54,9 @@ tracker = ProductTracker()
scrapers = {
"pokemoncenter": PokemonCenterScraper(),
"target": TargetScraper(),
+ "gamestop": GameStopScraper(),
+ "bestbuy": BestBuyScraper(),
+ "walmart": WalmartScraper(),
}
@@ -54,7 +68,7 @@ def check_pokemoncenter():
try:
for url in POKEMON_CENTER_URLS:
logger.info(f"Scraping: {url}")
- products = scraper.scrape_category_page(url)
+ products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1))
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
@@ -64,18 +78,19 @@ def check_pokemoncenter():
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
- # Send notifications for new products
- for product in new_products:
- if product.in_stock:
- send_stock_alert(
- product_name=product.name,
- product_url=product.url,
- price=product.price or "See link",
- site="pokemoncenter",
- alert_type="new_drop",
- image_url=product.image_url,
- )
- logger.info(f"Sent notification for new product: {product.name}")
+ # Send notifications for new products (if enabled)
+ if NEW_DROP_NOTIFICATIONS:
+ for product in new_products:
+ if product.in_stock:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="pokemoncenter",
+ alert_type="new_drop",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
@@ -110,7 +125,10 @@ def check_target():
try:
for url in TARGET_URLS:
logger.info(f"Scraping: {url}")
- products = scraper.scrape_category_page(url)
+ products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("target", 1))
+
+ # Always filter to Pokemon products only
+ products = scraper.filter_pokemon_products(products)
# Apply keyword filter if enabled
if KEYWORD_FILTER_ENABLED and KEYWORDS:
@@ -120,18 +138,19 @@ def check_target():
# Process and detect changes
new_products, restocked_products = tracker.process_products(products)
- # Send notifications for new products
- for product in new_products:
- if product.in_stock:
- send_stock_alert(
- product_name=product.name,
- product_url=product.url,
- price=product.price or "See link",
- site="target",
- alert_type="new_drop",
- image_url=product.image_url,
- )
- logger.info(f"Sent notification for new product: {product.name}")
+ # Send notifications for new products (if enabled)
+ if NEW_DROP_NOTIFICATIONS:
+ for product in new_products:
+ if product.in_stock:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="target",
+ alert_type="new_drop",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for new product: {product.name}")
# Send notifications for restocks
for product in restocked_products:
@@ -158,6 +177,186 @@ def check_target():
send_error_notification(f"Error checking Target: {str(e)}", "Target")
+def check_gamestop():
+ """Check GameStop for restocks and new drops"""
+ logger.info("Checking GameStop...")
+ scraper = scrapers["gamestop"]
+
+ try:
+ for url in GAMESTOP_URLS:
+ logger.info(f"Scraping: {url}")
+ products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("gamestop", 1))
+
+ # Always filter to Pokemon products only
+ products = scraper.filter_pokemon_products(products)
+
+ # Apply keyword filter if enabled
+ if KEYWORD_FILTER_ENABLED and KEYWORDS:
+ products = scraper.filter_by_keywords(products, KEYWORDS)
+ logger.info(f"After keyword filter: {len(products)} products")
+
+ # Process and detect changes
+ new_products, restocked_products = tracker.process_products(products)
+
+ # Send notifications for new products (if enabled)
+ if NEW_DROP_NOTIFICATIONS:
+ for product in new_products:
+ if product.in_stock:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="gamestop",
+ alert_type="new_drop",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for new product: {product.name}")
+
+ # Send notifications for restocks
+ for product in restocked_products:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="gamestop",
+ alert_type="restock",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for restock: {product.name}")
+
+ # Log stats
+ stats = tracker.get_stats()
+ logger.info(
+ f"Stats: {stats['total_products']} total, "
+ f"{stats['in_stock']} in stock, "
+ f"{stats['out_of_stock']} out of stock"
+ )
+
+ except Exception as e:
+ logger.error(f"Error checking GameStop: {e}")
+ send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
+
+
+def check_bestbuy():
+ """Check Best Buy for restocks and new drops"""
+ logger.info("Checking Best Buy...")
+ scraper = scrapers["bestbuy"]
+
+ try:
+ for url in BESTBUY_URLS:
+ logger.info(f"Scraping: {url}")
+ products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("bestbuy", 1))
+
+ # Always filter to Pokemon products only
+ products = scraper.filter_pokemon_products(products)
+
+ # Apply keyword filter if enabled
+ if KEYWORD_FILTER_ENABLED and KEYWORDS:
+ products = scraper.filter_by_keywords(products, KEYWORDS)
+ logger.info(f"After keyword filter: {len(products)} products")
+
+ # Process and detect changes
+ new_products, restocked_products = tracker.process_products(products)
+
+ # Send notifications for new products (if enabled)
+ if NEW_DROP_NOTIFICATIONS:
+ for product in new_products:
+ if product.in_stock:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="bestbuy",
+ alert_type="new_drop",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for new product: {product.name}")
+
+ # Send notifications for restocks
+ for product in restocked_products:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="bestbuy",
+ alert_type="restock",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for restock: {product.name}")
+
+ # Log stats
+ stats = tracker.get_stats()
+ logger.info(
+ f"Stats: {stats['total_products']} total, "
+ f"{stats['in_stock']} in stock, "
+ f"{stats['out_of_stock']} out of stock"
+ )
+
+ except Exception as e:
+ logger.error(f"Error checking Best Buy: {e}")
+ send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
+
+
+def check_walmart():
+ """Check Walmart for restocks and new drops"""
+ logger.info("Checking Walmart...")
+ scraper = scrapers["walmart"]
+
+ try:
+ for url in WALMART_URLS:
+ logger.info(f"Scraping: {url}")
+ products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("walmart", 1))
+
+ # Always filter to Pokemon products only
+ products = scraper.filter_pokemon_products(products)
+
+ # Apply keyword filter if enabled
+ if KEYWORD_FILTER_ENABLED and KEYWORDS:
+ products = scraper.filter_by_keywords(products, KEYWORDS)
+ logger.info(f"After keyword filter: {len(products)} products")
+
+ # Process and detect changes
+ new_products, restocked_products = tracker.process_products(products)
+
+ # Send notifications for new products (if enabled)
+ if NEW_DROP_NOTIFICATIONS:
+ for product in new_products:
+ if product.in_stock:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="walmart",
+ alert_type="new_drop",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for new product: {product.name}")
+
+ # Send notifications for restocks
+ for product in restocked_products:
+ send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price or "See link",
+ site="walmart",
+ alert_type="restock",
+ image_url=product.image_url,
+ )
+ logger.info(f"Sent notification for restock: {product.name}")
+
+ # Log stats
+ stats = tracker.get_stats()
+ logger.info(
+ f"Stats: {stats['total_products']} total, "
+ f"{stats['in_stock']} in stock, "
+ f"{stats['out_of_stock']} out of stock"
+ )
+
+ except Exception as e:
+ logger.error(f"Error checking Walmart: {e}")
+ send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
+
+
def run_checks():
"""Run all enabled site checks"""
logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
@@ -168,6 +367,15 @@ def run_checks():
if "target" in SITES_ENABLED:
check_target()
+ if "gamestop" in SITES_ENABLED:
+ check_gamestop()
+
+ if "bestbuy" in SITES_ENABLED:
+ check_bestbuy()
+
+ if "walmart" in SITES_ENABLED:
+ check_walmart()
+
logger.info("Check cycle complete")
diff --git a/product_tracker.py b/product_tracker.py
deleted file mode 100644
index 18c7f79..0000000
--- a/product_tracker.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""
-Product tracker - keeps track of known products to detect new drops and restocks
-"""
-
-import json
-import logging
-from pathlib import Path
-from typing import Dict, List, Set, Optional
-from dataclasses import asdict
-from datetime import datetime
-
-from scrapers.base import Product
-
-logger = logging.getLogger(__name__)
-
-# File to store known products
-PRODUCTS_FILE = Path(__file__).parent / "products.json"
-
-
-class ProductTracker:
- """Tracks known products to detect new listings and stock changes"""
-
- def __init__(self, products_file: Path = PRODUCTS_FILE):
- self.products_file = products_file
- self.products: Dict[str, dict] = {} # URL -> product data
- self.load()
-
- def load(self):
- """Load products from file"""
- if self.products_file.exists():
- try:
- with open(self.products_file, "r", encoding="utf-8") as f:
- self.products = json.load(f)
- logger.info(f"Loaded {len(self.products)} tracked products")
- except (json.JSONDecodeError, IOError) as e:
- logger.error(f"Error loading products file: {e}")
- self.products = {}
- else:
- self.products = {}
- logger.info("No existing products file, starting fresh")
-
- def save(self):
- """Save products to file"""
- try:
- with open(self.products_file, "w", encoding="utf-8") as f:
- json.dump(self.products, f, indent=2, ensure_ascii=False)
- logger.debug(f"Saved {len(self.products)} products")
- except IOError as e:
- logger.error(f"Error saving products file: {e}")
-
- def process_products(self, products: List[Product]) -> tuple[List[Product], List[Product]]:
- """
- Process a list of scraped products and detect changes
-
- Args:
- products: List of products from scraper
-
- Returns:
- Tuple of (new_products, restocked_products)
- """
- new_products = []
- restocked_products = []
-
- for product in products:
- url = product.url
-
- if url not in self.products:
- # New product!
- new_products.append(product)
- self.products[url] = {
- "name": product.name,
- "url": url,
- "price": product.price,
- "in_stock": product.in_stock,
- "image_url": product.image_url,
- "site": product.site,
- "product_id": product.product_id,
- "first_seen": datetime.now().isoformat(),
- "last_seen": datetime.now().isoformat(),
- "last_in_stock": datetime.now().isoformat() if product.in_stock else None,
- }
- logger.info(f"NEW PRODUCT: {product.name}")
-
- else:
- # Existing product - check for restock
- existing = self.products[url]
- was_in_stock = existing.get("in_stock", False)
-
- # Update last seen
- existing["last_seen"] = datetime.now().isoformat()
- existing["price"] = product.price or existing.get("price")
- existing["image_url"] = product.image_url or existing.get("image_url")
-
- if product.in_stock and not was_in_stock:
- # RESTOCK!
- restocked_products.append(product)
- existing["last_in_stock"] = datetime.now().isoformat()
- logger.info(f"RESTOCK: {product.name}")
-
- existing["in_stock"] = product.in_stock
- self.products[url] = existing
-
- self.save()
- return new_products, restocked_products
-
- def get_known_urls(self) -> Set[str]:
- """Get all known product URLs"""
- return set(self.products.keys())
-
- def get_product(self, url: str) -> Optional[dict]:
- """Get a specific product by URL"""
- return self.products.get(url)
-
- def mark_out_of_stock(self, url: str):
- """Mark a product as out of stock"""
- if url in self.products:
- self.products[url]["in_stock"] = False
- self.save()
-
- def clear(self):
- """Clear all tracked products"""
- self.products = {}
- self.save()
- logger.info("Cleared all tracked products")
-
- def get_stats(self) -> dict:
- """Get tracking statistics"""
- total = len(self.products)
- in_stock = sum(1 for p in self.products.values() if p.get("in_stock", False))
- out_of_stock = total - in_stock
-
- return {
- "total_products": total,
- "in_stock": in_stock,
- "out_of_stock": out_of_stock,
- }
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..8cff5f3
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,9 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = -v --tb=short
+filterwarnings =
+ ignore::DeprecationWarning
+ ignore::PendingDeprecationWarning
diff --git a/requirements.txt b/requirements.txt
index c88db0d..d6aba90 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,13 @@ playwright-stealth>=1.0.6
requests>=2.31.0
schedule>=1.2.1
beautifulsoup4>=4.12.0
+flask>=3.0.0
+flask-cors>=4.0.0
+psutil>=5.9.0
+undetected-chromedriver>=3.5.0
+selenium>=4.15.0
+setuptools>=70.0.0 # Required for Python 3.13+ (distutils compatibility)
+discord.py>=2.3.0 # Discord bot for location-based store search
+nltk>=3.8.0 # Sentiment analysis using VADER
+pytest>=8.0.0 # Testing framework
+pytest-cov>=4.0.0 # Coverage reporting
diff --git a/scrapers/__init__.py b/scrapers/__init__.py
index dd14901..4414ce3 100644
--- a/scrapers/__init__.py
+++ b/scrapers/__init__.py
@@ -1,5 +1,15 @@
# Scrapers package
from .pokemoncenter import PokemonCenterScraper
from .target import TargetScraper
+from .gamestop import GameStopScraper, warmup_gamestop
+from .bestbuy import BestBuyScraper
+from .walmart import WalmartScraper
-__all__ = ["PokemonCenterScraper", "TargetScraper"]
+__all__ = [
+ "PokemonCenterScraper",
+ "TargetScraper",
+ "GameStopScraper",
+ "BestBuyScraper",
+ "WalmartScraper",
+ "warmup_gamestop",
+]
diff --git a/scrapers/base.py b/scrapers/base.py
index d70c017..abde8a2 100644
--- a/scrapers/base.py
+++ b/scrapers/base.py
@@ -35,6 +35,50 @@ class BaseScraper(ABC):
site_name: str = "unknown"
+ # Terms that indicate a Pokemon product
+ POKEMON_TERMS = [
+ "pokemon", "pokémon", "poke", "tcg",
+ "pikachu", "charizard", "mewtwo", "eevee", "snorlax",
+ "booster", "elite trainer", "etb",
+ "scarlet", "violet", "prismatic", "evolutions",
+ ]
+
+ # Terms that indicate NOT a Pokemon product (false positives from search)
+ EXCLUDE_TERMS = [
+ "ice cube", "oven", "barbie", "hot wheels", "lego",
+ "furniture", "appliance", "kitchen", "bedding",
+ "glitter girls", "masters of the universe", "transformers",
+ "room essentials", "threshold",
+ ]
+
+ def is_pokemon_product(self, product: Product) -> bool:
+ """
+ Check if a product is actually a Pokemon product.
+ Filters out false positives from search results.
+ """
+ name_lower = product.name.lower()
+
+ # Check for exclusion terms first
+ for term in self.EXCLUDE_TERMS:
+ if term in name_lower:
+ return False
+
+ # Check for Pokemon terms
+ for term in self.POKEMON_TERMS:
+ if term in name_lower:
+ return True
+
+ # If no Pokemon terms found, reject it
+ return False
+
+ def filter_pokemon_products(self, products: List[Product]) -> List[Product]:
+ """Filter to only include valid Pokemon products"""
+ filtered = [p for p in products if self.is_pokemon_product(p)]
+ rejected = len(products) - len(filtered)
+ if rejected > 0:
+ logger.info(f"Filtered out {rejected} non-Pokemon products")
+ return filtered
+
@abstractmethod
def scrape_category_page(self, url: str) -> List[Product]:
"""
diff --git a/scrapers/bestbuy.py b/scrapers/bestbuy.py
new file mode 100644
index 0000000..8dcb5cb
--- /dev/null
+++ b/scrapers/bestbuy.py
@@ -0,0 +1,505 @@
+"""
+Best Buy scraper - Uses undetected-chromedriver to bypass bot protection
+"""
+
+import re
+import json
+import logging
+import time
+import sys
+import os
+from typing import List, Optional
+from bs4 import BeautifulSoup
+
+from .base import BaseScraper, Product
+
+# Add parent directory to path for tools import
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+logger = logging.getLogger(__name__)
+
+
+class BestBuyScraper(BaseScraper):
+ """Scraper for BestBuy.com - Uses undetected-chromedriver"""
+
+ site_name = "bestbuy"
+ base_url = "https://www.bestbuy.com"
+
+ def __init__(self):
+ self._stealth_browser = None
+
+ def _get_stealth_browser(self):
+ """Get or create stealth browser for Best Buy"""
+ if self._stealth_browser is None:
+ from tools.stealth_browser import StealthBrowser
+ self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
+ self._stealth_browser.start()
+ return self._stealth_browser
+
+ def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
+ """
+ Scrape a Best Buy search/category page for all products
+ Uses undetected-chromedriver to bypass bot protection
+
+ Args:
+ url: Search/category page URL
+ max_pages: Maximum number of pages to scrape (default 1)
+
+ Returns:
+ List of Product objects
+ """
+ all_products = []
+
+ # Add sort by newest if not already in URL
+ if "sort=" not in url:
+ separator = "&" if "?" in url else "?"
+ url = f"{url}{separator}sp=-releasedate"
+
+ # Use stealth browser
+ try:
+ browser = self._get_stealth_browser()
+ except Exception as e:
+ logger.error(f"Failed to start stealth browser: {e}")
+ return all_products
+
+ for page_num in range(1, max_pages + 1):
+ products = []
+ page_url = url if page_num == 1 else f"{url}&cp={page_num}"
+
+ try:
+ logger.info(f"Navigating to: {page_url}")
+ browser.driver.get(page_url)
+ time.sleep(8) # Wait longer for initial page load
+
+ # Scroll for lazy loading - wait longer between scrolls
+ browser.driver.execute_script("window.scrollTo(0, 500)")
+ time.sleep(2)
+ browser.driver.execute_script("window.scrollTo(0, 1500)")
+ time.sleep(2)
+ browser.driver.execute_script("window.scrollTo(0, 3000)")
+ time.sleep(2)
+ browser.driver.execute_script("window.scrollTo(0, 0)") # Scroll back to top
+ time.sleep(3)
+
+ # Wait for products to load (check for content)
+ for _ in range(10):
+ html = browser.driver.page_source
+ if "sku-item" in html or "sku-title" in html or "priceView" in html:
+ logger.info("Product content detected")
+ break
+ time.sleep(1)
+
+ # Get HTML after waiting
+ html = browser.driver.page_source
+
+ # Save debug screenshot
+ try:
+ browser.driver.save_screenshot("debug_bestbuy.png")
+ logger.info("Saved debug screenshot to debug_bestbuy.png")
+ except:
+ pass
+
+ soup = BeautifulSoup(html, "html.parser")
+
+ # Try to find product data in JSON scripts
+ scripts = soup.find_all("script", type="application/json")
+ for script in scripts:
+ try:
+ data = json.loads(script.string)
+ products.extend(self._extract_products_from_json(data))
+ except (json.JSONDecodeError, TypeError):
+ continue
+
+ # Try multiple selectors for product cards - Best Buy updates these frequently
+ product_cards = (
+ soup.select("li.sku-item")
+ or soup.select("[data-sku-id]")
+ or soup.select(".sku-item")
+ or soup.select("[class*='sku-item']")
+ or soup.select("div.shop-sku-list-item")
+ or soup.select("[class*='productCard']")
+ or soup.select("[class*='product-card']")
+ or soup.select(".list-item")
+ )
+
+ logger.info(f"Found {len(product_cards)} product cards on page {page_num}")
+
+ for card in product_cards:
+ product = self._parse_product_card(card)
+ if product:
+ products.append(product)
+
+ # Fallback: parse product links
+ if not product_cards:
+ product_links = soup.select("a[href*='/site/'][href*='.p']")
+ logger.info(f"Fallback: Found {len(product_links)} product links")
+
+ href_to_links = {}
+ for link in product_links:
+ href = link.get("href", "")
+ if not href or ".p" not in href:
+ continue
+ if href not in href_to_links:
+ href_to_links[href] = []
+ href_to_links[href].append(link)
+
+ for href, links in href_to_links.items():
+ best_link = links[0]
+ for link in links:
+ if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
+ best_link = link
+ break
+
+ product = self._parse_product_link(best_link, soup)
+ if product:
+ products.append(product)
+
+ # Also try parsing from data attributes and script tags
+ if not products:
+ products.extend(self._extract_from_page_data(soup, browser))
+
+ # Don't close - reuse browser for next page
+
+ except Exception as e:
+ logger.error(f"Error scraping Best Buy page {page_num}: {e}")
+ break # Don't raise - just return what we have
+
+ all_products.extend(products)
+
+ # Stop if no products found on this page (no more pages)
+ if not products:
+ break
+
+ # Remove duplicates
+ seen_urls = set()
+ unique_products = []
+ for p in all_products:
+ if p.url not in seen_urls:
+ seen_urls.add(p.url)
+ unique_products.append(p)
+
+ logger.info(f"Scraped {len(unique_products)} unique products from Best Buy")
+ return unique_products
+
+ def _extract_from_page_data(self, soup, browser) -> List[Product]:
+ """Extract products from page data attributes and evaluate JS if needed"""
+ products = []
+
+ # Try to get product data from data attributes
+ items_with_data = soup.select("[data-testid][data-sku-id]")
+ for item in items_with_data:
+ sku_id = item.get("data-sku-id", "")
+ if sku_id:
+ # Find name and price within this element
+ name_elem = item.select_one("h4") or item.select_one("[class*='title']") or item.select_one("a")
+ name = name_elem.get_text(strip=True) if name_elem else ""
+
+ if name and len(name) > 5:
+ price_text = item.get_text()
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
+ price = price_match.group() if price_match else None
+
+ products.append(Product(
+ name=name,
+ url=f"{self.base_url}/site/{sku_id}.p",
+ price=price,
+ in_stock=True,
+ image_url=None,
+ site=self.site_name,
+ product_id=sku_id,
+ ))
+
+ # Try to extract from window.__INITIAL_STATE__ or similar JS objects
+ try:
+ initial_state = browser.driver.execute_script("""
+ if (window.__INITIAL_STATE__) return JSON.stringify(window.__INITIAL_STATE__);
+ if (window.__NEXT_DATA__) return JSON.stringify(window.__NEXT_DATA__);
+ return null;
+ """)
+ if initial_state:
+ data = json.loads(initial_state)
+ products.extend(self._extract_products_from_json(data))
+ except Exception as e:
+ logger.debug(f"Could not extract from JS state: {e}")
+
+ return products
+
+ def _extract_products_from_json(self, data, depth=0) -> List[Product]:
+ """Recursively search JSON for product data"""
+ products = []
+ if depth > 10:
+ return products
+
+ if isinstance(data, dict):
+ # Check if this looks like a Best Buy product
+ if "skuId" in data or ("name" in data and "regularPrice" in data):
+ product = self._parse_product_json(data)
+ if product:
+ products.append(product)
+
+ for value in data.values():
+ products.extend(self._extract_products_from_json(value, depth + 1))
+
+ elif isinstance(data, list):
+ for item in data:
+ products.extend(self._extract_products_from_json(item, depth + 1))
+
+ return products
+
+ def _parse_product_json(self, data: dict) -> Optional[Product]:
+ """Parse a product from Best Buy's JSON data"""
+ try:
+ name = data.get("name") or data.get("displayName", "")
+ if not name:
+ return None
+
+ sku_id = data.get("skuId") or data.get("sku", "")
+ url_slug = data.get("url") or ""
+
+ if url_slug:
+ url = url_slug if url_slug.startswith("http") else f"{self.base_url}{url_slug}"
+ elif sku_id:
+ url = f"{self.base_url}/site/{sku_id}.p"
+ else:
+ return None
+
+ # Get price
+ price = None
+ if "regularPrice" in data:
+ price = f"${data['regularPrice']:.2f}"
+ elif "salePrice" in data:
+ price = f"${data['salePrice']:.2f}"
+
+ # Check availability
+ in_stock = True
+ availability = data.get("availability", {})
+ if isinstance(availability, dict):
+ in_stock = availability.get("isAvailable", True)
+ elif data.get("orderable") is False:
+ in_stock = False
+
+ # Get image
+ image_url = data.get("image") or data.get("thumbnailImage")
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=str(sku_id),
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Best Buy product JSON: {e}")
+ return None
+
+ def _parse_product_card(self, card) -> Optional[Product]:
+ """Parse a product card element"""
+ try:
+ # Find link
+ link = card.select_one("a[href*='/site/'][href*='.p']") or card.select_one("a.image-link")
+ if not link:
+ link = card.select_one("a")
+ if not link:
+ return None
+
+ href = link.get("href", "")
+ if not href:
+ return None
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ # Get name
+ name_elem = (
+ card.select_one(".sku-title a")
+ or card.select_one("h4.sku-header a")
+ or card.select_one("[data-testid='product-title']")
+ or card.select_one(".sku-title")
+ or link
+ )
+ name = name_elem.get_text(strip=True) if name_elem else "Unknown"
+
+ # Skip if name too short
+ if len(name) < 5:
+ return None
+
+ # Get price
+ price_elem = (
+ card.select_one(".priceView-customer-price span")
+ or card.select_one("[data-testid='customer-price']")
+ or card.select_one(".pricing-price__regular-price")
+ or card.select_one("[class*='price']")
+ )
+ price = None
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ # Check stock
+ in_stock = self._check_card_stock_status(card)
+
+ # Get image
+ img = card.select_one("img")
+ image_url = None
+ if img:
+ image_url = img.get("src") or img.get("data-src")
+ if image_url and not image_url.startswith("http"):
+ image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}"
+
+ # Extract SKU ID from URL
+ product_id = ""
+ match = re.search(r"/(\d+)\.p", url)
+ if match:
+ product_id = match.group(1)
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Best Buy product card: {e}")
+ return None
+
+ def _parse_product_link(self, link, soup) -> Optional[Product]:
+ """Parse a product from a product link element"""
+ try:
+ href = link.get("href", "")
+ if not href:
+ return None
+
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ name = link.get("aria-label", "") or link.get_text(strip=True)
+
+ if not name or len(name) < 5:
+ return None
+
+ # Extract SKU ID
+ product_id = ""
+ match = re.search(r"/(\d+)\.p", url)
+ if match:
+ product_id = match.group(1)
+
+ # Find price near link
+ parent = link.find_parent()
+ price = None
+ for _ in range(5):
+ if parent:
+ price_text = parent.get_text()
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
+ if price_match:
+ price = price_match.group()
+ break
+ parent = parent.find_parent()
+
+ # Find image
+ image_url = None
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ img = parent.select_one("img")
+ if img:
+ image_url = img.get("src") or img.get("data-src")
+ break
+ parent = parent.find_parent()
+
+ # Check stock
+ in_stock = True
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ text = parent.get_text().lower()
+ if any(phrase in text for phrase in ["sold out", "out of stock", "unavailable"]):
+ in_stock = False
+ break
+ parent = parent.find_parent()
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Best Buy product link: {e}")
+ return None
+
+ def _check_card_stock_status(self, card) -> bool:
+ """Check if a product card indicates in-stock status"""
+ card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
+
+ # Check for disabled add to cart button
+ add_btn = card.select_one(".add-to-cart-button")
+ if add_btn and "btn-disabled" in add_btn.get("class", []):
+ return False
+
+ out_of_stock_phrases = [
+ "sold out",
+ "out of stock",
+ "unavailable",
+ "coming soon",
+ "not available",
+ ]
+
+ for phrase in out_of_stock_phrases:
+ if phrase in card_text:
+ return False
+
+ in_stock_phrases = [
+ "add to cart",
+ "add to bag",
+ "available",
+ "in stock",
+ ]
+
+ for phrase in in_stock_phrases:
+ if phrase in card_text:
+ return True
+
+ return True
+
+ def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
+ """Check if a specific product is in stock"""
+ try:
+ browser = self._get_stealth_browser()
+ browser.driver.get(product_url)
+ time.sleep(3)
+ html = browser.driver.page_source
+ soup = BeautifulSoup(html, "html.parser")
+ page_text = soup.get_text().lower()
+
+ out_of_stock = any(
+ phrase in page_text
+ for phrase in ["sold out", "out of stock", "unavailable", "coming soon"]
+ )
+
+ in_stock = not out_of_stock
+
+ # Get price
+ price = None
+ price_elem = (
+ soup.select_one(".priceView-customer-price span")
+ or soup.select_one("[data-testid='customer-price']")
+ )
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ return in_stock, price
+
+ except Exception as e:
+ logger.error(f"Error checking Best Buy product stock: {e}")
+ return False, None
diff --git a/scrapers/gamestop.py b/scrapers/gamestop.py
new file mode 100644
index 0000000..9937530
--- /dev/null
+++ b/scrapers/gamestop.py
@@ -0,0 +1,478 @@
+"""
+GameStop.com scraper
+"""
+
+import re
+import logging
+import time
+import sys
+import os
+from typing import List, Optional
+from bs4 import BeautifulSoup
+
+from .base import BaseScraper, Product
+from src.browser import get_browser
+from config import CAPTCHA_WAIT_TIMEOUT
+
+# Add parent directory to path for tools import
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+logger = logging.getLogger(__name__)
+
+
+def warmup_gamestop():
+ """
+ Warmup function to solve Cloudflare CAPTCHA manually.
+ Uses undetected-chromedriver to bypass bot detection.
+ """
+ from tools.stealth_browser import StealthBrowser
+
+ print("\n" + "=" * 60)
+ print("GAMESTOP WARMUP - Using Undetected Chrome")
+ print("=" * 60)
+ print("A browser window should open.")
+ print("If you see a Cloudflare challenge, solve it manually.")
+ print(f"Waiting up to {CAPTCHA_WAIT_TIMEOUT} seconds...")
+ print("=" * 60 + "\n")
+
+ logger.info("Starting GameStop warmup with undetected-chromedriver...")
+
+ # Use stealth browser instead of Playwright
+ browser = StealthBrowser(headless=False, session_name="gamestop")
+
+ try:
+ browser.start()
+ logger.info("Stealth browser started")
+
+ # Navigate to GameStop
+ browser.driver.get("https://www.gamestop.com")
+ time.sleep(3)
+
+ # Check for Cloudflare challenge
+ start_time = time.time()
+ challenge_detected = False
+
+ while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
+ try:
+ html = browser.driver.page_source
+ html_lower = html.lower()
+ title = browser.driver.title.lower()
+
+ # Cloudflare challenge indicators
+ is_cloudflare_challenge = (
+ "just a moment" in title or
+ "checking your browser" in html_lower or
+ "cf-challenge" in html_lower or
+ "turnstile" in html_lower or
+ (len(html) < 5000 and "challenge" in html_lower)
+ )
+
+ if is_cloudflare_challenge:
+ if not challenge_detected:
+ challenge_detected = True
+ print(">>> Cloudflare challenge detected! Please solve it in the browser window.")
+ logger.info("Cloudflare challenge detected - waiting for manual solve...")
+ time.sleep(5)
+ else:
+ # Check if we're on actual GameStop content
+ if len(html) > 10000 and "gamestop" in html_lower:
+ print(">>> Challenge solved! GameStop page loaded successfully.")
+ logger.info("GameStop warmup complete!")
+ browser.stop()
+ return True
+ time.sleep(2)
+ except Exception as e:
+ logger.debug(f"Error checking page: {e}")
+ time.sleep(2)
+
+ logger.warning("GameStop warmup timed out - CAPTCHA may not be solved")
+ print(">>> Warmup timed out. You may need to try again.")
+ browser.stop()
+ return False
+
+ except Exception as e:
+ logger.error(f"Error during warmup: {e}")
+ try:
+ browser.stop()
+ except:
+ pass
+ return False
+
+
+class GameStopScraper(BaseScraper):
+ """Scraper for GameStop.com - Uses undetected-chromedriver to bypass Cloudflare"""
+
+ site_name = "gamestop"
+ base_url = "https://www.gamestop.com"
+
+ def __init__(self):
+ self._stealth_browser = None
+
+ def _get_stealth_browser(self):
+ """Get or create stealth browser for GameStop"""
+ if self._stealth_browser is None:
+ from tools.stealth_browser import StealthBrowser
+ self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
+ self._stealth_browser.start()
+ return self._stealth_browser
+
+ def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
+ """
+ Scrape a GameStop search/category page for all products
+ Uses undetected-chromedriver to bypass Cloudflare
+
+ Args:
+ url: Search/category page URL
+ max_pages: Maximum number of pages to scrape (default 1)
+
+ Returns:
+ List of Product objects
+ """
+ all_products = []
+
+ # Add sort by newest if not already in URL
+ if "sort=" not in url.lower():
+ separator = "&" if "?" in url else "?"
+ url = f"{url}{separator}sort=date-desc"
+
+ # Use stealth browser for GameStop
+ try:
+ browser = self._get_stealth_browser()
+ except Exception as e:
+ logger.error(f"Failed to start stealth browser: {e}")
+ return all_products
+
+ for page_num in range(1, max_pages + 1):
+ products = []
+ page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}"
+
+ try:
+ logger.info(f"Navigating to: {page_url}")
+ browser.driver.get(page_url)
+ time.sleep(5) # Wait for page load
+
+ html = browser.driver.page_source
+
+ # Check for Cloudflare challenge
+ html_lower = html.lower()
+ title = browser.driver.title.lower()
+
+ is_cloudflare = (
+ "just a moment" in title or
+ "checking your browser" in html_lower or
+ "cf-challenge" in html_lower
+ )
+
+ if is_cloudflare:
+ logger.warning("Cloudflare challenge detected! Waiting for manual solve...")
+ start_time = time.time()
+ while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
+ time.sleep(5)
+ html = browser.driver.page_source
+ html_lower = html.lower()
+ title = browser.driver.title.lower()
+ if "just a moment" not in title and "cf-challenge" not in html_lower:
+ logger.info("Cloudflare challenge solved!")
+ break
+ else:
+ logger.error("Cloudflare challenge not solved in time")
+ return all_products
+
+ # Save debug screenshot
+ try:
+ browser.driver.save_screenshot("debug_gamestop.png")
+ logger.info("Saved debug screenshot to debug_gamestop.png")
+ except:
+ pass
+
+ soup = BeautifulSoup(html, "html.parser")
+
+ # Try multiple selectors for product cards
+ product_cards = (
+ soup.select(".product-tile")
+ or soup.select("[data-testid='product-tile']")
+ or soup.select(".grid-tile")
+ or soup.select(".product-grid-tile")
+ or soup.select("[class*='ProductTile']")
+ )
+
+ logger.info(f"Found {len(product_cards)} product cards")
+
+ # If no cards found, try link-based extraction
+ if not product_cards:
+ product_links = soup.select("a[href*='/products/']")
+ logger.info(f"Fallback: Found {len(product_links)} product links")
+
+ # Group by href
+ href_to_links = {}
+ for link in product_links:
+ href = link.get("href", "")
+ if not href or "/products/" not in href:
+ continue
+ if href not in href_to_links:
+ href_to_links[href] = []
+ href_to_links[href].append(link)
+
+ for href, links in href_to_links.items():
+ # Pick best link
+ best_link = links[0]
+ for link in links:
+ if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
+ best_link = link
+ break
+
+ product = self._parse_product_link(best_link, soup)
+ if product:
+ products.append(product)
+ else:
+ for card in product_cards:
+ product = self._parse_product_card(card)
+ if product:
+ products.append(product)
+
+ # Don't close - reuse browser for next page
+
+ except Exception as e:
+ logger.error(f"Error scraping GameStop page {page_num}: {e}")
+ break # Don't raise - just return what we have
+
+ all_products.extend(products)
+
+ # Stop if no products found on this page
+ if not products:
+ break
+
+ # Remove duplicates
+ seen_urls = set()
+ unique_products = []
+ for p in all_products:
+ if p.url not in seen_urls:
+ seen_urls.add(p.url)
+ unique_products.append(p)
+
+ logger.info(f"Scraped {len(unique_products)} unique products from GameStop")
+ return unique_products
+
+ def _parse_product_card(self, card) -> Optional[Product]:
+ """Parse a product card element"""
+ try:
+ # Find link - try multiple patterns
+ link = (
+ card.select_one("a[href*='/products/']") or
+ card.select_one("a[href*='/video-games/']") or
+ card.select_one("a[href*='/collectibles/']") or
+ card.select_one("a.product-tile-link") or
+ card.select_one("a")
+ )
+ if not link:
+ logger.debug("No link found in card")
+ return None
+
+ href = link.get("href", "")
+ if not href:
+ logger.debug("Empty href")
+ return None
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ # Get name - try multiple patterns
+ name_elem = (
+ card.select_one(".product-tile-title")
+ or card.select_one(".product-name a")
+ or card.select_one(".product-name")
+ or card.select_one(".product-title")
+ or card.select_one("[data-testid='product-name']")
+ or card.select_one("a[aria-label]")
+ or link
+ )
+
+ # Try to get name from aria-label first
+ name = ""
+ if name_elem:
+ name = name_elem.get("aria-label", "") or name_elem.get_text(strip=True)
+
+ # Skip if name too short
+ if len(name) < 5:
+ logger.debug(f"Name too short: '{name}' from {url[:50]}")
+ return None
+
+ # Get price
+ price_elem = (
+ card.select_one(".price-sales")
+ or card.select_one(".product-price")
+ or card.select_one("[data-testid='price']")
+ or card.select_one("[class*='price']")
+ )
+ price = None
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ # Check stock
+ in_stock = self._check_card_stock_status(card)
+
+ # Get image
+ img = card.select_one("img")
+ image_url = None
+ if img:
+ image_url = img.get("src") or img.get("data-src") or img.get("data-lazy")
+ if image_url and not image_url.startswith("http"):
+ image_url = f"{self.base_url}{image_url}"
+
+ # Extract product ID from URL
+ product_id = ""
+ match = re.search(r"/products/[^/]+/(\d+)", url)
+ if match:
+ product_id = match.group(1)
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing GameStop product card: {e}")
+ return None
+
+ def _parse_product_link(self, link, soup) -> Optional[Product]:
+ """Parse a product from a product link element"""
+ try:
+ href = link.get("href", "")
+ if not href:
+ return None
+
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ # Get name
+ name = link.get("aria-label", "") or link.get_text(strip=True)
+
+ # Skip if name too short
+ if not name or len(name) < 5:
+ return None
+
+ # Extract product ID from URL
+ product_id = ""
+ match = re.search(r"/products/[^/]+/(\d+)", url)
+ if match:
+ product_id = match.group(1)
+
+ # Try to find price near this link
+ parent = link.find_parent()
+ price = None
+ for _ in range(5):
+ if parent:
+ price_text = parent.get_text()
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
+ if price_match:
+ price = price_match.group()
+ break
+ parent = parent.find_parent()
+
+ # Try to find image
+ image_url = None
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ img = parent.select_one("img")
+ if img:
+ image_url = img.get("src") or img.get("data-src")
+ if image_url and not image_url.startswith("http"):
+ image_url = f"{self.base_url}{image_url}"
+ break
+ parent = parent.find_parent()
+
+ # Check stock
+ in_stock = True
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ text = parent.get_text().lower()
+ if any(phrase in text for phrase in ["sold out", "out of stock", "not available"]):
+ in_stock = False
+ break
+ parent = parent.find_parent()
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing GameStop product link: {e}")
+ return None
+
+ def _check_card_stock_status(self, card) -> bool:
+ """Check if a product card indicates in-stock status"""
+ card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
+
+ out_of_stock_phrases = [
+ "sold out",
+ "out of stock",
+ "not available",
+ "unavailable",
+ "currently unavailable",
+ ]
+
+ for phrase in out_of_stock_phrases:
+ if phrase in card_text:
+ return False
+
+ in_stock_phrases = [
+ "add to cart",
+ "add to bag",
+ "available",
+ "buy now",
+ "in stock",
+ ]
+
+ for phrase in in_stock_phrases:
+ if phrase in card_text:
+ return True
+
+ # Default: assume in stock if listed
+ return True
+
+ def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
+ """Check if a specific product is in stock"""
+ try:
+ browser = self._get_stealth_browser()
+ browser.driver.get(product_url)
+ time.sleep(3)
+ html = browser.driver.page_source
+ soup = BeautifulSoup(html, "html.parser")
+ page_text = soup.get_text().lower()
+
+ out_of_stock = any(
+ phrase in page_text
+ for phrase in ["sold out", "out of stock", "not available", "unavailable"]
+ )
+
+ in_stock = not out_of_stock
+
+ # Get price
+ price = None
+ price_elem = (
+ soup.select_one(".product-price")
+ or soup.select_one(".price-sales")
+ or soup.select_one("[data-testid='price']")
+ )
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ return in_stock, price
+
+ except Exception as e:
+ logger.error(f"Error checking GameStop product stock: {e}")
+ return False, None
diff --git a/scrapers/pokemoncenter.py b/scrapers/pokemoncenter.py
index 87bccd4..5dc0316 100644
--- a/scrapers/pokemoncenter.py
+++ b/scrapers/pokemoncenter.py
@@ -1,6 +1,6 @@
"""
PokemonCenter.com scraper
-Handles bot protection with Playwright stealth
+Handles bot protection with undetected-chromedriver stealth browser
"""
import re
@@ -9,44 +9,83 @@ from typing import List, Optional
from bs4 import BeautifulSoup
from .base import BaseScraper, Product
-from browser import get_browser
+import config
logger = logging.getLogger(__name__)
+def get_stealth_or_regular_browser():
+ """Get appropriate browser based on config"""
+ if config.USE_STEALTH_BROWSER:
+ from tools.stealth_browser import get_stealth_browser
+ return get_stealth_browser(
+ headless=config.STEALTH_HEADLESS,
+ session_name=config.STEALTH_SESSION_NAME
+ ), True
+ else:
+ from src.browser import get_browser
+ return get_browser(), False
+
+
class PokemonCenterScraper(BaseScraper):
"""Scraper for PokemonCenter.com"""
site_name = "pokemoncenter"
base_url = "https://www.pokemoncenter.com"
- def scrape_category_page(self, url: str) -> List[Product]:
+ def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
"""
Scrape a PokemonCenter category page for all products
Args:
url: Category page URL
+ max_pages: Maximum pages to scrape (not yet implemented for PokemonCenter)
Returns:
List of Product objects
"""
- browser = get_browser()
+ # TODO: Implement pagination for PokemonCenter when needed
+ browser, is_stealth = get_stealth_or_regular_browser()
products = []
try:
- # Navigate and wait for page to load (don't wait for specific selector)
- page, html = browser.get_page_content(
- url,
- wait_for_selector=None, # Let it use networkidle instead
- timeout=60000,
- )
+ if is_stealth:
+ # Use stealth browser (undetected-chromedriver)
+ html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
- # Save screenshot for debugging if needed
- try:
- page.screenshot(path="debug_screenshot.png")
- logger.info("Saved debug screenshot to debug_screenshot.png")
- except:
- pass
+ # Check for CAPTCHA
+ if browser.check_for_captcha():
+ logger.warning("CAPTCHA detected on PokemonCenter!")
+ if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
+ browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
+ # Re-fetch page after CAPTCHA solve
+ html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
+ else:
+ logger.error("Cannot solve CAPTCHA in headless mode")
+ return []
+
+ # Save screenshot for debugging
+ try:
+ browser.screenshot("debug_screenshot.png")
+ logger.info("Saved debug screenshot to debug_screenshot.png")
+ except:
+ pass
+
+ page = None # Stealth browser doesn't return page object
+ else:
+ # Use regular Playwright browser
+ page, html = browser.get_page_content(
+ url,
+ wait_for_selector=None, # Let it use networkidle instead
+ timeout=60000,
+ )
+
+ # Save screenshot for debugging if needed
+ try:
+ page.screenshot(path="debug_screenshot.png")
+ logger.info("Saved debug screenshot to debug_screenshot.png")
+ except:
+ pass
soup = BeautifulSoup(html, "html.parser")
@@ -96,7 +135,9 @@ class PokemonCenterScraper(BaseScraper):
if product:
products.append(product)
- page.close()
+ # Close page only for Playwright browser
+ if page is not None:
+ page.close()
except Exception as e:
logger.error(f"Error scraping PokemonCenter category page: {e}")
@@ -224,14 +265,30 @@ class PokemonCenterScraper(BaseScraper):
Returns:
Tuple of (is_in_stock, price)
"""
- browser = get_browser()
+ browser, is_stealth = get_stealth_or_regular_browser()
try:
- page, html = browser.get_page_content(
- product_url,
- wait_for_selector="button, [data-testid]",
- timeout=30000,
- )
+ if is_stealth:
+ # Use stealth browser
+ html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
+
+ # Check for CAPTCHA
+ if browser.check_for_captcha():
+ logger.warning("CAPTCHA detected on product page!")
+ if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
+ browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
+ html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
+ else:
+ return False, None
+
+ page = None
+ else:
+ # Use regular Playwright browser
+ page, html = browser.get_page_content(
+ product_url,
+ wait_for_selector="button, [data-testid]",
+ timeout=30000,
+ )
soup = BeautifulSoup(html, "html.parser")
page_text = soup.get_text().lower()
@@ -257,7 +314,8 @@ class PokemonCenterScraper(BaseScraper):
if price_match:
price = price_match.group()
- page.close()
+ if page is not None:
+ page.close()
return in_stock, price
except Exception as e:
diff --git a/scrapers/target.py b/scrapers/target.py
index 026ebdd..0ab3a58 100644
--- a/scrapers/target.py
+++ b/scrapers/target.py
@@ -9,7 +9,7 @@ from typing import List, Optional
from bs4 import BeautifulSoup
from .base import BaseScraper, Product
-from browser import get_browser
+from src.browser import get_browser
logger = logging.getLogger(__name__)
@@ -20,87 +20,105 @@ class TargetScraper(BaseScraper):
site_name = "target"
base_url = "https://www.target.com"
- def scrape_category_page(self, url: str) -> List[Product]:
+ def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
"""
Scrape a Target search/category page for all products
Args:
url: Search/category page URL
+ max_pages: Maximum number of pages to scrape (default 1)
Returns:
List of Product objects
"""
browser = get_browser()
- products = []
+ all_products = []
- try:
- page, html = browser.get_page_content(
- url,
- wait_for_selector=None,
- timeout=60000,
- )
+ # Add sort by newest if not already in URL
+ if "sortBy=" not in url:
+ separator = "&" if "?" in url else "?"
+ url = f"{url}{separator}sortBy=newest"
+
+ for page_num in range(1, max_pages + 1):
+ products = []
+ page_url = url if page_num == 1 else f"{url}&Nao={24 * (page_num - 1)}"
- # Save debug screenshot
try:
- page.screenshot(path="debug_target.png")
- logger.info("Saved debug screenshot to debug_target.png")
- except:
- pass
+ page, html = browser.get_page_content(
+ page_url,
+ wait_for_selector=None,
+ timeout=60000,
+ )
- soup = BeautifulSoup(html, "html.parser")
-
- # Try to find product data in page scripts (Target uses React/hydration)
- scripts = soup.find_all("script", type="application/json")
- for script in scripts:
+ # Save debug screenshot
try:
- data = json.loads(script.string)
- # Look for product data in the JSON
- products.extend(self._extract_products_from_json(data))
- except (json.JSONDecodeError, TypeError):
- continue
+ page.screenshot(path="debug_target.png")
+ logger.info("Saved debug screenshot to debug_target.png")
+ except:
+ pass
- # Parse product links from the page
- product_links = soup.select("a[href*='/p/']")
- logger.info(f"Found {len(product_links)} product links")
+ soup = BeautifulSoup(html, "html.parser")
- # Group links by href and pick the best one (with aria-label or text)
- href_to_links = {}
- for link in product_links:
- href = link.get("href", "")
- if not href:
- continue
- if href not in href_to_links:
- href_to_links[href] = []
- href_to_links[href].append(link)
+ # Try to find product data in page scripts (Target uses React/hydration)
+ scripts = soup.find_all("script", type="application/json")
+ for script in scripts:
+ try:
+ data = json.loads(script.string)
+ # Look for product data in the JSON
+ products.extend(self._extract_products_from_json(data))
+ except (json.JSONDecodeError, TypeError):
+ continue
- logger.info(f"Found {len(href_to_links)} unique hrefs")
+ # Parse product links from the page
+ product_links = soup.select("a[href*='/p/']")
+ logger.info(f"Found {len(product_links)} product links")
- for href, links in href_to_links.items():
- # Find the best link (one with aria-label or text content)
- best_link = None
- for link in links:
- aria = link.get("aria-label", "")
- text = link.get_text(strip=True)
- if aria or (text and len(text) > 10):
- best_link = link
- break
- if not best_link:
- best_link = links[0] # Fallback to first link
+ # Group links by href and pick the best one (with aria-label or text)
+ href_to_links = {}
+ for link in product_links:
+ href = link.get("href", "")
+ if not href:
+ continue
+ if href not in href_to_links:
+ href_to_links[href] = []
+ href_to_links[href].append(link)
- product = self._parse_product_link(best_link, soup)
- if product:
- products.append(product)
+ logger.info(f"Found {len(href_to_links)} unique hrefs")
- page.close()
+ for href, links in href_to_links.items():
+ # Find the best link (one with aria-label or text content)
+ best_link = None
+ for link in links:
+ aria = link.get("aria-label", "")
+ text = link.get_text(strip=True)
+ if aria or (text and len(text) > 10):
+ best_link = link
+ break
+ if not best_link:
+ best_link = links[0] # Fallback to first link
- except Exception as e:
- logger.error(f"Error scraping Target category page: {e}")
- raise
+ product = self._parse_product_link(best_link, soup)
+ if product:
+ products.append(product)
+
+ page.close()
+
+ except Exception as e:
+ logger.error(f"Error scraping Target page {page_num}: {e}")
+ if page_num == 1:
+ raise
+ break
+
+ all_products.extend(products)
+
+ # Stop if no products found on this page
+ if not products:
+ break
# Remove duplicates
seen_urls = set()
unique_products = []
- for p in products:
+ for p in all_products:
if p.url not in seen_urls:
seen_urls.add(p.url)
unique_products.append(p)
diff --git a/scrapers/walmart.py b/scrapers/walmart.py
new file mode 100644
index 0000000..cb1a227
--- /dev/null
+++ b/scrapers/walmart.py
@@ -0,0 +1,444 @@
+"""
+Walmart.com scraper
+
+Note: Walmart has aggressive bot protection (PerimeterX).
+May need stealth browser mode for sustained use.
+"""
+
+import re
+import json
+import logging
+from typing import List, Optional
+from bs4 import BeautifulSoup
+
+from .base import BaseScraper, Product
+from src.browser import get_browser
+
+logger = logging.getLogger(__name__)
+
+
+class WalmartScraper(BaseScraper):
+ """Scraper for Walmart.com"""
+
+ site_name = "walmart"
+ base_url = "https://www.walmart.com"
+
+ def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
+ """
+ Scrape a Walmart search/category page for all products
+
+ Args:
+ url: Search/category page URL
+ max_pages: Maximum pages to scrape (not yet implemented for Walmart)
+
+ Returns:
+ List of Product objects
+ """
+ # TODO: Implement pagination for Walmart when needed
+ browser = get_browser()
+ products = []
+
+ try:
+ page, html = browser.get_page_content(
+ url,
+ wait_for_selector=None,
+ timeout=60000,
+ )
+
+ # Save debug screenshot
+ try:
+ page.screenshot(path="debug_walmart.png")
+ logger.info("Saved debug screenshot to debug_walmart.png")
+ except:
+ pass
+
+ soup = BeautifulSoup(html, "html.parser")
+
+ # Try to extract from __NEXT_DATA__ JSON (Walmart uses Next.js)
+ next_data = soup.select_one("script#__NEXT_DATA__")
+ if next_data:
+ try:
+ data = json.loads(next_data.string)
+ products.extend(self._extract_products_from_next_data(data))
+ except (json.JSONDecodeError, TypeError) as e:
+ logger.debug(f"Error parsing __NEXT_DATA__: {e}")
+
+ # Also try application/json scripts
+ scripts = soup.find_all("script", type="application/json")
+ for script in scripts:
+ try:
+ data = json.loads(script.string)
+ products.extend(self._extract_products_from_json(data))
+ except (json.JSONDecodeError, TypeError):
+ continue
+
+ # Try multiple selectors for product cards
+ product_cards = (
+ soup.select("[data-item-id]")
+ or soup.select(".search-result-gridview-item")
+ or soup.select("[data-testid='list-view']")
+ or soup.select("[class*='product-item']")
+ or soup.select("[class*='ProductCard']")
+ )
+
+ logger.info(f"Found {len(product_cards)} product cards")
+
+ for card in product_cards:
+ product = self._parse_product_card(card)
+ if product:
+ products.append(product)
+
+ # Fallback: parse product links
+ if not product_cards and not products:
+ product_links = soup.select("a[href*='/ip/']")
+ logger.info(f"Fallback: Found {len(product_links)} product links")
+
+ href_to_links = {}
+ for link in product_links:
+ href = link.get("href", "")
+ if not href or "/ip/" not in href:
+ continue
+ if href not in href_to_links:
+ href_to_links[href] = []
+ href_to_links[href].append(link)
+
+ for href, links in href_to_links.items():
+ best_link = links[0]
+ for link in links:
+ if link.get("aria-label") or len(link.get_text(strip=True)) > 10:
+ best_link = link
+ break
+
+ product = self._parse_product_link(best_link, soup)
+ if product:
+ products.append(product)
+
+ page.close()
+
+ except Exception as e:
+ logger.error(f"Error scraping Walmart category page: {e}")
+ raise
+
+ # Remove duplicates
+ seen_urls = set()
+ unique_products = []
+ for p in products:
+ if p.url not in seen_urls:
+ seen_urls.add(p.url)
+ unique_products.append(p)
+
+ logger.info(f"Scraped {len(unique_products)} unique products from Walmart")
+ return unique_products
+
+ def _extract_products_from_next_data(self, data: dict) -> List[Product]:
+ """Extract products from Next.js __NEXT_DATA__ JSON"""
+ products = []
+
+ try:
+ # Navigate to search results in Next.js data structure
+ props = data.get("props", {})
+ page_props = props.get("pageProps", {})
+ initial_data = page_props.get("initialData", {})
+ search_result = initial_data.get("searchResult", {})
+ item_stacks = search_result.get("itemStacks", [])
+
+ for stack in item_stacks:
+ items = stack.get("items", [])
+ for item in items:
+ product = self._parse_walmart_item(item)
+ if product:
+ products.append(product)
+
+ except Exception as e:
+ logger.debug(f"Error extracting from __NEXT_DATA__: {e}")
+
+ return products
+
+ def _extract_products_from_json(self, data, depth=0) -> List[Product]:
+ """Recursively search JSON for product data"""
+ products = []
+ if depth > 10:
+ return products
+
+ if isinstance(data, dict):
+ # Check if this looks like a Walmart product
+ if "usItemId" in data or ("name" in data and "priceInfo" in data):
+ product = self._parse_walmart_item(data)
+ if product:
+ products.append(product)
+
+ for value in data.values():
+ products.extend(self._extract_products_from_json(value, depth + 1))
+
+ elif isinstance(data, list):
+ for item in data:
+ products.extend(self._extract_products_from_json(item, depth + 1))
+
+ return products
+
+ def _parse_walmart_item(self, data: dict) -> Optional[Product]:
+ """Parse a product from Walmart's JSON data"""
+ try:
+ name = data.get("name") or data.get("title", "")
+ if not name:
+ return None
+
+ item_id = data.get("usItemId") or data.get("id", "")
+ canonical_url = data.get("canonicalUrl") or data.get("productPageUrl", "")
+
+ if canonical_url:
+ url = canonical_url if canonical_url.startswith("http") else f"{self.base_url}{canonical_url}"
+ elif item_id:
+ url = f"{self.base_url}/ip/{item_id}"
+ else:
+ return None
+
+ # Get price
+ price = None
+ price_info = data.get("priceInfo", {})
+ if isinstance(price_info, dict):
+ current_price = price_info.get("currentPrice", {})
+ if isinstance(current_price, dict):
+ price = current_price.get("priceString")
+ elif price_info.get("priceString"):
+ price = price_info.get("priceString")
+
+ if not price and "price" in data:
+ price_val = data.get("price")
+ if isinstance(price_val, (int, float)):
+ price = f"${price_val:.2f}"
+
+ # Check availability
+ in_stock = True
+ availability = data.get("availabilityStatusV2", {})
+ if isinstance(availability, dict):
+ status = availability.get("value", "").upper()
+ in_stock = status not in ["OUT_OF_STOCK", "NOT_AVAILABLE"]
+ elif data.get("availabilityStatus"):
+ in_stock = data.get("availabilityStatus") != "OUT_OF_STOCK"
+
+ # Get image
+ image_url = None
+ image_info = data.get("imageInfo", {})
+ if isinstance(image_info, dict):
+ image_url = image_info.get("thumbnailUrl") or image_info.get("url")
+ elif data.get("image"):
+ image_url = data.get("image")
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=str(item_id),
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Walmart product JSON: {e}")
+ return None
+
+ def _parse_product_card(self, card) -> Optional[Product]:
+ """Parse a product card element"""
+ try:
+ # Find link
+ link = card.select_one("a[href*='/ip/']") or card.select_one("a")
+ if not link:
+ return None
+
+ href = link.get("href", "")
+ if not href:
+ return None
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ # Get name
+ name_elem = (
+ card.select_one("[data-automation-id='product-title']")
+ or card.select_one(".product-title-link span")
+ or card.select_one("[class*='ProductTitle']")
+ or card.select_one("[class*='product-title']")
+ or link
+ )
+ name = name_elem.get_text(strip=True) if name_elem else "Unknown"
+
+ # Skip if name too short
+ if len(name) < 5:
+ return None
+
+ # Get price
+ price_elem = (
+ card.select_one("[data-automation-id='product-price']")
+ or card.select_one(".price-current")
+ or card.select_one("[class*='ProductPrice']")
+ or card.select_one("[class*='price']")
+ )
+ price = None
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ # Check stock
+ in_stock = self._check_card_stock_status(card)
+
+ # Get image
+ img = card.select_one("img")
+ image_url = None
+ if img:
+ image_url = img.get("src") or img.get("data-src")
+ if image_url and not image_url.startswith("http"):
+ image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}"
+
+ # Extract item ID from URL
+ product_id = ""
+ match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url)
+ if match:
+ product_id = match.group(1)
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Walmart product card: {e}")
+ return None
+
+ def _parse_product_link(self, link, soup) -> Optional[Product]:
+ """Parse a product from a product link element"""
+ try:
+ href = link.get("href", "")
+ if not href:
+ return None
+
+ url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+ name = link.get("aria-label", "") or link.get_text(strip=True)
+
+ if not name or len(name) < 5:
+ return None
+
+ # Extract item ID
+ product_id = ""
+ match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url)
+ if match:
+ product_id = match.group(1)
+
+ # Find price near link
+ parent = link.find_parent()
+ price = None
+ for _ in range(5):
+ if parent:
+ price_text = parent.get_text()
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
+ if price_match:
+ price = price_match.group()
+ break
+ parent = parent.find_parent()
+
+ # Find image
+ image_url = None
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ img = parent.select_one("img")
+ if img:
+ image_url = img.get("src") or img.get("data-src")
+ break
+ parent = parent.find_parent()
+
+ # Check stock
+ in_stock = True
+ parent = link.find_parent()
+ for _ in range(5):
+ if parent:
+ text = parent.get_text().lower()
+ if any(phrase in text for phrase in ["out of stock", "unavailable", "not available"]):
+ in_stock = False
+ break
+ parent = parent.find_parent()
+
+ return Product(
+ name=name,
+ url=url,
+ price=price,
+ in_stock=in_stock,
+ image_url=image_url,
+ site=self.site_name,
+ product_id=product_id,
+ )
+
+ except Exception as e:
+ logger.debug(f"Error parsing Walmart product link: {e}")
+ return None
+
+ def _check_card_stock_status(self, card) -> bool:
+ """Check if a product card indicates in-stock status"""
+ card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
+
+ out_of_stock_phrases = [
+ "out of stock",
+ "unavailable",
+ "not available",
+ "sold out",
+ "pickup not available",
+ ]
+
+ for phrase in out_of_stock_phrases:
+ if phrase in card_text:
+ return False
+
+ in_stock_phrases = [
+ "add to cart",
+ "available",
+ "in stock",
+ "pickup available",
+ "delivery available",
+ ]
+
+ for phrase in in_stock_phrases:
+ if phrase in card_text:
+ return True
+
+ return True
+
+ def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
+ """Check if a specific product is in stock"""
+ browser = get_browser()
+
+ try:
+ page, html = browser.get_page_content(product_url, timeout=30000)
+ soup = BeautifulSoup(html, "html.parser")
+ page_text = soup.get_text().lower()
+
+ out_of_stock = any(
+ phrase in page_text
+ for phrase in ["out of stock", "unavailable", "not available", "sold out"]
+ )
+
+ in_stock = not out_of_stock
+
+ # Get price
+ price = None
+ price_elem = (
+ soup.select_one("[data-automation-id='product-price']")
+ or soup.select_one("[itemprop='price']")
+ or soup.select_one("[class*='price']")
+ )
+ if price_elem:
+ price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
+ if price_match:
+ price = price_match.group()
+
+ page.close()
+ return in_stock, price
+
+ except Exception as e:
+ logger.error(f"Error checking Walmart product stock: {e}")
+ return False, None
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..8eaed50
--- /dev/null
+++ b/src/__init__.py
@@ -0,0 +1 @@
+# Core application modules
diff --git a/browser.py b/src/browser.py
similarity index 94%
rename from browser.py
rename to src/browser.py
index 3c8af36..1f6fc57 100644
--- a/browser.py
+++ b/src/browser.py
@@ -71,6 +71,8 @@ class StealthBrowser:
"""Start using an existing Chrome profile (better for bot detection)"""
self.using_persistent = True
+ logger.info(f"Using profile path: {CHROME_USER_DATA_DIR}")
+
# Launch persistent context with user's REAL Chrome (not Playwright's Chromium)
# This is needed because Chrome encrypts credentials with its own binary
self.context = self.playwright.chromium.launch_persistent_context(
@@ -84,9 +86,19 @@ class StealthBrowser:
args=[
"--disable-blink-features=AutomationControlled",
"--disable-dev-shm-usage",
+ "--no-first-run",
+ "--no-default-browser-check",
]
)
+ # Close any default blank pages that were opened
+ for page in self.context.pages:
+ if page.url == "about:blank":
+ try:
+ page.close()
+ except:
+ pass
+
# Apply stealth
stealth.apply_stealth_sync(self.context)
diff --git a/src/database.py b/src/database.py
new file mode 100644
index 0000000..465a668
--- /dev/null
+++ b/src/database.py
@@ -0,0 +1,929 @@
+"""
+SQLite database for Pokemon Stock Monitor stats tracking.
+Stores product history, price changes, stock events, and favorites.
+"""
+
+import sqlite3
+import json
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from contextlib import contextmanager
+
+logger = logging.getLogger(__name__)
+
+# Database file path
+DB_PATH = Path(__file__).parent.parent / "data" / "stats.db"
+
+
+class Database:
+ """SQLite database wrapper for stats tracking"""
+
+ def __init__(self, db_path: str = None):
+ self.db_path = db_path or str(DB_PATH)
+ self._init_schema()
+
+ @contextmanager
+ def get_connection(self):
+ """Context manager for database connections"""
+ conn = sqlite3.connect(self.db_path)
+ conn.row_factory = sqlite3.Row # Return rows as dictionaries
+ try:
+ yield conn
+ conn.commit()
+ except Exception as e:
+ conn.rollback()
+ raise e
+ finally:
+ conn.close()
+
+ def _init_schema(self):
+ """Initialize database schema"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # Products table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS products (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ url TEXT UNIQUE NOT NULL,
+ name TEXT NOT NULL,
+ site TEXT NOT NULL,
+ product_id TEXT,
+ image_url TEXT,
+ current_price TEXT,
+ in_stock BOOLEAN DEFAULT 0,
+ first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
+ last_seen DATETIME,
+ category TEXT
+ )
+ """)
+
+ # Price history table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS price_history (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ product_id INTEGER NOT NULL,
+ price TEXT,
+ recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ FOREIGN KEY (product_id) REFERENCES products(id)
+ )
+ """)
+
+ # Stock events table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS stock_events (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ product_id INTEGER NOT NULL,
+ event_type TEXT NOT NULL,
+ recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ FOREIGN KEY (product_id) REFERENCES products(id)
+ )
+ """)
+
+ # Favorites table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS favorites (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ type TEXT NOT NULL,
+ value TEXT NOT NULL,
+ display_name TEXT,
+ priority TEXT DEFAULT 'high',
+ notify_discord BOOLEAN DEFAULT 1,
+ notify_sound BOOLEAN DEFAULT 0,
+ custom_webhook TEXT,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(type, value)
+ )
+ """)
+
+ # Check logs table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS check_logs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ site TEXT,
+ products_found INTEGER DEFAULT 0,
+ new_products INTEGER DEFAULT 0,
+ restocks INTEGER DEFAULT 0,
+ duration_ms INTEGER,
+ success BOOLEAN DEFAULT 1,
+ error_message TEXT,
+ checked_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ """)
+
+ # News articles table
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS news_articles (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ source TEXT NOT NULL,
+ source_account TEXT,
+ external_id TEXT UNIQUE,
+ title TEXT,
+ content TEXT NOT NULL,
+ url TEXT,
+ author TEXT,
+ image_url TEXT,
+ published_at DATETIME,
+ fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ sentiment_score REAL,
+ sentiment_label TEXT,
+ keywords TEXT,
+ related_product_ids TEXT,
+ is_drop_related BOOLEAN DEFAULT 0,
+ is_restock_related BOOLEAN DEFAULT 0
+ )
+ """)
+
+ # Users table for multi-user support
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS users (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE,
+ zip_code TEXT,
+ radius_miles INTEGER DEFAULT 25,
+ discord_webhook TEXT,
+ notify_enabled BOOLEAN DEFAULT 1,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ last_active DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ """)
+
+ # Create indexes for common queries
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_site ON products(site)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_product ON stock_events(product_id)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_type ON stock_events(event_type)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_check_logs_site ON check_logs(site)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_source ON news_articles(source)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_sentiment ON news_articles(sentiment_label)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_users_name ON users(name)")
+
+ logger.info("Database schema initialized")
+
+ # ==================== Product Methods ====================
+
+ def get_or_create_product(self, url: str, name: str, site: str,
+ product_id: str = None, image_url: str = None,
+ price: str = None, in_stock: bool = False) -> int:
+ """Get existing product or create new one. Returns product ID."""
+ category = self._detect_category(name)
+
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # Try to get existing product
+ cursor.execute("SELECT id FROM products WHERE url = ?", (url,))
+ row = cursor.fetchone()
+
+ if row:
+ # Update existing product
+ cursor.execute("""
+ UPDATE products
+ SET name = ?, current_price = ?, in_stock = ?,
+ last_seen = CURRENT_TIMESTAMP, image_url = COALESCE(?, image_url),
+ category = COALESCE(?, category)
+ WHERE id = ?
+ """, (name, price, in_stock, image_url, category, row['id']))
+ return row['id']
+ else:
+ # Create new product
+ cursor.execute("""
+ INSERT INTO products (url, name, site, product_id, image_url,
+ current_price, in_stock, last_seen, category)
+ VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
+ """, (url, name, site, product_id, image_url, price, in_stock, category))
+ return cursor.lastrowid
+
+ def get_product(self, product_id: int) -> Optional[Dict]:
+ """Get product by ID"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM products WHERE id = ?", (product_id,))
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def get_product_by_url(self, url: str) -> Optional[Dict]:
+ """Get product by URL"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM products WHERE url = ?", (url,))
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def update_product_price(self, product_id: int, price: str):
+ """Update product price and record in history"""
+ if not price:
+ return
+
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ UPDATE products SET current_price = ?, last_seen = CURRENT_TIMESTAMP
+ WHERE id = ?
+ """, (price, product_id))
+
+ # Record in price history (only if changed)
+ self.record_price(product_id, price)
+
+ def update_product_stock(self, product_id: int, in_stock: bool):
+ """Update product stock status"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ UPDATE products SET in_stock = ?, last_seen = CURRENT_TIMESTAMP
+ WHERE id = ?
+ """, (in_stock, product_id))
+
+ def get_products(self, site: str = None, category: str = None,
+ in_stock: bool = None, favorites_only: bool = False,
+ event_type: str = None, period: str = None,
+ limit: int = 100, offset: int = 0) -> List[Dict]:
+ """Get products with optional filters
+
+ Args:
+ event_type: Filter by recent event type ('new_drop' or 'restock')
+ period: Time period for event filter ('today' or 'week')
+ """
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # If filtering by event type, use a join with stock_events
+ if event_type and period:
+ period_sql = "-1 day" if period == "today" else "-7 days"
+ query = """
+ SELECT DISTINCT p.* FROM products p
+ INNER JOIN stock_events e ON p.id = e.product_id
+ WHERE e.event_type = ?
+ AND e.recorded_at >= datetime('now', ?)
+ """
+ params = [event_type, period_sql]
+ else:
+ query = "SELECT * FROM products WHERE 1=1"
+ params = []
+
+ if site:
+ query += " AND p.site = ?" if event_type else " AND site = ?"
+ params.append(site)
+ if category:
+ query += " AND p.category = ?" if event_type else " AND category = ?"
+ params.append(category)
+ if in_stock is not None:
+ query += " AND p.in_stock = ?" if event_type else " AND in_stock = ?"
+ params.append(in_stock)
+ if favorites_only:
+ prefix = "p." if event_type else ""
+ query += f""" AND (
+ {prefix}url IN (SELECT value FROM favorites WHERE type = 'product')
+ OR {prefix}category IN (SELECT value FROM favorites WHERE type = 'category')
+ )"""
+
+ order_col = "p.last_seen" if event_type else "last_seen"
+ query += f" ORDER BY {order_col} DESC LIMIT ? OFFSET ?"
+ params.extend([limit, offset])
+
+ cursor.execute(query, params)
+ return [dict(row) for row in cursor.fetchall()]
+
+ def get_product_count(self, site: str = None, in_stock: bool = None) -> int:
+ """Get count of products matching filters"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = "SELECT COUNT(*) FROM products WHERE 1=1"
+ params = []
+
+ if site:
+ query += " AND site = ?"
+ params.append(site)
+ if in_stock is not None:
+ query += " AND in_stock = ?"
+ params.append(in_stock)
+
+ cursor.execute(query, params)
+ return cursor.fetchone()[0]
+
+ def _detect_category(self, name: str) -> str:
+ """Auto-detect product category from name"""
+ name_lower = name.lower()
+
+ if "elite trainer" in name_lower or "etb" in name_lower:
+ return "ETB"
+ elif "booster bundle" in name_lower:
+ return "Booster Bundle"
+ elif "booster box" in name_lower:
+ return "Booster Box"
+ elif "booster pack" in name_lower or "sleeved booster" in name_lower:
+ return "Booster Pack"
+ elif "collection" in name_lower:
+ return "Collection Box"
+ elif "tin" in name_lower:
+ return "Tin"
+ elif "blister" in name_lower:
+ return "Blister"
+ elif "binder" in name_lower or "album" in name_lower:
+ return "Accessories"
+ else:
+ return "Other"
+
+ # ==================== Price History Methods ====================
+
+ def record_price(self, product_id: int, price: str):
+ """Record a price point for a product"""
+ if not price:
+ return
+
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # Check if price changed from last record
+ cursor.execute("""
+ SELECT price FROM price_history
+ WHERE product_id = ?
+ ORDER BY recorded_at DESC LIMIT 1
+ """, (product_id,))
+ row = cursor.fetchone()
+
+ # Only record if price changed or no history exists
+ if not row or row['price'] != price:
+ cursor.execute("""
+ INSERT INTO price_history (product_id, price)
+ VALUES (?, ?)
+ """, (product_id, price))
+ logger.debug(f"Recorded price change for product {product_id}: {price}")
+
+ def get_price_history(self, product_id: int, days: int = 30) -> List[Dict]:
+ """Get price history for a product"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT price, recorded_at FROM price_history
+ WHERE product_id = ? AND recorded_at >= datetime('now', ?)
+ ORDER BY recorded_at ASC
+ """, (product_id, f"-{days} days"))
+ return [dict(row) for row in cursor.fetchall()]
+
+ # ==================== Stock Event Methods ====================
+
+ def record_stock_event(self, product_id: int, event_type: str):
+ """Record a stock event (new_drop, restock, in_stock, out_of_stock)"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO stock_events (product_id, event_type)
+ VALUES (?, ?)
+ """, (product_id, event_type))
+ logger.debug(f"Recorded {event_type} event for product {product_id}")
+
+ def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
+ """Get recent stock events with product info"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = """
+ SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
+ FROM stock_events e
+ JOIN products p ON e.product_id = p.id
+ """
+ params = []
+
+ if event_types:
+ placeholders = ",".join("?" * len(event_types))
+ query += f" WHERE e.event_type IN ({placeholders})"
+ params.extend(event_types)
+
+ query += " ORDER BY e.recorded_at DESC LIMIT ?"
+ params.append(limit)
+
+ cursor.execute(query, params)
+ return [dict(row) for row in cursor.fetchall()]
+
+ def get_events_today(self, event_type: str = None) -> int:
+ """Count events from today"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = """
+ SELECT COUNT(*) FROM stock_events
+ WHERE DATE(recorded_at) = DATE('now')
+ """
+ params = []
+
+ if event_type:
+ query += " AND event_type = ?"
+ params.append(event_type)
+
+ cursor.execute(query, params)
+ return cursor.fetchone()[0]
+
+ def get_events_this_week(self, event_type: str = None) -> int:
+ """Count events from this week"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = """
+ SELECT COUNT(*) FROM stock_events
+ WHERE recorded_at >= datetime('now', '-7 days')
+ """
+ params = []
+
+ if event_type:
+ query += " AND event_type = ?"
+ params.append(event_type)
+
+ cursor.execute(query, params)
+ return cursor.fetchone()[0]
+
+ # ==================== Analytics Methods ====================
+
+ def get_drop_timing_stats(self, days: int = 30) -> List[Dict]:
+ """Get drop timing by hour of day"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT
+ strftime('%H', recorded_at) as hour,
+ COUNT(*) as count
+ FROM stock_events
+ WHERE event_type IN ('new_drop', 'restock')
+ AND recorded_at >= datetime('now', ?)
+ GROUP BY hour
+ ORDER BY hour
+ """, (f"-{days} days",))
+ return [dict(row) for row in cursor.fetchall()]
+
+ def get_stock_duration_stats(self) -> List[Dict]:
+ """Get average time items stay in stock"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ # Calculate duration between in_stock and out_of_stock events
+ cursor.execute("""
+ SELECT
+ p.category,
+ AVG(
+ CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER)
+ ) as avg_minutes_in_stock
+ FROM stock_events in_event
+ JOIN stock_events out_event ON in_event.product_id = out_event.product_id
+ AND out_event.event_type = 'out_of_stock'
+ AND out_event.recorded_at > in_event.recorded_at
+ JOIN products p ON in_event.product_id = p.id
+ WHERE in_event.event_type IN ('restock', 'new_drop')
+ GROUP BY p.category
+ """)
+ return [dict(row) for row in cursor.fetchall()]
+
+ def get_site_stats(self) -> List[Dict]:
+ """Get stats per site"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT
+ site,
+ COUNT(*) as total_products,
+ SUM(CASE WHEN in_stock = 1 THEN 1 ELSE 0 END) as in_stock_count
+ FROM products
+ GROUP BY site
+ """)
+ return [dict(row) for row in cursor.fetchall()]
+
+ # ==================== Check Log Methods ====================
+
+ def log_check(self, site: str, products_found: int, new_products: int,
+ restocks: int, duration_ms: int, success: bool = True,
+ error_message: str = None):
+ """Log a monitoring check"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO check_logs (site, products_found, new_products,
+ restocks, duration_ms, success, error_message)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """, (site, products_found, new_products, restocks, duration_ms,
+ success, error_message))
+
+ def get_last_check(self, site: str = None) -> Optional[Dict]:
+ """Get the most recent check log"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = "SELECT * FROM check_logs"
+ params = []
+
+ if site:
+ query += " WHERE site = ?"
+ params.append(site)
+
+ query += " ORDER BY checked_at DESC LIMIT 1"
+ cursor.execute(query, params)
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def get_check_history(self, site: str = None, limit: int = 100) -> List[Dict]:
+ """Get check history"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = "SELECT * FROM check_logs"
+ params = []
+
+ if site:
+ query += " WHERE site = ?"
+ params.append(site)
+
+ query += " ORDER BY checked_at DESC LIMIT ?"
+ params.append(limit)
+
+ cursor.execute(query, params)
+ return [dict(row) for row in cursor.fetchall()]
+
+ # ==================== Favorites Methods ====================
+
+ def add_favorite(self, fav_type: str, value: str, display_name: str = None,
+ priority: str = "high", notify_discord: bool = True,
+ notify_sound: bool = False, custom_webhook: str = None) -> int:
+ """Add a favorite. Returns favorite ID."""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT OR REPLACE INTO favorites
+ (type, value, display_name, priority, notify_discord, notify_sound, custom_webhook)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ """, (fav_type, value, display_name or value, priority,
+ notify_discord, notify_sound, custom_webhook))
+ return cursor.lastrowid
+
+ def remove_favorite(self, favorite_id: int):
+ """Remove a favorite by ID"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("DELETE FROM favorites WHERE id = ?", (favorite_id,))
+
+ def get_favorites(self, fav_type: str = None) -> List[Dict]:
+ """Get all favorites, optionally filtered by type"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ if fav_type:
+ cursor.execute("SELECT * FROM favorites WHERE type = ? ORDER BY created_at DESC",
+ (fav_type,))
+ else:
+ cursor.execute("SELECT * FROM favorites ORDER BY created_at DESC")
+
+ return [dict(row) for row in cursor.fetchall()]
+
+ def get_favorite(self, favorite_id: int) -> Optional[Dict]:
+ """Get a favorite by ID"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM favorites WHERE id = ?", (favorite_id,))
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def update_favorite(self, favorite_id: int, **kwargs):
+ """Update favorite settings"""
+ allowed_fields = ['display_name', 'priority', 'notify_discord',
+ 'notify_sound', 'custom_webhook']
+ updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
+
+ if not updates:
+ return
+
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
+ cursor.execute(
+ f"UPDATE favorites SET {set_clause} WHERE id = ?",
+ list(updates.values()) + [favorite_id]
+ )
+
+ def check_is_favorite(self, url: str = None, category: str = None) -> Optional[Dict]:
+ """Check if a product URL or category is favorited"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ if url:
+ cursor.execute("""
+ SELECT * FROM favorites
+ WHERE type = 'product' AND value = ?
+ """, (url,))
+ row = cursor.fetchone()
+ if row:
+ return dict(row)
+
+ if category:
+ cursor.execute("""
+ SELECT * FROM favorites
+ WHERE type = 'category' AND value = ?
+ """, (category,))
+ row = cursor.fetchone()
+ if row:
+ return dict(row)
+
+ return None
+
+ # ==================== News Article Methods ====================
+
+ def add_news_article(self, source: str, content: str, source_account: str = None,
+ external_id: str = None, title: str = None, url: str = None,
+ author: str = None, image_url: str = None, published_at: str = None,
+ sentiment_score: float = None, sentiment_label: str = None,
+ keywords: List[str] = None, related_product_ids: List[int] = None,
+ is_drop_related: bool = False, is_restock_related: bool = False) -> int:
+ """Add a news article. Returns article ID."""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # Check if article already exists by external_id
+ if external_id:
+ cursor.execute("SELECT id FROM news_articles WHERE external_id = ?", (external_id,))
+ existing = cursor.fetchone()
+ if existing:
+ return existing['id']
+
+ cursor.execute("""
+ INSERT INTO news_articles (source, source_account, external_id, title, content,
+ url, author, image_url, published_at, sentiment_score,
+ sentiment_label, keywords, related_product_ids,
+ is_drop_related, is_restock_related)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """, (
+ source, source_account, external_id, title, content,
+ url, author, image_url, published_at, sentiment_score,
+ sentiment_label,
+ json.dumps(keywords) if keywords else None,
+ json.dumps(related_product_ids) if related_product_ids else None,
+ is_drop_related, is_restock_related
+ ))
+ return cursor.lastrowid
+
+ def get_news_articles(self, source: str = None, sentiment: str = None,
+ drop_related: bool = None, limit: int = 50,
+ offset: int = 0) -> List[Dict]:
+ """Get news articles with optional filters"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ query = "SELECT * FROM news_articles WHERE 1=1"
+ params = []
+
+ if source:
+ query += " AND source = ?"
+ params.append(source)
+ if sentiment:
+ query += " AND sentiment_label = ?"
+ params.append(sentiment)
+ if drop_related is not None:
+ query += " AND (is_drop_related = ? OR is_restock_related = ?)"
+ params.extend([drop_related, drop_related])
+
+ query += " ORDER BY published_at DESC, fetched_at DESC LIMIT ? OFFSET ?"
+ params.extend([limit, offset])
+
+ cursor.execute(query, params)
+ articles = []
+ for row in cursor.fetchall():
+ article = dict(row)
+ # Parse JSON fields
+ if article.get('keywords'):
+ article['keywords'] = json.loads(article['keywords'])
+ if article.get('related_product_ids'):
+ article['related_product_ids'] = json.loads(article['related_product_ids'])
+ articles.append(article)
+ return articles
+
+ def get_news_article(self, article_id: int) -> Optional[Dict]:
+ """Get a news article by ID"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM news_articles WHERE id = ?", (article_id,))
+ row = cursor.fetchone()
+ if row:
+ article = dict(row)
+ if article.get('keywords'):
+ article['keywords'] = json.loads(article['keywords'])
+ if article.get('related_product_ids'):
+ article['related_product_ids'] = json.loads(article['related_product_ids'])
+ return article
+ return None
+
+ def get_news_stats(self) -> Dict:
+ """Get news statistics"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+
+ # Total articles
+ cursor.execute("SELECT COUNT(*) FROM news_articles")
+ total = cursor.fetchone()[0]
+
+ # By source
+ cursor.execute("""
+ SELECT source, COUNT(*) as count
+ FROM news_articles GROUP BY source
+ """)
+ by_source = {row['source']: row['count'] for row in cursor.fetchall()}
+
+ # By sentiment
+ cursor.execute("""
+ SELECT sentiment_label, COUNT(*) as count
+ FROM news_articles WHERE sentiment_label IS NOT NULL
+ GROUP BY sentiment_label
+ """)
+ by_sentiment = {row['sentiment_label']: row['count'] for row in cursor.fetchall()}
+
+ # Today's articles
+ cursor.execute("""
+ SELECT COUNT(*) FROM news_articles
+ WHERE DATE(fetched_at) = DATE('now')
+ """)
+ today = cursor.fetchone()[0]
+
+ # Drop related
+ cursor.execute("""
+ SELECT COUNT(*) FROM news_articles
+ WHERE is_drop_related = 1 OR is_restock_related = 1
+ """)
+ drop_related = cursor.fetchone()[0]
+
+ return {
+ 'total': total,
+ 'today': today,
+ 'by_source': by_source,
+ 'by_sentiment': by_sentiment,
+ 'drop_related': drop_related
+ }
+
+ def update_news_correlation(self, article_id: int, related_product_ids: List[int],
+ is_drop_related: bool, is_restock_related: bool):
+ """Update news article with correlation data"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ UPDATE news_articles
+ SET related_product_ids = ?, is_drop_related = ?, is_restock_related = ?
+ WHERE id = ?
+ """, (json.dumps(related_product_ids), is_drop_related, is_restock_related, article_id))
+
+ def get_correlated_news(self, hours: int = 24) -> List[Dict]:
+ """Get news articles correlated with recent drops"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ SELECT * FROM news_articles
+ WHERE (is_drop_related = 1 OR is_restock_related = 1)
+ AND fetched_at >= datetime('now', ?)
+ ORDER BY published_at DESC
+ """, (f"-{hours} hours",))
+ articles = []
+ for row in cursor.fetchall():
+ article = dict(row)
+ if article.get('keywords'):
+ article['keywords'] = json.loads(article['keywords'])
+ if article.get('related_product_ids'):
+ article['related_product_ids'] = json.loads(article['related_product_ids'])
+ articles.append(article)
+ return articles
+
+ def delete_old_news(self, days: int = 30):
+ """Delete news articles older than specified days"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ DELETE FROM news_articles
+ WHERE fetched_at < datetime('now', ?)
+ """, (f"-{days} days",))
+ deleted = cursor.rowcount
+ logger.info(f"Deleted {deleted} old news articles")
+ return deleted
+
+ # ==================== Migration Methods ====================
+
+ def migrate_from_json(self, json_path: str):
+ """Migrate existing products.json to database"""
+ try:
+ with open(json_path, 'r') as f:
+ products = json.load(f)
+ except (FileNotFoundError, json.JSONDecodeError):
+ logger.warning(f"Could not load {json_path} for migration")
+ return
+
+ migrated = 0
+ for url, data in products.items():
+ product_id = self.get_or_create_product(
+ url=url,
+ name=data.get('name', 'Unknown'),
+ site=data.get('site', 'unknown'),
+ product_id=data.get('product_id'),
+ image_url=data.get('image_url'),
+ price=data.get('price'),
+ in_stock=data.get('in_stock', False)
+ )
+
+ # Record initial price if available
+ if data.get('price'):
+ self.record_price(product_id, data['price'])
+
+ # Record as existing product (not new_drop since it's historical)
+ if data.get('in_stock'):
+ self.record_stock_event(product_id, 'in_stock')
+
+ migrated += 1
+
+ logger.info(f"Migrated {migrated} products from {json_path}")
+
+ # ==================== User Methods ====================
+
+ def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,
+ discord_webhook: str = None, notify_enabled: bool = True) -> int:
+ """Create a new user. Returns user ID."""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("""
+ INSERT INTO users (name, zip_code, radius_miles, discord_webhook, notify_enabled)
+ VALUES (?, ?, ?, ?, ?)
+ """, (name, zip_code, radius_miles, discord_webhook, notify_enabled))
+ logger.info(f"Created user: {name}")
+ return cursor.lastrowid
+
+ def get_user(self, user_id: int) -> Optional[Dict]:
+ """Get user by ID"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,))
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def get_user_by_name(self, name: str) -> Optional[Dict]:
+ """Get user by name"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM users WHERE name = ?", (name,))
+ row = cursor.fetchone()
+ return dict(row) if row else None
+
+ def get_all_users(self) -> List[Dict]:
+ """Get all users"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT * FROM users ORDER BY name")
+ return [dict(row) for row in cursor.fetchall()]
+
+ def update_user(self, user_id: int, **kwargs) -> bool:
+ """Update user settings"""
+ allowed_fields = ['name', 'zip_code', 'radius_miles', 'discord_webhook', 'notify_enabled']
+ updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
+
+ if not updates:
+ return False
+
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
+ cursor.execute(
+ f"UPDATE users SET {set_clause}, last_active = CURRENT_TIMESTAMP WHERE id = ?",
+ list(updates.values()) + [user_id]
+ )
+ logger.info(f"Updated user {user_id}: {updates}")
+ return cursor.rowcount > 0
+
+ def delete_user(self, user_id: int) -> bool:
+ """Delete a user"""
+ with self.get_connection() as conn:
+ cursor = conn.cursor()
+ cursor.execute("DELETE FROM users WHERE id = ?", (user_id,))
+ return cursor.rowcount > 0
+
+ def update_user_location(self, user_id: int, zip_code: str, radius_miles: int = 25) -> bool:
+ """Update user's location settings"""
+ return self.update_user(user_id, zip_code=zip_code, radius_miles=radius_miles)
+
+ # ==================== Stats Summary ====================
+
+ def get_dashboard_stats(self) -> Dict:
+ """Get summary stats for dashboard"""
+ return {
+ 'total_products': self.get_product_count(),
+ 'in_stock_count': self.get_product_count(in_stock=True),
+ 'new_drops_today': self.get_events_today('new_drop'),
+ 'new_drops_week': self.get_events_this_week('new_drop'),
+ 'restocks_today': self.get_events_today('restock'),
+ 'restocks_week': self.get_events_this_week('restock'),
+ 'last_check': self.get_last_check(),
+ 'sites': self.get_site_stats(),
+ 'favorites_count': len(self.get_favorites())
+ }
+
+
+# Global database instance
+_db: Database = None
+
+
+def get_database() -> Database:
+ """Get the global database instance"""
+ global _db
+ if _db is None:
+ _db = Database()
+ return _db
diff --git a/src/discord_bot.py b/src/discord_bot.py
new file mode 100644
index 0000000..4a73de8
--- /dev/null
+++ b/src/discord_bot.py
@@ -0,0 +1,353 @@
+"""
+Pokemon Stock Monitor Discord Bot
+
+Interactive bot for setting location and checking local store stock.
+
+Commands:
+ !setlocation - Set your location
+ !stores - Find nearby stores
+ !stock - Check Pokemon TCG stock at local stores
+ !help - Show commands
+
+Setup:
+ 1. Create a bot at https://discord.com/developers/applications
+ 2. Get your bot token
+ 3. Add bot to your server with Message Content Intent enabled
+ 4. Set DISCORD_BOT_TOKEN in config.py or environment variable
+"""
+
+import os
+import asyncio
+import logging
+from datetime import datetime
+from typing import Optional
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Try to import discord.py
+try:
+ import discord
+ from discord.ext import commands
+ DISCORD_AVAILABLE = True
+except ImportError:
+ DISCORD_AVAILABLE = False
+ logger.warning("discord.py not installed. Run: pip install discord.py")
+
+from .store_locator import (
+ find_all_nearby_stores,
+ TargetLocator,
+ BestBuyLocator,
+ GameStopLocator,
+ WalmartLocator,
+ load_location_config,
+ save_location_config,
+ format_stores_for_discord
+)
+
+# User locations stored by Discord user ID
+user_locations = {}
+
+
+def load_user_locations():
+ """Load user locations from file"""
+ import json
+ from pathlib import Path
+
+ loc_file = Path(__file__).parent.parent / "data" / "user_locations.json"
+ if loc_file.exists():
+ with open(loc_file, 'r') as f:
+ return json.load(f)
+ return {}
+
+
+def save_user_locations():
+ """Save user locations to file"""
+ import json
+ from pathlib import Path
+
+ loc_file = Path(__file__).parent.parent / "data" / "user_locations.json"
+ with open(loc_file, 'w') as f:
+ json.dump(user_locations, f, indent=2)
+
+
+if DISCORD_AVAILABLE:
+
+ # Bot setup with intents
+ intents = discord.Intents.default()
+ intents.message_content = True
+
+ bot = commands.Bot(command_prefix='!', intents=intents, help_command=None)
+
+ @bot.event
+ async def on_ready():
+ global user_locations
+ user_locations = load_user_locations()
+ logger.info(f"Bot is ready! Logged in as {bot.user}")
+ logger.info(f"Loaded {len(user_locations)} user locations")
+
+ @bot.command(name='help')
+ async def help_command(ctx):
+ """Show help message"""
+ embed = discord.Embed(
+ title="Pokemon Stock Monitor Bot",
+ description="Find Pokemon TCG products at local stores!",
+ color=0xFFCC00
+ )
+
+ embed.add_field(
+ name="Commands",
+ value="""
+`!setlocation ` - Set your location (e.g., `!setlocation 90210`)
+`!stores` - Find nearby Target, Best Buy, GameStop, Walmart
+`!stock` - Check Pokemon TCG stock at your local stores
+`!stock target` - Check Target specifically
+`!mylocation` - Show your saved location
+ """,
+ inline=False
+ )
+
+ embed.add_field(
+ name="Supported Retailers",
+ value="Target, Best Buy, GameStop, Walmart",
+ inline=False
+ )
+
+ embed.set_footer(text="Pokemon Stock Monitor")
+ await ctx.send(embed=embed)
+
+ @bot.command(name='setlocation')
+ async def set_location(ctx, zip_code: str, radius: int = 25):
+ """Set user's location"""
+ # Validate zip code (basic check)
+ if not zip_code.isdigit() or len(zip_code) != 5:
+ await ctx.send("Please provide a valid 5-digit US zip code. Example: `!setlocation 90210`")
+ return
+
+ user_id = str(ctx.author.id)
+ user_locations[user_id] = {
+ "zip_code": zip_code,
+ "radius_miles": radius,
+ "set_at": datetime.now().isoformat()
+ }
+ save_user_locations()
+
+ embed = discord.Embed(
+ title="Location Saved!",
+ description=f"Your location has been set to **{zip_code}** with a **{radius} mile** search radius.",
+ color=0x00FF00
+ )
+ embed.add_field(name="Next Steps", value="Use `!stores` to find nearby stores or `!stock` to check inventory.")
+ await ctx.send(embed=embed)
+
+ @bot.command(name='mylocation')
+ async def my_location(ctx):
+ """Show user's saved location"""
+ user_id = str(ctx.author.id)
+
+ if user_id not in user_locations:
+ await ctx.send("You haven't set a location yet. Use `!setlocation ` to set one.")
+ return
+
+ loc = user_locations[user_id]
+ embed = discord.Embed(
+ title="Your Location",
+ color=0x0099FF
+ )
+ embed.add_field(name="Zip Code", value=loc["zip_code"], inline=True)
+ embed.add_field(name="Search Radius", value=f"{loc['radius_miles']} miles", inline=True)
+ await ctx.send(embed=embed)
+
+ @bot.command(name='stores')
+ async def find_stores(ctx):
+ """Find nearby stores"""
+ user_id = str(ctx.author.id)
+
+ if user_id not in user_locations:
+ await ctx.send("Please set your location first with `!setlocation `")
+ return
+
+ loc = user_locations[user_id]
+ zip_code = loc["zip_code"]
+ radius = loc["radius_miles"]
+
+ # Send "searching" message
+ searching_msg = await ctx.send(f"Searching for stores near {zip_code}...")
+
+ # Find stores (run in thread to avoid blocking)
+ loop = asyncio.get_event_loop()
+ stores = await loop.run_in_executor(
+ None,
+ find_all_nearby_stores,
+ zip_code,
+ radius
+ )
+
+ # Build embed
+ embed = discord.Embed(
+ title=f"Stores Near {zip_code}",
+ description=f"Within {radius} miles",
+ color=0xFFCC00
+ )
+
+ total_stores = 0
+ for retailer, store_list in stores.items():
+ if store_list:
+ total_stores += len(store_list)
+ stores_text = ""
+ for store in store_list[:5]: # Show top 5 per retailer
+ stores_text += f"**{store.name}** ({store.distance_miles:.1f} mi)\n"
+ stores_text += f"{store.city}, {store.state}\n\n"
+
+ embed.add_field(
+ name=f"{retailer} ({len(store_list)} stores)",
+ value=stores_text[:1024] if stores_text else "No stores found",
+ inline=False
+ )
+ else:
+ embed.add_field(
+ name=retailer,
+ value="No stores found nearby",
+ inline=False
+ )
+
+ embed.set_footer(text=f"Total: {total_stores} stores found")
+
+ await searching_msg.edit(content=None, embed=embed)
+
+ @bot.command(name='stock')
+ async def check_stock(ctx, retailer: Optional[str] = None):
+ """Check Pokemon TCG stock at local stores"""
+ user_id = str(ctx.author.id)
+
+ if user_id not in user_locations:
+ await ctx.send("Please set your location first with `!setlocation `")
+ return
+
+ loc = user_locations[user_id]
+ zip_code = loc["zip_code"]
+ radius = loc["radius_miles"]
+
+ # Determine which retailers to check
+ retailers_to_check = []
+ if retailer:
+ retailer_lower = retailer.lower()
+ if "target" in retailer_lower:
+ retailers_to_check = [("Target", TargetLocator)]
+ elif "best" in retailer_lower or "buy" in retailer_lower:
+ retailers_to_check = [("Best Buy", BestBuyLocator)]
+ elif "game" in retailer_lower or "stop" in retailer_lower:
+ retailers_to_check = [("GameStop", GameStopLocator)]
+ elif "walmart" in retailer_lower:
+ retailers_to_check = [("Walmart", WalmartLocator)]
+ else:
+ await ctx.send(f"Unknown retailer: {retailer}. Try: target, bestbuy, gamestop, walmart")
+ return
+ else:
+ # Check all
+ retailers_to_check = [
+ ("Target", TargetLocator),
+ ("Best Buy", BestBuyLocator),
+ ("GameStop", GameStopLocator),
+ ("Walmart", WalmartLocator),
+ ]
+
+ searching_msg = await ctx.send(f"Checking Pokemon TCG stock near {zip_code}...")
+
+ embed = discord.Embed(
+ title="Pokemon TCG Local Stock",
+ description=f"Checking stores near {zip_code}",
+ color=0xFF6600
+ )
+
+ for name, LocatorClass in retailers_to_check:
+ try:
+ locator = LocatorClass(zip_code, radius)
+ stores = locator.find_stores()[:3] # Check closest 3
+
+ if stores:
+ store_info = ""
+ for store in stores:
+ # Note: Full stock check requires product IDs
+ # For now, just show store info
+ store_info += f"**{store.name}**\n"
+ store_info += f"{store.address}\n"
+ store_info += f"{store.city}, {store.state} ({store.distance_miles:.1f} mi)\n"
+ if store.phone:
+ store_info += f"Phone: {store.phone}\n"
+ store_info += "\n"
+
+ embed.add_field(
+ name=f"{name}",
+ value=store_info[:1024] or "No stores found",
+ inline=False
+ )
+ else:
+ embed.add_field(name=name, value="No stores found nearby", inline=False)
+
+ except Exception as e:
+ logger.error(f"Error checking {name}: {e}")
+ embed.add_field(name=name, value="Error checking stores", inline=False)
+
+ embed.add_field(
+ name="Tip",
+ value="Call ahead to confirm Pokemon TCG availability. Stock changes frequently!",
+ inline=False
+ )
+
+ embed.set_footer(text="Pokemon Stock Monitor")
+ await searching_msg.edit(content=None, embed=embed)
+
+ @bot.command(name='ping')
+ async def ping(ctx):
+ """Check if bot is alive"""
+ await ctx.send(f"Pong! Latency: {round(bot.latency * 1000)}ms")
+
+
+def run_bot(token: str):
+ """Run the Discord bot"""
+ if not DISCORD_AVAILABLE:
+ print("discord.py is not installed!")
+ print("Run: pip install discord.py")
+ return
+
+ bot.run(token)
+
+
+if __name__ == "__main__":
+ # Try to get token from config or environment
+ token = os.environ.get("DISCORD_BOT_TOKEN")
+
+ if not token:
+ try:
+ from config import DISCORD_BOT_TOKEN
+ token = DISCORD_BOT_TOKEN
+ except ImportError:
+ pass
+
+ if not token:
+ print("=" * 60)
+ print("Discord Bot Setup")
+ print("=" * 60)
+ print()
+ print("To run the bot, you need a Discord Bot Token.")
+ print()
+ print("1. Go to https://discord.com/developers/applications")
+ print("2. Create a New Application")
+ print("3. Go to 'Bot' section, create a bot")
+ print("4. Enable 'Message Content Intent' under Privileged Intents")
+ print("5. Copy the token")
+ print()
+ print("Then either:")
+ print(" - Set DISCORD_BOT_TOKEN in config.py")
+ print(" - Or set DISCORD_BOT_TOKEN environment variable")
+ print()
+ print("To invite bot to your server:")
+ print(" - Go to OAuth2 > URL Generator")
+ print(" - Select 'bot' scope")
+ print(" - Select permissions: Send Messages, Embed Links, Read Message History")
+ print(" - Use the generated URL to invite")
+ print()
+ else:
+ print("Starting Discord bot...")
+ run_bot(token)
diff --git a/src/discord_notifier.py b/src/discord_notifier.py
new file mode 100644
index 0000000..d34db72
--- /dev/null
+++ b/src/discord_notifier.py
@@ -0,0 +1,340 @@
+"""
+Discord webhook notifications for stock alerts
+Sends rich embeds with product info and direct links
+Now supports favorites with priority notifications and custom webhooks
+"""
+
+import json
+import logging
+import requests
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict, Set
+from config import DISCORD_WEBHOOK_URL, NOTIFICATIONS_ENABLED, SKIP_DUPLICATE_SKUS
+
+logger = logging.getLogger(__name__)
+
+# Track notified products to avoid duplicates
+NOTIFIED_FILE = Path(__file__).parent.parent / "data" / "notified_products.json"
+_notified_urls: Set[str] = set()
+
+
+def _load_notified():
+ """Load previously notified product URLs"""
+ global _notified_urls
+ try:
+ if NOTIFIED_FILE.exists():
+ with open(NOTIFIED_FILE, 'r') as f:
+ _notified_urls = set(json.load(f))
+ except Exception:
+ _notified_urls = set()
+
+
+def _save_notified():
+ """Save notified product URLs"""
+ try:
+ NOTIFIED_FILE.parent.mkdir(parents=True, exist_ok=True)
+ with open(NOTIFIED_FILE, 'w') as f:
+ json.dump(list(_notified_urls), f)
+ except Exception as e:
+ logger.error(f"Failed to save notified products: {e}")
+
+
+def was_already_notified(product_url: str) -> bool:
+ """Check if we already sent a notification for this product"""
+ if not _notified_urls:
+ _load_notified()
+ return product_url in _notified_urls
+
+
+def mark_as_notified(product_url: str):
+ """Mark a product as notified"""
+ _notified_urls.add(product_url)
+ _save_notified()
+
+# Colors for different notification types
+COLOR_RESTOCK = 0x00FF00 # Green - item back in stock
+COLOR_NEW_DROP = 0x0099FF # Blue - new product listing
+COLOR_PREORDER = 0xFFAA00 # Orange - pre-order available
+COLOR_ERROR = 0xFF0000 # Red - error notification
+COLOR_FAVORITE_HIGH = 0xFF0000 # Red - high priority favorite
+COLOR_FAVORITE_MEDIUM = 0xFFAA00 # Orange - medium priority favorite
+
+# Site icons/emojis
+SITE_EMOJIS = {
+ "pokemoncenter": "\U0001F7E1", # Yellow circle
+ "target": "\U0001F534", # Red circle
+ "walmart": "\U0001F535", # Blue circle
+ "bestbuy": "\U0001F7E1", # Yellow circle
+}
+
+# Priority indicators
+PRIORITY_EMOJI = {
+ "high": "\u2B50\u2B50\u2B50", # Three stars
+ "medium": "\u2B50\u2B50", # Two stars
+ "low": "\u2B50", # One star
+ "normal": "", # No stars
+}
+
+
+def send_stock_alert(
+ product_name: str,
+ product_url: str,
+ price: str,
+ site: str,
+ alert_type: str = "restock",
+ image_url: Optional[str] = None,
+ priority_settings: Optional[Dict] = None,
+):
+ """
+ Send a Discord notification for a stock alert
+
+ Args:
+ product_name: Name of the product
+ product_url: Direct link to the product
+ price: Price string (e.g., "$49.99")
+ site: Site name (pokemoncenter, target, etc.)
+ alert_type: "restock", "new_drop", or "preorder"
+ image_url: Optional product image URL
+ priority_settings: Optional dict with favorite/priority info
+ """
+ # Check if notifications are enabled
+ if not NOTIFICATIONS_ENABLED:
+ logger.debug(f"Notifications disabled - skipping alert for: {product_name}")
+ return True # Return True so caller thinks it succeeded
+
+ # Skip duplicate notifications for new drops (restocks always go through)
+ if SKIP_DUPLICATE_SKUS and alert_type == "new_drop":
+ if was_already_notified(product_url):
+ logger.debug(f"Already notified about this product - skipping: {product_name}")
+ return True
+
+ # Determine which webhook to use
+ webhook_url = DISCORD_WEBHOOK_URL
+ if priority_settings and priority_settings.get('custom_webhook'):
+ webhook_url = priority_settings['custom_webhook']
+
+ if webhook_url == "YOUR_WEBHOOK_URL_HERE":
+ logger.error("Discord webhook URL not configured! Update config.py")
+ return False
+
+ # Check if this is a favorite
+ is_favorite = priority_settings.get('is_favorite', False) if priority_settings else False
+ priority = priority_settings.get('priority', 'normal') if priority_settings else 'normal'
+
+ # Choose color based on alert type and priority
+ if is_favorite and priority == 'high':
+ color = COLOR_FAVORITE_HIGH
+ elif is_favorite and priority == 'medium':
+ color = COLOR_FAVORITE_MEDIUM
+ elif alert_type == "restock":
+ color = COLOR_RESTOCK
+ elif alert_type == "new_drop":
+ color = COLOR_NEW_DROP
+ elif alert_type == "preorder":
+ color = COLOR_PREORDER
+ else:
+ color = COLOR_RESTOCK
+
+ # Build title with priority indicator
+ priority_indicator = PRIORITY_EMOJI.get(priority, "")
+ favorite_badge = "\u2B50 FAVORITE " if is_favorite else ""
+
+ if alert_type == "restock":
+ title = f"\U0001F6A8 {favorite_badge}RESTOCK ALERT {priority_indicator}"
+ elif alert_type == "new_drop":
+ title = f"\U0001F195 {favorite_badge}NEW DROP {priority_indicator}"
+ elif alert_type == "preorder":
+ title = f"\u23F0 {favorite_badge}PRE-ORDER AVAILABLE {priority_indicator}"
+ else:
+ title = f"\U0001F514 {favorite_badge}STOCK ALERT {priority_indicator}"
+
+ site_emoji = SITE_EMOJIS.get(site.lower(), "\U0001F6D2")
+ site_display = site.replace("pokemoncenter", "Pokemon Center").title()
+
+ # Build the embed
+ embed = {
+ "title": title.strip(),
+ "description": f"**{product_name}**",
+ "url": product_url,
+ "color": color,
+ "fields": [
+ {"name": "Price", "value": price or "See link", "inline": True},
+ {"name": "Store", "value": f"{site_emoji} {site_display}", "inline": True},
+ {"name": "Link", "value": f"[\U0001F6D2 BUY NOW]({product_url})", "inline": False},
+ ],
+ "footer": {"text": "Pokemon Stock Monitor"},
+ "timestamp": datetime.utcnow().isoformat(),
+ }
+
+ # Add priority field for favorites
+ if is_favorite:
+ embed["fields"].insert(0, {
+ "name": "Priority",
+ "value": f"{priority_indicator} {priority.upper()}",
+ "inline": True
+ })
+
+ if image_url:
+ embed["thumbnail"] = {"url": image_url}
+
+ # Ping @everyone for high priority favorites, or just regular ping otherwise
+ if is_favorite and priority == 'high':
+ content = "@everyone \U0001F6A8 HIGH PRIORITY ALERT!"
+ else:
+ content = "@everyone"
+
+ payload = {
+ "content": content,
+ "embeds": [embed],
+ }
+
+ try:
+ response = requests.post(
+ webhook_url,
+ json=payload,
+ timeout=10,
+ )
+ response.raise_for_status()
+ logger.info(f"Discord notification sent for: {product_name} (priority: {priority})")
+ # Track that we notified about this product
+ if alert_type == "new_drop":
+ mark_as_notified(product_url)
+ return True
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Failed to send Discord notification: {e}")
+ return False
+
+
+def send_stock_alert_with_priority(product, alert_type: str = "restock",
+ priority_settings: Optional[Dict] = None):
+ """
+ Convenience method to send alert from Product object with priority settings
+
+ Args:
+ product: Product object from scraper
+ alert_type: "restock", "new_drop", or "preorder"
+ priority_settings: Optional dict with favorite/priority info
+ """
+ return send_stock_alert(
+ product_name=product.name,
+ product_url=product.url,
+ price=product.price,
+ site=product.site,
+ alert_type=alert_type,
+ image_url=product.image_url,
+ priority_settings=priority_settings
+ )
+
+
+def send_error_notification(error_message: str, site: str = "Unknown"):
+ """Send an error notification to Discord"""
+ if not NOTIFICATIONS_ENABLED:
+ return True
+ if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
+ return False
+
+ embed = {
+ "title": "\u26A0\uFE0F Monitor Error",
+ "description": error_message,
+ "color": COLOR_ERROR,
+ "fields": [{"name": "Site", "value": site, "inline": True}],
+ "footer": {"text": "Pokemon Stock Monitor"},
+ "timestamp": datetime.utcnow().isoformat(),
+ }
+
+ payload = {"embeds": [embed]}
+
+ try:
+ response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
+ response.raise_for_status()
+ return True
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Failed to send error notification: {e}")
+ return False
+
+
+def send_startup_notification():
+ """Send a notification that the monitor has started"""
+ if not NOTIFICATIONS_ENABLED:
+ return True
+ if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
+ return False
+
+ embed = {
+ "title": "\u2705 Monitor Started",
+ "description": "Pokemon Stock Monitor is now running and watching for restocks!",
+ "color": 0x00FF00,
+ "footer": {"text": "Pokemon Stock Monitor"},
+ "timestamp": datetime.utcnow().isoformat(),
+ }
+
+ payload = {"embeds": [embed]}
+
+ try:
+ response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
+ response.raise_for_status()
+ return True
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Failed to send startup notification: {e}")
+ return False
+
+
+def test_webhook():
+ """Test the Discord webhook connection"""
+ print("Testing Discord webhook...")
+
+ if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
+ print("ERROR: Webhook URL not configured!")
+ print("Edit config.py and set DISCORD_WEBHOOK_URL")
+ return False
+
+ # Send a test notification
+ success = send_stock_alert(
+ product_name="Test Product - Pokemon TCG Booster",
+ product_url="https://www.pokemoncenter.com/test",
+ price="$4.99",
+ site="pokemoncenter",
+ alert_type="restock",
+ )
+
+ if success:
+ print("SUCCESS! Check your Discord channel for the test message.")
+ else:
+ print("FAILED! Check the webhook URL and try again.")
+
+ return success
+
+
+def test_priority_webhook():
+ """Test a high priority favorite notification"""
+ print("Testing high priority webhook...")
+
+ if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE":
+ print("ERROR: Webhook URL not configured!")
+ return False
+
+ success = send_stock_alert(
+ product_name="Chaos Rising Elite Trainer Box",
+ product_url="https://www.pokemoncenter.com/test",
+ price="$54.99",
+ site="pokemoncenter",
+ alert_type="new_drop",
+ priority_settings={
+ 'is_favorite': True,
+ 'priority': 'high',
+ 'custom_webhook': None
+ }
+ )
+
+ if success:
+ print("SUCCESS! Check your Discord channel for the priority test message.")
+ else:
+ print("FAILED! Check the webhook URL and try again.")
+
+ return success
+
+
+if __name__ == "__main__":
+ # Run webhook test
+ test_webhook()
diff --git a/src/favorites.py b/src/favorites.py
new file mode 100644
index 0000000..652e8df
--- /dev/null
+++ b/src/favorites.py
@@ -0,0 +1,192 @@
+"""
+Favorites system for Pokemon Stock Monitor.
+Handles priority products and categories with custom notification settings.
+"""
+
+import logging
+from typing import Optional, List, Dict, Any
+from .database import get_database
+
+logger = logging.getLogger(__name__)
+
+
+# Common category keywords for auto-detection
+CATEGORY_KEYWORDS = {
+ "ETB": ["elite trainer", "etb"],
+ "Booster Bundle": ["booster bundle"],
+ "Booster Box": ["booster box"],
+ "Booster Pack": ["booster pack", "sleeved booster"],
+ "Collection Box": ["collection", "premium collection", "special collection"],
+ "Tin": ["tin", "mini tin"],
+ "Blister": ["blister", "3-pack", "check lane"],
+ "Accessories": ["binder", "album", "sleeves", "deck box", "playmat"],
+}
+
+# Known Pokemon TCG sets for category matching
+KNOWN_SETS = [
+ "chaos rising",
+ "prismatic evolutions",
+ "surging sparks",
+ "twilight masquerade",
+ "temporal forces",
+ "paldean fates",
+ "obsidian flames",
+ "paldea evolved",
+ "scarlet & violet",
+ "crown zenith",
+ "silver tempest",
+ "lost origin",
+ "pokemon go",
+ "astral radiance",
+ "brilliant stars",
+ "fusion strike",
+ "celebrations",
+]
+
+
+class FavoritesManager:
+ """Manages favorites and priority notifications"""
+
+ def __init__(self):
+ self.db = get_database()
+
+ def add_product_favorite(self, url: str, display_name: str = None,
+ priority: str = "high", notify_discord: bool = True,
+ notify_sound: bool = False, custom_webhook: str = None) -> int:
+ """Add a product URL as favorite"""
+ return self.db.add_favorite(
+ fav_type="product",
+ value=url,
+ display_name=display_name,
+ priority=priority,
+ notify_discord=notify_discord,
+ notify_sound=notify_sound,
+ custom_webhook=custom_webhook
+ )
+
+ def add_category_favorite(self, category: str, display_name: str = None,
+ priority: str = "high", notify_discord: bool = True,
+ notify_sound: bool = False, custom_webhook: str = None) -> int:
+ """Add a category (e.g., 'ETB', 'Chaos Rising') as favorite"""
+ return self.db.add_favorite(
+ fav_type="category",
+ value=category.lower(),
+ display_name=display_name or category,
+ priority=priority,
+ notify_discord=notify_discord,
+ notify_sound=notify_sound,
+ custom_webhook=custom_webhook
+ )
+
+ def remove_favorite(self, favorite_id: int):
+ """Remove a favorite"""
+ self.db.remove_favorite(favorite_id)
+
+ def get_all_favorites(self) -> List[Dict]:
+ """Get all favorites"""
+ return self.db.get_favorites()
+
+ def get_product_favorites(self) -> List[Dict]:
+ """Get product favorites only"""
+ return self.db.get_favorites(fav_type="product")
+
+ def get_category_favorites(self) -> List[Dict]:
+ """Get category favorites only"""
+ return self.db.get_favorites(fav_type="category")
+
+ def update_favorite(self, favorite_id: int, **kwargs):
+ """Update favorite settings"""
+ self.db.update_favorite(favorite_id, **kwargs)
+
+ def check_product_priority(self, url: str, name: str, category: str = None) -> Optional[Dict]:
+ """
+ Check if a product matches any favorites.
+ Returns favorite info if matched, None otherwise.
+ """
+ # Check direct URL match
+ fav = self.db.check_is_favorite(url=url)
+ if fav:
+ logger.debug(f"Product {name} matched favorite URL")
+ return fav
+
+ # Check category match
+ if category:
+ fav = self.db.check_is_favorite(category=category.lower())
+ if fav:
+ logger.debug(f"Product {name} matched category favorite: {category}")
+ return fav
+
+ # Check if product name contains any category favorite keywords
+ name_lower = name.lower()
+ category_favorites = self.get_category_favorites()
+
+ for fav in category_favorites:
+ keyword = fav['value'].lower()
+ if keyword in name_lower:
+ logger.debug(f"Product {name} matched keyword favorite: {keyword}")
+ return fav
+
+ return None
+
+ def get_notification_settings(self, url: str, name: str, category: str = None) -> Dict:
+ """
+ Get notification settings for a product based on favorites.
+ Returns default settings if not a favorite.
+ """
+ favorite = self.check_product_priority(url, name, category)
+
+ if favorite:
+ return {
+ 'is_favorite': True,
+ 'priority': favorite['priority'],
+ 'notify_discord': bool(favorite['notify_discord']),
+ 'notify_sound': bool(favorite['notify_sound']),
+ 'custom_webhook': favorite['custom_webhook'],
+ 'display_name': favorite['display_name']
+ }
+ else:
+ return {
+ 'is_favorite': False,
+ 'priority': 'normal',
+ 'notify_discord': True,
+ 'notify_sound': False,
+ 'custom_webhook': None,
+ 'display_name': None
+ }
+
+ def get_suggested_categories(self) -> List[str]:
+ """Get list of suggested categories for favorites"""
+ categories = list(CATEGORY_KEYWORDS.keys())
+ categories.extend([s.title() for s in KNOWN_SETS])
+ return sorted(set(categories))
+
+ def detect_matching_categories(self, name: str) -> List[str]:
+ """Detect which categories a product name matches"""
+ name_lower = name.lower()
+ matches = []
+
+ # Check product type categories
+ for category, keywords in CATEGORY_KEYWORDS.items():
+ for keyword in keywords:
+ if keyword in name_lower:
+ matches.append(category)
+ break
+
+ # Check set names
+ for set_name in KNOWN_SETS:
+ if set_name in name_lower:
+ matches.append(set_name.title())
+
+ return matches
+
+
+# Global favorites manager instance
+_favorites: FavoritesManager = None
+
+
+def get_favorites_manager() -> FavoritesManager:
+ """Get the global favorites manager instance"""
+ global _favorites
+ if _favorites is None:
+ _favorites = FavoritesManager()
+ return _favorites
diff --git a/src/news/__init__.py b/src/news/__init__.py
new file mode 100644
index 0000000..31f0f5c
--- /dev/null
+++ b/src/news/__init__.py
@@ -0,0 +1,9 @@
+"""
+News aggregation module for Pokemon Stock Monitor.
+Fetches news from Twitter, Pokemon.com, and manual Discord input.
+"""
+
+from .base import NewsArticle
+from .sentiment import analyze_sentiment, extract_keywords
+
+__all__ = ['NewsArticle', 'analyze_sentiment', 'extract_keywords']
diff --git a/src/news/base.py b/src/news/base.py
new file mode 100644
index 0000000..de25168
--- /dev/null
+++ b/src/news/base.py
@@ -0,0 +1,96 @@
+"""
+Base classes and data structures for news aggregation.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional, List
+
+
+@dataclass
+class NewsArticle:
+ """Represents a news article from any source"""
+ source: str # 'twitter', 'pokemon_official', 'discord_manual'
+ source_account: str # '@PokemonRestocks', 'Pokemon.com', 'PokePings Discord'
+ content: str # Tweet text, article summary, or message content
+ author: str # Username or account name
+ published_at: datetime
+
+ # Optional fields
+ external_id: Optional[str] = None # Tweet ID, article URL slug
+ title: Optional[str] = None # For official news articles
+ url: Optional[str] = None # Link to original source
+ image_url: Optional[str] = None # Attached image if any
+
+ # Sentiment analysis results (populated after analysis)
+ sentiment_score: Optional[float] = None # -1.0 to 1.0
+ sentiment_label: Optional[str] = None # 'positive', 'negative', 'neutral'
+ keywords: List[str] = field(default_factory=list) # Extracted keywords
+
+ # Correlation with stock events (populated after correlation)
+ related_product_ids: List[int] = field(default_factory=list)
+ is_drop_related: bool = False
+ is_restock_related: bool = False
+
+ def __hash__(self):
+ """Hash by external_id or content for deduplication"""
+ if self.external_id:
+ return hash(self.external_id)
+ return hash((self.source, self.content[:100], self.published_at))
+
+ def __eq__(self, other):
+ if not isinstance(other, NewsArticle):
+ return False
+ if self.external_id and other.external_id:
+ return self.external_id == other.external_id
+ return (self.source == other.source and
+ self.content[:100] == other.content[:100] and
+ self.published_at == other.published_at)
+
+ def to_dict(self) -> dict:
+ """Convert to dictionary for database storage"""
+ return {
+ 'source': self.source,
+ 'source_account': self.source_account,
+ 'external_id': self.external_id,
+ 'title': self.title,
+ 'content': self.content,
+ 'url': self.url,
+ 'author': self.author,
+ 'image_url': self.image_url,
+ 'published_at': self.published_at.isoformat() if self.published_at else None,
+ 'sentiment_score': self.sentiment_score,
+ 'sentiment_label': self.sentiment_label,
+ 'keywords': self.keywords,
+ 'related_product_ids': self.related_product_ids,
+ 'is_drop_related': self.is_drop_related,
+ 'is_restock_related': self.is_restock_related
+ }
+
+ @classmethod
+ def from_dict(cls, data: dict) -> 'NewsArticle':
+ """Create from dictionary (database row)"""
+ published_at = data.get('published_at')
+ if published_at and isinstance(published_at, str):
+ try:
+ published_at = datetime.fromisoformat(published_at)
+ except ValueError:
+ published_at = datetime.now()
+
+ return cls(
+ source=data['source'],
+ source_account=data.get('source_account', ''),
+ content=data['content'],
+ author=data.get('author', ''),
+ published_at=published_at or datetime.now(),
+ external_id=data.get('external_id'),
+ title=data.get('title'),
+ url=data.get('url'),
+ image_url=data.get('image_url'),
+ sentiment_score=data.get('sentiment_score'),
+ sentiment_label=data.get('sentiment_label'),
+ keywords=data.get('keywords', []),
+ related_product_ids=data.get('related_product_ids', []),
+ is_drop_related=data.get('is_drop_related', False),
+ is_restock_related=data.get('is_restock_related', False)
+ )
diff --git a/src/news/pokemon_fetcher.py b/src/news/pokemon_fetcher.py
new file mode 100644
index 0000000..858f2fb
--- /dev/null
+++ b/src/news/pokemon_fetcher.py
@@ -0,0 +1,222 @@
+"""
+Fetches official Pokemon TCG news from Pokemon.com
+"""
+
+import logging
+import re
+import requests
+from datetime import datetime
+from typing import List, Optional
+from bs4 import BeautifulSoup
+
+from .base import NewsArticle
+from .sentiment import analyze_article
+
+logger = logging.getLogger(__name__)
+
+# Pokemon.com TCG news URL
+POKEMON_NEWS_URL = "https://www.pokemon.com/us/pokemon-tcg-news/"
+
+# Request headers to mimic browser
+HEADERS = {
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+ 'Accept-Language': 'en-US,en;q=0.5',
+}
+
+
+class PokemonNewsFetcher:
+ """Fetches news from Pokemon.com TCG news page"""
+
+ def __init__(self):
+ self.session = requests.Session()
+ self.session.headers.update(HEADERS)
+
+ def fetch_latest(self, limit: int = 20) -> List[NewsArticle]:
+ """
+ Fetch latest Pokemon TCG news articles.
+
+ Args:
+ limit: Maximum number of articles to fetch
+
+ Returns:
+ List of NewsArticle objects
+ """
+ try:
+ response = self.session.get(POKEMON_NEWS_URL, timeout=30)
+ response.raise_for_status()
+ return self._parse_news_page(response.text, limit)
+ except requests.RequestException as e:
+ logger.error(f"Failed to fetch Pokemon news: {e}")
+ return []
+
+ def _parse_news_page(self, html: str, limit: int) -> List[NewsArticle]:
+ """Parse the news page HTML and extract articles"""
+ soup = BeautifulSoup(html, 'html.parser')
+ articles = []
+
+ # Find article containers - Pokemon.com uses various class names
+ # Try multiple selectors for robustness
+ article_containers = soup.select('article, .news-article, .article-item, .content-block')
+
+ if not article_containers:
+ # Fallback: look for links in the main content area
+ article_containers = soup.select('.main-content a, .news-list a, section a')
+
+ for container in article_containers[:limit]:
+ try:
+ article = self._parse_article(container)
+ if article:
+ articles.append(article)
+ except Exception as e:
+ logger.debug(f"Failed to parse article: {e}")
+ continue
+
+ logger.info(f"Fetched {len(articles)} Pokemon.com news articles")
+ return articles
+
+ def _parse_article(self, container) -> Optional[NewsArticle]:
+ """Parse a single article container"""
+ # Extract title
+ title_elem = container.select_one('h1, h2, h3, h4, .title, .headline')
+ if not title_elem:
+ # Check if container itself is a link with text
+ title = container.get_text(strip=True)
+ if not title or len(title) < 10:
+ return None
+ else:
+ title = title_elem.get_text(strip=True)
+
+ # Extract URL
+ link = container.select_one('a[href]')
+ if not link:
+ if container.name == 'a':
+ link = container
+ else:
+ return None
+
+ url = link.get('href', '')
+ if url and not url.startswith('http'):
+ url = f"https://www.pokemon.com{url}"
+
+ # Skip non-TCG articles
+ if url and '/pokemon-tcg' not in url.lower() and 'tcg' not in title.lower():
+ return None
+
+ # Extract summary/description
+ summary_elem = container.select_one('p, .summary, .description, .excerpt')
+ summary = summary_elem.get_text(strip=True) if summary_elem else title
+
+ # Extract image
+ img_elem = container.select_one('img')
+ image_url = None
+ if img_elem:
+ image_url = img_elem.get('src') or img_elem.get('data-src')
+ if image_url and not image_url.startswith('http'):
+ image_url = f"https://www.pokemon.com{image_url}"
+
+ # Extract date (if available)
+ date_elem = container.select_one('time, .date, .published-date')
+ published_at = datetime.now()
+ if date_elem:
+ date_str = date_elem.get('datetime') or date_elem.get_text(strip=True)
+ published_at = self._parse_date(date_str) or datetime.now()
+
+ # Generate external ID from URL
+ external_id = f"pokemon_{self._url_to_id(url)}"
+
+ # Analyze sentiment
+ analysis = analyze_article(summary, title)
+
+ article = NewsArticle(
+ source='pokemon_official',
+ source_account='Pokemon.com',
+ external_id=external_id,
+ title=title,
+ content=summary,
+ url=url,
+ author='The Pokemon Company',
+ image_url=image_url,
+ published_at=published_at,
+ sentiment_score=analysis['sentiment_score'],
+ sentiment_label=analysis['sentiment_label'],
+ keywords=analysis['keywords'],
+ is_drop_related=analysis['is_drop_related'],
+ is_restock_related=analysis['is_restock_related']
+ )
+
+ return article
+
+ def _parse_date(self, date_str: str) -> Optional[datetime]:
+ """Parse various date formats"""
+ if not date_str:
+ return None
+
+ # Try ISO format first
+ try:
+ return datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+ except ValueError:
+ pass
+
+ # Try common formats
+ formats = [
+ '%B %d, %Y', # January 15, 2024
+ '%b %d, %Y', # Jan 15, 2024
+ '%m/%d/%Y', # 01/15/2024
+ '%Y-%m-%d', # 2024-01-15
+ ]
+
+ for fmt in formats:
+ try:
+ return datetime.strptime(date_str.strip(), fmt)
+ except ValueError:
+ continue
+
+ return None
+
+ def _url_to_id(self, url: str) -> str:
+ """Convert URL to a simple ID"""
+ if not url:
+ return str(hash(datetime.now()))
+
+ # Extract the last path segment
+ parts = url.rstrip('/').split('/')
+ slug = parts[-1] if parts else ''
+
+ # Remove query params
+ slug = slug.split('?')[0]
+
+ # If slug is too long or not useful, use hash
+ if not slug or len(slug) > 100:
+ return str(abs(hash(url)))
+
+ return slug
+
+
+def fetch_pokemon_news(limit: int = 20) -> List[NewsArticle]:
+ """
+ Convenience function to fetch Pokemon.com news.
+
+ Args:
+ limit: Maximum number of articles to fetch
+
+ Returns:
+ List of NewsArticle objects
+ """
+ fetcher = PokemonNewsFetcher()
+ return fetcher.fetch_latest(limit)
+
+
+# Allow running as a standalone script for testing
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO)
+
+ print("Fetching Pokemon.com TCG news...")
+ articles = fetch_pokemon_news(limit=10)
+
+ for i, article in enumerate(articles, 1):
+ print(f"\n{i}. {article.title}")
+ print(f" Source: {article.source_account}")
+ print(f" Sentiment: {article.sentiment_label} ({article.sentiment_score:.2f})")
+ print(f" Keywords: {', '.join(article.keywords) if article.keywords else 'None'}")
+ print(f" URL: {article.url}")
diff --git a/src/news/sentiment.py b/src/news/sentiment.py
new file mode 100644
index 0000000..405404a
--- /dev/null
+++ b/src/news/sentiment.py
@@ -0,0 +1,218 @@
+"""
+Sentiment analysis for Pokemon TCG news articles.
+Uses VADER (Valence Aware Dictionary and sEntiment Reasoner) from NLTK.
+"""
+
+import re
+import logging
+from typing import Tuple, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Try to import NLTK's VADER
+try:
+ import nltk
+ from nltk.sentiment.vader import SentimentIntensityAnalyzer
+ NLTK_AVAILABLE = True
+except ImportError:
+ NLTK_AVAILABLE = False
+ logger.warning("NLTK not installed. Sentiment analysis will use fallback method.")
+
+
+# Keywords that indicate stock-related news
+STOCK_KEYWORDS = {
+ 'positive': [
+ 'restock', 'restocked', 'restocking',
+ 'live', 'in stock', 'instock', 'in-stock',
+ 'available', 'back in stock', 'just dropped',
+ 'drop', 'dropped', 'dropping', 'new drop',
+ 'go go go', 'hurry', 'quick', 'fast',
+ 'preorder', 'pre-order', 'pre order',
+ 'release', 'released', 'launching'
+ ],
+ 'negative': [
+ 'sold out', 'soldout', 'oos', 'out of stock',
+ 'unavailable', 'gone', 'missed', 'too late',
+ 'cancelled', 'canceled', 'delayed',
+ 'scalper', 'scalped', 'bot', 'botted'
+ ],
+ 'neutral': [
+ 'announcement', 'announced', 'reveal', 'revealed',
+ 'coming soon', 'upcoming', 'rumor', 'leak',
+ 'spoiler', 'preview', 'teaser'
+ ]
+}
+
+# Pokemon TCG product keywords for extraction
+PRODUCT_KEYWORDS = [
+ 'etb', 'elite trainer box', 'elite trainer',
+ 'booster box', 'booster bundle', 'booster pack',
+ 'collection', 'premium collection', 'super premium',
+ 'tin', 'blister', 'pack', 'box',
+ 'prismatic evolutions', 'surging sparks', 'stellar crown',
+ 'twilight masquerade', 'temporal forces', 'paldean fates',
+ 'obsidian flames', 'scarlet violet', '151', 'sv',
+ 'charizard', 'pikachu', 'mewtwo', 'mew',
+ 'pokemon center', 'target', 'walmart', 'best buy', 'gamestop'
+]
+
+
+def _ensure_vader_data():
+ """Download VADER lexicon if not present"""
+ if not NLTK_AVAILABLE:
+ return False
+ try:
+ nltk.data.find('sentiment/vader_lexicon.zip')
+ return True
+ except LookupError:
+ try:
+ logger.info("Downloading VADER lexicon...")
+ nltk.download('vader_lexicon', quiet=True)
+ return True
+ except Exception as e:
+ logger.error(f"Failed to download VADER lexicon: {e}")
+ return False
+
+
+def analyze_sentiment(text: str) -> Tuple[float, str]:
+ """
+ Analyze sentiment of text using VADER.
+
+ Args:
+ text: The text to analyze
+
+ Returns:
+ Tuple of (compound_score, label)
+ - compound_score: float from -1.0 (negative) to 1.0 (positive)
+ - label: 'positive', 'negative', or 'neutral'
+ """
+ if not text:
+ return 0.0, 'neutral'
+
+ # Try VADER first
+ if NLTK_AVAILABLE and _ensure_vader_data():
+ try:
+ sia = SentimentIntensityAnalyzer()
+ scores = sia.polarity_scores(text)
+ compound = scores['compound']
+
+ if compound >= 0.05:
+ label = 'positive'
+ elif compound <= -0.05:
+ label = 'negative'
+ else:
+ label = 'neutral'
+
+ return compound, label
+ except Exception as e:
+ logger.warning(f"VADER analysis failed: {e}, using fallback")
+
+ # Fallback: simple keyword-based sentiment
+ return _fallback_sentiment(text)
+
+
+def _fallback_sentiment(text: str) -> Tuple[float, str]:
+ """Simple keyword-based sentiment analysis as fallback"""
+ text_lower = text.lower()
+
+ positive_count = sum(1 for kw in STOCK_KEYWORDS['positive'] if kw in text_lower)
+ negative_count = sum(1 for kw in STOCK_KEYWORDS['negative'] if kw in text_lower)
+
+ # Simple scoring
+ if positive_count > negative_count:
+ score = min(0.5 + (positive_count * 0.1), 1.0)
+ return score, 'positive'
+ elif negative_count > positive_count:
+ score = max(-0.5 - (negative_count * 0.1), -1.0)
+ return score, 'negative'
+ else:
+ return 0.0, 'neutral'
+
+
+def extract_keywords(text: str) -> List[str]:
+ """
+ Extract relevant Pokemon TCG keywords from text.
+
+ Args:
+ text: The text to analyze
+
+ Returns:
+ List of found keywords
+ """
+ if not text:
+ return []
+
+ text_lower = text.lower()
+ found = []
+
+ for keyword in PRODUCT_KEYWORDS:
+ if keyword in text_lower:
+ found.append(keyword)
+
+ # Remove duplicates while preserving order
+ seen = set()
+ unique = []
+ for kw in found:
+ if kw not in seen:
+ seen.add(kw)
+ unique.append(kw)
+
+ return unique
+
+
+def detect_stock_status(text: str) -> Tuple[bool, bool]:
+ """
+ Detect if news is related to drops or restocks.
+
+ Args:
+ text: The text to analyze
+
+ Returns:
+ Tuple of (is_drop_related, is_restock_related)
+ """
+ if not text:
+ return False, False
+
+ text_lower = text.lower()
+
+ # Check for drop-related keywords
+ drop_keywords = ['new drop', 'just dropped', 'dropping', 'release', 'launched', 'preorder', 'pre-order']
+ is_drop = any(kw in text_lower for kw in drop_keywords)
+
+ # Check for restock-related keywords
+ restock_keywords = ['restock', 'back in stock', 'restocked', 'restocking', 'available again']
+ is_restock = any(kw in text_lower for kw in restock_keywords)
+
+ # Generic "in stock" or "live" could be either
+ if not is_drop and not is_restock:
+ if any(kw in text_lower for kw in ['live', 'in stock', 'available', 'go go']):
+ # Default to restock if not clearly a new release
+ is_restock = True
+
+ return is_drop, is_restock
+
+
+def analyze_article(text: str, title: str = None) -> dict:
+ """
+ Full analysis of a news article.
+
+ Args:
+ text: Article content
+ title: Optional title for additional context
+
+ Returns:
+ Dictionary with analysis results
+ """
+ full_text = f"{title or ''} {text}".strip()
+
+ sentiment_score, sentiment_label = analyze_sentiment(full_text)
+ keywords = extract_keywords(full_text)
+ is_drop, is_restock = detect_stock_status(full_text)
+
+ return {
+ 'sentiment_score': sentiment_score,
+ 'sentiment_label': sentiment_label,
+ 'keywords': keywords,
+ 'is_drop_related': is_drop,
+ 'is_restock_related': is_restock
+ }
diff --git a/src/product_tracker.py b/src/product_tracker.py
new file mode 100644
index 0000000..cf5989b
--- /dev/null
+++ b/src/product_tracker.py
@@ -0,0 +1,246 @@
+"""
+Product tracker - keeps track of known products to detect new drops and restocks
+Now integrates with SQLite database for historical tracking.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, List, Set, Optional, Tuple
+from dataclasses import asdict
+from datetime import datetime
+
+from scrapers.base import Product
+from .database import get_database
+from .favorites import get_favorites_manager
+
+logger = logging.getLogger(__name__)
+
+# File to store known products (kept for backwards compatibility)
+PRODUCTS_FILE = Path(__file__).parent.parent / "data" / "products.json"
+
+
+class ProductTracker:
+ """Tracks known products to detect new listings and stock changes"""
+
+ def __init__(self, products_file: Path = PRODUCTS_FILE):
+ self.products_file = products_file
+ self.products: Dict[str, dict] = {} # URL -> product data
+ self.db = get_database()
+ self.favorites = get_favorites_manager()
+ self.load()
+
+ def load(self):
+ """Load products from file"""
+ if self.products_file.exists():
+ try:
+ with open(self.products_file, "r", encoding="utf-8") as f:
+ self.products = json.load(f)
+ logger.info(f"Loaded {len(self.products)} tracked products")
+ except (json.JSONDecodeError, IOError) as e:
+ logger.error(f"Error loading products file: {e}")
+ self.products = {}
+ else:
+ self.products = {}
+ logger.info("No existing products file, starting fresh")
+
+ def save(self):
+ """Save products to file"""
+ try:
+ with open(self.products_file, "w", encoding="utf-8") as f:
+ json.dump(self.products, f, indent=2, ensure_ascii=False)
+ logger.debug(f"Saved {len(self.products)} products")
+ except IOError as e:
+ logger.error(f"Error saving products file: {e}")
+
+ def process_products(self, products: List[Product]) -> Tuple[List[Product], List[Product]]:
+ """
+ Process a list of scraped products and detect changes.
+ Also logs events to database for stats tracking.
+
+ Args:
+ products: List of products from scraper
+
+ Returns:
+ Tuple of (new_products, restocked_products)
+ """
+ new_products = []
+ restocked_products = []
+
+ for product in products:
+ url = product.url
+ now = datetime.now().isoformat()
+
+ # Get or create product in database
+ db_product_id = self.db.get_or_create_product(
+ url=url,
+ name=product.name,
+ site=product.site,
+ product_id=product.product_id,
+ image_url=product.image_url,
+ price=product.price,
+ in_stock=product.in_stock
+ )
+
+ if url not in self.products:
+ # New product!
+ new_products.append(product)
+ self.products[url] = {
+ "name": product.name,
+ "url": url,
+ "price": product.price,
+ "in_stock": product.in_stock,
+ "image_url": product.image_url,
+ "site": product.site,
+ "product_id": product.product_id,
+ "first_seen": now,
+ "last_seen": now,
+ "last_in_stock": now if product.in_stock else None,
+ "db_id": db_product_id,
+ }
+ logger.info(f"NEW PRODUCT: {product.name}")
+
+ # Log new drop event to database
+ self.db.record_stock_event(db_product_id, "new_drop")
+ if product.in_stock:
+ self.db.record_stock_event(db_product_id, "in_stock")
+
+ # Record initial price
+ if product.price:
+ self.db.record_price(db_product_id, product.price)
+
+ else:
+ # Existing product - check for changes
+ existing = self.products[url]
+ was_in_stock = existing.get("in_stock", False)
+ old_price = existing.get("price")
+
+ # Update last seen
+ existing["last_seen"] = now
+ existing["image_url"] = product.image_url or existing.get("image_url")
+ existing["db_id"] = db_product_id
+
+ # Check for price change
+ if product.price and product.price != old_price:
+ existing["price"] = product.price
+ self.db.record_price(db_product_id, product.price)
+ logger.debug(f"Price change for {product.name}: {old_price} -> {product.price}")
+
+ # Check for restock
+ if product.in_stock and not was_in_stock:
+ # RESTOCK!
+ restocked_products.append(product)
+ existing["last_in_stock"] = now
+ logger.info(f"RESTOCK: {product.name}")
+
+ # Log restock event to database
+ self.db.record_stock_event(db_product_id, "restock")
+ self.db.record_stock_event(db_product_id, "in_stock")
+
+ # Check for out of stock
+ elif not product.in_stock and was_in_stock:
+ # Just went out of stock
+ self.db.record_stock_event(db_product_id, "out_of_stock")
+ logger.debug(f"Out of stock: {product.name}")
+
+ existing["in_stock"] = product.in_stock
+ self.products[url] = existing
+
+ self.save()
+ return new_products, restocked_products
+
+ def process_products_with_priority(self, products: List[Product]) -> Tuple[List[Tuple[Product, dict]], List[Tuple[Product, dict]]]:
+ """
+ Process products and return with priority/favorite info.
+
+ Returns:
+ Tuple of (new_products_with_priority, restocked_products_with_priority)
+ Each item is a tuple of (Product, notification_settings)
+ """
+ new_products, restocked_products = self.process_products(products)
+
+ def add_priority(product: Product) -> Tuple[Product, dict]:
+ # Get product from DB for category
+ db_product = self.db.get_product_by_url(product.url)
+ category = db_product.get('category') if db_product else None
+
+ settings = self.favorites.get_notification_settings(
+ url=product.url,
+ name=product.name,
+ category=category
+ )
+ return (product, settings)
+
+ new_with_priority = [add_priority(p) for p in new_products]
+ restocked_with_priority = [add_priority(p) for p in restocked_products]
+
+ return new_with_priority, restocked_with_priority
+
+ def log_check(self, site: str, products_found: int, new_count: int,
+ restock_count: int, duration_ms: int, success: bool = True,
+ error_message: str = None):
+ """Log a monitoring check to database"""
+ self.db.log_check(
+ site=site,
+ products_found=products_found,
+ new_products=new_count,
+ restocks=restock_count,
+ duration_ms=duration_ms,
+ success=success,
+ error_message=error_message
+ )
+
+ def get_known_urls(self) -> Set[str]:
+ """Get all known product URLs"""
+ return set(self.products.keys())
+
+ def get_product(self, url: str) -> Optional[dict]:
+ """Get a specific product by URL"""
+ return self.products.get(url)
+
+ def mark_out_of_stock(self, url: str):
+ """Mark a product as out of stock"""
+ if url in self.products:
+ self.products[url]["in_stock"] = False
+
+ # Also update database
+ db_product = self.db.get_product_by_url(url)
+ if db_product:
+ self.db.record_stock_event(db_product['id'], "out_of_stock")
+
+ self.save()
+
+ def clear(self):
+ """Clear all tracked products (JSON only, database preserved for history)"""
+ self.products = {}
+ self.save()
+ logger.info("Cleared all tracked products from JSON")
+
+ def get_stats(self) -> dict:
+ """Get tracking statistics from both JSON and database"""
+ # JSON stats
+ total = len(self.products)
+ in_stock = sum(1 for p in self.products.values() if p.get("in_stock", False))
+ out_of_stock = total - in_stock
+
+ # Database stats
+ db_stats = self.db.get_dashboard_stats()
+
+ return {
+ "total_products": total,
+ "in_stock": in_stock,
+ "out_of_stock": out_of_stock,
+ # From database
+ "new_drops_today": db_stats.get("new_drops_today", 0),
+ "new_drops_week": db_stats.get("new_drops_week", 0),
+ "restocks_today": db_stats.get("restocks_today", 0),
+ "restocks_week": db_stats.get("restocks_week", 0),
+ "last_check": db_stats.get("last_check"),
+ "favorites_count": db_stats.get("favorites_count", 0),
+ }
+
+ def migrate_to_database(self):
+ """Migrate existing products.json to database"""
+ if self.products_file.exists():
+ self.db.migrate_from_json(str(self.products_file))
+ logger.info("Migration to database complete")
diff --git a/src/scraper_state.py b/src/scraper_state.py
new file mode 100644
index 0000000..6fb9c18
--- /dev/null
+++ b/src/scraper_state.py
@@ -0,0 +1,264 @@
+"""
+Scraper State Manager
+Manages runtime state for scrapers - enabling/disabling, running status, etc.
+"""
+
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json"
+
+# Default state
+DEFAULT_STATE = {
+ "scrapers": {
+ "pokemoncenter": {"enabled": False, "running": False, "last_run": None, "last_error": None},
+ "target": {"enabled": True, "running": False, "last_run": None, "last_error": None},
+ "gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None},
+ "walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None},
+ "bestbuy": {"enabled": False, "running": False, "last_run": None, "last_error": None},
+ },
+ "monitor_running": False,
+ "monitor_pid": None,
+ "check_interval": 60,
+ "last_check": None,
+}
+
+
+class ScraperStateManager:
+ """Manages scraper state with file persistence"""
+
+ _instance = None
+ _lock = threading.Lock()
+ _monitor_process = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance._initialized = False
+ return cls._instance
+
+ def __init__(self):
+ if self._initialized:
+ return
+ self._initialized = True
+ self.state = self._load_state()
+
+ def _load_state(self) -> dict:
+ """Load state from file or return defaults"""
+ try:
+ if STATE_FILE.exists():
+ with open(STATE_FILE, 'r') as f:
+ saved = json.load(f)
+ # Merge with defaults in case new fields added
+ state = DEFAULT_STATE.copy()
+ state.update(saved)
+ # Ensure all scrapers exist
+ for scraper in DEFAULT_STATE["scrapers"]:
+ if scraper not in state["scrapers"]:
+ state["scrapers"][scraper] = DEFAULT_STATE["scrapers"][scraper]
+ return state
+ except Exception:
+ pass
+ return DEFAULT_STATE.copy()
+
+ def _save_state(self):
+ """Save state to file"""
+ with self._lock:
+ try:
+ with open(STATE_FILE, 'w') as f:
+ json.dump(self.state, f, indent=2, default=str)
+ except Exception as e:
+ print(f"Error saving scraper state: {e}")
+
+ def get_state(self) -> dict:
+ """Get current state"""
+ # Check if monitor is still running
+ if self.state["monitor_pid"]:
+ try:
+ # Check if process exists
+ import psutil
+ if not psutil.pid_exists(self.state["monitor_pid"]):
+ self.state["monitor_running"] = False
+ self.state["monitor_pid"] = None
+ for scraper in self.state["scrapers"]:
+ self.state["scrapers"][scraper]["running"] = False
+ self._save_state()
+ except ImportError:
+ pass # psutil not installed, skip check
+
+ return self.state.copy()
+
+ def set_scraper_enabled(self, scraper: str, enabled: bool) -> bool:
+ """Enable or disable a scraper"""
+ if scraper not in self.state["scrapers"]:
+ return False
+
+ self.state["scrapers"][scraper]["enabled"] = enabled
+ self._save_state()
+ return True
+
+ def get_enabled_scrapers(self) -> list:
+ """Get list of enabled scraper names"""
+ return [
+ name for name, data in self.state["scrapers"].items()
+ if data.get("enabled", False)
+ ]
+
+ def set_check_interval(self, seconds: int) -> bool:
+ """Set check interval"""
+ if seconds < 10:
+ return False
+ self.state["check_interval"] = seconds
+ self._save_state()
+ return True
+
+ def record_check(self, scraper: str, success: bool, error: str = None):
+ """Record a scraper check"""
+ if scraper in self.state["scrapers"]:
+ self.state["scrapers"][scraper]["last_run"] = datetime.now().isoformat()
+ if not success:
+ self.state["scrapers"][scraper]["last_error"] = error
+ else:
+ self.state["scrapers"][scraper]["last_error"] = None
+ self.state["last_check"] = datetime.now().isoformat()
+ self._save_state()
+
+ def start_monitor(self) -> dict:
+ """Start the monitor process"""
+ if self.state["monitor_running"]:
+ return {"success": False, "error": "Monitor already running"}
+
+ enabled = self.get_enabled_scrapers()
+ if not enabled:
+ return {"success": False, "error": "No scrapers enabled"}
+
+ try:
+ # Start main.py as a subprocess
+ project_dir = Path(__file__).parent.parent
+ main_script = project_dir / "main.py"
+
+ # Create a modified config for enabled scrapers
+ self._update_config_file(enabled)
+
+ # Start the process
+ if sys.platform == 'win32':
+ # Windows: use CREATE_NEW_PROCESS_GROUP for proper handling
+ process = subprocess.Popen(
+ [sys.executable, str(main_script)],
+ cwd=str(project_dir),
+ creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ else:
+ process = subprocess.Popen(
+ [sys.executable, str(main_script)],
+ cwd=str(project_dir),
+ start_new_session=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+ self._monitor_process = process
+ self.state["monitor_running"] = True
+ self.state["monitor_pid"] = process.pid
+
+ for scraper in enabled:
+ self.state["scrapers"][scraper]["running"] = True
+
+ self._save_state()
+
+ return {"success": True, "pid": process.pid, "scrapers": enabled}
+
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ def stop_monitor(self) -> dict:
+ """Stop the monitor process"""
+ if not self.state["monitor_running"] and not self.state["monitor_pid"]:
+ return {"success": False, "error": "Monitor not running"}
+
+ try:
+ pid = self.state["monitor_pid"]
+
+ if pid:
+ # On Windows, use taskkill for reliable process termination
+ if sys.platform == 'win32':
+ try:
+ # Kill the process tree forcefully
+ subprocess.run(
+ ['taskkill', '/F', '/T', '/PID', str(pid)],
+ capture_output=True,
+ timeout=10
+ )
+ except Exception:
+ pass
+ else:
+ # On Unix, use psutil
+ try:
+ import psutil
+ proc = psutil.Process(pid)
+ # Kill child processes first
+ children = proc.children(recursive=True)
+ for child in children:
+ try:
+ child.kill() # Use kill instead of terminate
+ except:
+ pass
+ # Kill main process
+ proc.kill()
+ proc.wait(timeout=5)
+ except:
+ pass
+
+ self.state["monitor_running"] = False
+ self.state["monitor_pid"] = None
+ for scraper in self.state["scrapers"]:
+ self.state["scrapers"][scraper]["running"] = False
+ self._save_state()
+
+ return {"success": True}
+
+ except Exception as e:
+ # Still update state even if kill failed
+ self.state["monitor_running"] = False
+ self.state["monitor_pid"] = None
+ for scraper in self.state["scrapers"]:
+ self.state["scrapers"][scraper]["running"] = False
+ self._save_state()
+ return {"success": True, "warning": f"Process may need manual cleanup: {str(e)}"}
+
+ def _update_config_file(self, enabled_scrapers: list):
+ """Update config.py with enabled scrapers"""
+ config_path = Path(__file__).parent.parent / "config.py"
+
+ try:
+ with open(config_path, 'r') as f:
+ content = f.read()
+
+ # Find and replace SITES_ENABLED line
+ import re
+ new_line = f'SITES_ENABLED = {json.dumps(enabled_scrapers)}'
+ pattern = r'SITES_ENABLED\s*=\s*\[.*?\]'
+ content = re.sub(pattern, new_line, content)
+
+ # Also update CHECK_INTERVAL_SECONDS
+ interval_line = f'CHECK_INTERVAL_SECONDS = {self.state["check_interval"]}'
+ interval_pattern = r'CHECK_INTERVAL_SECONDS\s*=\s*\d+'
+ content = re.sub(interval_pattern, interval_line, content)
+
+ with open(config_path, 'w') as f:
+ f.write(content)
+
+ except Exception as e:
+ print(f"Error updating config: {e}")
+
+
+# Singleton instance
+scraper_state = ScraperStateManager()
diff --git a/src/store_locator.py b/src/store_locator.py
new file mode 100644
index 0000000..d493353
--- /dev/null
+++ b/src/store_locator.py
@@ -0,0 +1,382 @@
+"""
+Store Locator Module
+
+Finds Pokemon TCG products at local retail stores.
+Supports Target, Best Buy, GameStop, Walmart.
+"""
+
+import json
+import requests
+import logging
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# User location config
+CONFIG_FILE = Path(__file__).parent / "location_config.json"
+
+
+@dataclass
+class Store:
+ name: str
+ retailer: str
+ address: str
+ city: str
+ state: str
+ zip_code: str
+ distance_miles: float
+ phone: Optional[str] = None
+ store_id: Optional[str] = None
+
+
+@dataclass
+class StoreStock:
+ store: Store
+ product_name: str
+ product_url: str
+ in_stock: bool
+ quantity: Optional[int] = None
+ price: Optional[str] = None
+
+
+class StoreLocator:
+ """Base class for store locators"""
+
+ def __init__(self, zip_code: str, radius_miles: int = 25):
+ self.zip_code = zip_code
+ self.radius_miles = radius_miles
+ self.session = requests.Session()
+ self.session.headers.update({
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+ })
+
+ def find_stores(self) -> List[Store]:
+ """Find nearby stores - override in subclasses"""
+ raise NotImplementedError
+
+ def check_stock(self, product_id: str) -> List[StoreStock]:
+ """Check stock at nearby stores - override in subclasses"""
+ raise NotImplementedError
+
+
+class TargetLocator(StoreLocator):
+ """Target store locator and stock checker"""
+
+ BASE_URL = "https://redsky.target.com"
+
+ def find_stores(self) -> List[Store]:
+ """Find nearby Target stores"""
+ url = f"{self.BASE_URL}/v3/stores/nearby/{self.zip_code}"
+ params = {
+ "limit": 20,
+ "within": self.radius_miles,
+ "key": "9f36aeafbe60771e321a7cc95a78140772ab3e96" # Public API key
+ }
+
+ try:
+ response = self.session.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ stores = []
+ for loc in data.get("locations", []):
+ addr = loc.get("address", {})
+ stores.append(Store(
+ name=loc.get("location_name", "Target"),
+ retailer="Target",
+ address=addr.get("address_line1", ""),
+ city=addr.get("city", ""),
+ state=addr.get("state", ""),
+ zip_code=addr.get("postal_code", ""),
+ distance_miles=loc.get("distance", 0),
+ phone=loc.get("telephone", ""),
+ store_id=str(loc.get("location_id", ""))
+ ))
+ return stores
+ except Exception as e:
+ logger.error(f"Error finding Target stores: {e}")
+
+ return []
+
+ def check_stock(self, tcin: str, store_ids: List[str] = None) -> List[StoreStock]:
+ """
+ Check Target stock for a product (TCIN).
+
+ Popular Pokemon TCINs:
+ - 89776270: Pokemon TCG general
+ - Search for specific products on Target.com and get TCIN from URL
+ """
+ if not store_ids:
+ stores = self.find_stores()
+ store_ids = [s.store_id for s in stores[:5]] # Check closest 5
+
+ results = []
+
+ for store_id in store_ids:
+ url = f"{self.BASE_URL}/v3/fulfillment/stores/{store_id}/products/{tcin}"
+ params = {
+ "key": "9f36aeafbe60771e321a7cc95a78140772ab3e96"
+ }
+
+ try:
+ response = self.session.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ product = data.get("product", {})
+ avail = product.get("fulfillment", {}).get("store_options", [])
+
+ for opt in avail:
+ if opt.get("location_id") == store_id:
+ in_stock = opt.get("in_store_only", {}).get("availability_status") == "IN_STOCK"
+ results.append(StoreStock(
+ store=Store(
+ name="Target",
+ retailer="Target",
+ address="",
+ city="",
+ state="",
+ zip_code="",
+ distance_miles=0,
+ store_id=store_id
+ ),
+ product_name=product.get("item", {}).get("product_description", {}).get("title", "Unknown"),
+ product_url=f"https://www.target.com/p/-/A-{tcin}",
+ in_stock=in_stock,
+ price=product.get("price", {}).get("current_retail_min")
+ ))
+ except Exception as e:
+ logger.error(f"Error checking Target stock: {e}")
+
+ return results
+
+
+class BestBuyLocator(StoreLocator):
+ """Best Buy store locator and stock checker"""
+
+ def find_stores(self) -> List[Store]:
+ """Find nearby Best Buy stores"""
+ url = "https://www.bestbuy.com/site/store-locator/v1/stores"
+ params = {
+ "postalCode": self.zip_code,
+ "radius": self.radius_miles
+ }
+
+ try:
+ response = self.session.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ stores = []
+ for store in data.get("stores", []):
+ stores.append(Store(
+ name=store.get("name", "Best Buy"),
+ retailer="Best Buy",
+ address=store.get("address", ""),
+ city=store.get("city", ""),
+ state=store.get("state", ""),
+ zip_code=store.get("postalCode", ""),
+ distance_miles=store.get("distance", 0),
+ phone=store.get("phone", ""),
+ store_id=str(store.get("storeId", ""))
+ ))
+ return stores
+ except Exception as e:
+ logger.error(f"Error finding Best Buy stores: {e}")
+
+ return []
+
+ def check_stock(self, sku: str) -> List[StoreStock]:
+ """
+ Check Best Buy stock for a product (SKU).
+
+ Example: Search for Pokemon on bestbuy.com, get SKU from product page
+ """
+ stores = self.find_stores()[:5]
+ store_ids = [s.store_id for s in stores]
+
+ if not store_ids:
+ return []
+
+ url = f"https://www.bestbuy.com/fulfillment/v1/stores/{','.join(store_ids)}/sku/{sku}"
+
+ try:
+ response = self.session.get(url, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ results = []
+ for store_data in data.get("stores", []):
+ store = next((s for s in stores if s.store_id == str(store_data.get("storeId"))), None)
+ if store:
+ results.append(StoreStock(
+ store=store,
+ product_name=data.get("productName", "Unknown"),
+ product_url=f"https://www.bestbuy.com/site/-/{sku}.p",
+ in_stock=store_data.get("storeAvailability", {}).get("inStoreAvailable", False),
+ quantity=store_data.get("storeAvailability", {}).get("quantity")
+ ))
+ return results
+ except Exception as e:
+ logger.error(f"Error checking Best Buy stock: {e}")
+
+ return []
+
+
+class GameStopLocator(StoreLocator):
+ """GameStop store locator"""
+
+ def find_stores(self) -> List[Store]:
+ """Find nearby GameStop stores"""
+ url = "https://www.gamestop.com/api/location/v1/stores/search"
+ params = {
+ "postalCode": self.zip_code,
+ "radius": self.radius_miles,
+ "count": 20
+ }
+
+ try:
+ response = self.session.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ stores = []
+ for store in data.get("stores", []):
+ stores.append(Store(
+ name=store.get("storeName", "GameStop"),
+ retailer="GameStop",
+ address=store.get("address1", ""),
+ city=store.get("city", ""),
+ state=store.get("state", ""),
+ zip_code=store.get("postalCode", ""),
+ distance_miles=store.get("distance", 0),
+ phone=store.get("phone", ""),
+ store_id=str(store.get("storeNumber", ""))
+ ))
+ return stores
+ except Exception as e:
+ logger.error(f"Error finding GameStop stores: {e}")
+
+ return []
+
+
+class WalmartLocator(StoreLocator):
+ """Walmart store locator"""
+
+ def find_stores(self) -> List[Store]:
+ """Find nearby Walmart stores"""
+ url = "https://www.walmart.com/store/finder/electrode/api/stores"
+ params = {
+ "singleLineAddr": self.zip_code,
+ "distance": self.radius_miles
+ }
+
+ try:
+ response = self.session.get(url, params=params, timeout=10)
+ if response.status_code == 200:
+ data = response.json()
+ stores = []
+ for store in data.get("payload", {}).get("storesData", {}).get("stores", []):
+ addr = store.get("address", {})
+ stores.append(Store(
+ name=store.get("displayName", "Walmart"),
+ retailer="Walmart",
+ address=addr.get("streetAddress", ""),
+ city=addr.get("city", ""),
+ state=addr.get("state", ""),
+ zip_code=addr.get("postalCode", ""),
+ distance_miles=store.get("distance", 0),
+ phone=store.get("phone", ""),
+ store_id=str(store.get("id", ""))
+ ))
+ return stores
+ except Exception as e:
+ logger.error(f"Error finding Walmart stores: {e}")
+
+ return []
+
+
+# Pokemon TCG product IDs for various retailers
+POKEMON_PRODUCTS = {
+ "target": {
+ "pokemon_tcg_general": "89776270",
+ "etb_search": "pokemon+elite+trainer+box",
+ },
+ "bestbuy": {
+ "pokemon_cards": "6542167",
+ },
+ "gamestop": {
+ "pokemon_tcg": "pokemon-trading-cards",
+ },
+ "walmart": {
+ "pokemon_tcg": "pokemon-trading-card-game",
+ }
+}
+
+
+def load_location_config() -> Dict:
+ """Load user location configuration"""
+ if CONFIG_FILE.exists():
+ with open(CONFIG_FILE, 'r') as f:
+ return json.load(f)
+ return {"zip_code": None, "radius_miles": 25}
+
+
+def save_location_config(zip_code: str, radius_miles: int = 25):
+ """Save user location configuration"""
+ config = {
+ "zip_code": zip_code,
+ "radius_miles": radius_miles
+ }
+ with open(CONFIG_FILE, 'w') as f:
+ json.dump(config, f, indent=2)
+ logger.info(f"Location saved: {zip_code}, radius: {radius_miles} miles")
+
+
+def find_all_nearby_stores(zip_code: str, radius_miles: int = 25) -> Dict[str, List[Store]]:
+ """Find all nearby stores across all supported retailers"""
+ results = {}
+
+ locators = [
+ ("Target", TargetLocator(zip_code, radius_miles)),
+ ("Best Buy", BestBuyLocator(zip_code, radius_miles)),
+ ("GameStop", GameStopLocator(zip_code, radius_miles)),
+ ("Walmart", WalmartLocator(zip_code, radius_miles)),
+ ]
+
+ for name, locator in locators:
+ try:
+ stores = locator.find_stores()
+ results[name] = stores
+ logger.info(f"Found {len(stores)} {name} stores near {zip_code}")
+ except Exception as e:
+ logger.error(f"Error with {name}: {e}")
+ results[name] = []
+
+ return results
+
+
+def format_stores_for_discord(stores_by_retailer: Dict[str, List[Store]]) -> str:
+ """Format store list for Discord message"""
+ lines = ["**Nearby Stores:**\n"]
+
+ for retailer, stores in stores_by_retailer.items():
+ if stores:
+ lines.append(f"\n**{retailer}** ({len(stores)} stores)")
+ for store in stores[:3]: # Show top 3
+ lines.append(f" - {store.name} ({store.distance_miles:.1f} mi)")
+ lines.append(f" {store.address}, {store.city}, {store.state}")
+
+ return "\n".join(lines)
+
+
+if __name__ == "__main__":
+ # Test with a sample zip code
+ test_zip = "90210"
+ print(f"Finding stores near {test_zip}...")
+
+ stores = find_all_nearby_stores(test_zip, 25)
+
+ for retailer, store_list in stores.items():
+ print(f"\n{retailer}: {len(store_list)} stores found")
+ for store in store_list[:3]:
+ print(f" - {store.name} ({store.distance_miles:.1f} mi)")
+ print(f" {store.address}, {store.city}, {store.state}")
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..374728e
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+# Test modules
diff --git a/tests/test_api.py b/tests/test_api.py
new file mode 100644
index 0000000..c688072
--- /dev/null
+++ b/tests/test_api.py
@@ -0,0 +1,68 @@
+"""
+Test Pokemon Center API endpoints directly
+"""
+
+import requests
+import json
+
+# Headers extracted from HAR capture
+HEADERS = {
+ "Accept": "application/json",
+ "Accept-Version": "1",
+ "Content-Type": "application/json",
+ "X-Store-Locale": "en-us",
+ "X-Store-Scope": "pokemon",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+ "Referer": "https://www.pokemoncenter.com/category/tcg-cards",
+}
+
+# Test endpoints
+ENDPOINTS = [
+ # Category listing
+ ("GET", "https://www.pokemoncenter.com/site/resourceapi/category/new-releases"),
+
+ # Product details (example SKU)
+ ("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/699-17113"),
+
+ # Product status
+ ("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/status/qgqvbkjwhe4s2mjxgeytg="),
+]
+
+print("=" * 70)
+print("Pokemon Center API Test")
+print("=" * 70)
+print()
+
+for method, url in ENDPOINTS:
+ print(f"[{method}] {url[:80]}...")
+
+ try:
+ if method == "GET":
+ response = requests.get(url, headers=HEADERS, timeout=10)
+ else:
+ response = requests.post(url, headers=HEADERS, timeout=10)
+
+ print(f" Status: {response.status_code}")
+ print(f" Content-Type: {response.headers.get('Content-Type', 'N/A')}")
+
+ if response.status_code == 200:
+ try:
+ data = response.json()
+ print(f" Response keys: {list(data.keys())[:5]}...")
+
+ # Show a preview
+ preview = json.dumps(data, indent=2)[:500]
+ print(f" Preview:\n{preview}")
+ except:
+ print(f" Raw: {response.text[:200]}")
+ else:
+ print(f" Response: {response.text[:200]}")
+
+ except Exception as e:
+ print(f" Error: {e}")
+
+ print()
+
+print("=" * 70)
+print("If APIs return 200, we can monitor without a browser!")
+print("=" * 70)
diff --git a/tests/test_dashboard_api.py b/tests/test_dashboard_api.py
new file mode 100644
index 0000000..3272a72
--- /dev/null
+++ b/tests/test_dashboard_api.py
@@ -0,0 +1,407 @@
+"""
+Unit tests for Dashboard REST API endpoints.
+"""
+
+import pytest
+import tempfile
+import os
+import sys
+import json
+
+# Add project root to path
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, project_root)
+
+from flask import Flask
+from src.database import Database
+
+# Import api module directly to avoid circular imports via dashboard/__init__.py
+import importlib.util
+spec = importlib.util.spec_from_file_location(
+ "dashboard_api",
+ os.path.join(project_root, "dashboard", "api.py")
+)
+dashboard_api = importlib.util.module_from_spec(spec)
+sys.modules["dashboard_api"] = dashboard_api
+spec.loader.exec_module(dashboard_api)
+api_bp = dashboard_api.api_bp
+
+
+@pytest.fixture
+def app():
+ """Create test Flask app"""
+ app = Flask(__name__)
+ app.config['TESTING'] = True
+ app.register_blueprint(api_bp, url_prefix='/api')
+
+ # Use temp database
+ with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
+ db_path = f.name
+
+ # Monkey-patch the database
+ import src.database as db_module
+ db_module._db = Database(db_path)
+
+ # Also reset the favorites manager so it picks up the new database
+ import src.favorites as fav_module
+ fav_module._favorites = None # Reset so it gets recreated with new db
+
+ yield app
+
+ # Cleanup
+ os.unlink(db_path)
+
+
+@pytest.fixture
+def client(app):
+ """Create test client"""
+ return app.test_client()
+
+
+class TestStatsEndpoints:
+ """Tests for stats and health endpoints"""
+
+ def test_get_stats(self, client):
+ """Test /api/stats endpoint"""
+ response = client.get('/api/stats')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'total_products' in data
+ assert 'in_stock_count' in data
+ assert 'new_drops_today' in data
+
+ def test_get_health(self, client):
+ """Test /api/health endpoint"""
+ response = client.get('/api/health')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert data['status'] == 'ok'
+
+
+class TestProductsEndpoints:
+ """Tests for product endpoints"""
+
+ def test_get_products_empty(self, client):
+ """Test getting products when empty"""
+ response = client.get('/api/products')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'products' in data
+ assert 'total' in data
+
+ def test_get_products_with_filters(self, client):
+ """Test product filtering"""
+ response = client.get('/api/products?site=pokemoncenter&in_stock=true')
+
+ assert response.status_code == 200
+
+ def test_get_product_not_found(self, client):
+ """Test getting non-existent product"""
+ response = client.get('/api/products/99999')
+
+ assert response.status_code == 404
+
+
+class TestEventsEndpoints:
+ """Tests for events endpoints"""
+
+ def test_get_events(self, client):
+ """Test /api/events endpoint"""
+ response = client.get('/api/events')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'events' in data
+
+ def test_get_events_with_type_filter(self, client):
+ """Test filtering events by type"""
+ response = client.get('/api/events?type=restock&type=new_drop')
+
+ assert response.status_code == 200
+
+
+class TestAnalyticsEndpoints:
+ """Tests for analytics endpoints"""
+
+ def test_get_drop_timing(self, client):
+ """Test /api/analytics/drops endpoint"""
+ response = client.get('/api/analytics/drops')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'drop_timing' in data
+
+ def test_get_stock_duration(self, client):
+ """Test /api/analytics/stock endpoint"""
+ response = client.get('/api/analytics/stock')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'stock_duration' in data
+
+ def test_get_selling_rates(self, client):
+ """Test /api/analytics/selling-rates endpoint"""
+ response = client.get('/api/analytics/selling-rates')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'products' in data
+
+ def test_get_site_stats(self, client):
+ """Test /api/analytics/sites endpoint"""
+ response = client.get('/api/analytics/sites')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'sites' in data
+
+
+class TestExtensionSyncEndpoint:
+ """Tests for Chrome extension sync endpoint"""
+
+ def test_sync_empty_data(self, client):
+ """Test syncing with no data returns error"""
+ response = client.post('/api/extension/sync',
+ content_type='application/json')
+
+ assert response.status_code == 400
+
+ def test_sync_skus(self, client):
+ """Test syncing SKUs from extension"""
+ data = {
+ 'skus': ['699-17113', '191-85953', '100-12345'],
+ 'products': [],
+ 'events': []
+ }
+
+ response = client.post('/api/extension/sync',
+ data=json.dumps(data),
+ content_type='application/json')
+
+ assert response.status_code == 200
+ result = json.loads(response.data)
+
+ assert result['success'] is True
+ assert result['total_skus'] == 3
+ assert result['new_skus_added'] == 3
+
+ def test_sync_products(self, client):
+ """Test syncing products from extension"""
+ data = {
+ 'skus': [],
+ 'products': [
+ {
+ 'url': 'https://www.pokemoncenter.com/product/699-17113',
+ 'name': 'Pokemon ETB Prismatic Evolutions',
+ 'price': '$49.99',
+ 'inStock': True,
+ 'imageUrl': 'https://example.com/image.jpg',
+ 'productId': '699-17113'
+ }
+ ],
+ 'events': []
+ }
+
+ response = client.post('/api/extension/sync',
+ data=json.dumps(data),
+ content_type='application/json')
+
+ assert response.status_code == 200
+ result = json.loads(response.data)
+
+ assert result['success'] is True
+ assert result['total_products'] >= 1
+
+ def test_sync_events(self, client):
+ """Test syncing events (restocks, drops) from extension"""
+ # First sync a product
+ product_data = {
+ 'skus': [],
+ 'products': [
+ {
+ 'url': 'https://www.pokemoncenter.com/product/test-123',
+ 'name': 'Test Product',
+ 'price': '$29.99',
+ 'inStock': True
+ }
+ ],
+ 'events': []
+ }
+ client.post('/api/extension/sync',
+ data=json.dumps(product_data),
+ content_type='application/json')
+
+ # Now sync events
+ event_data = {
+ 'skus': [],
+ 'products': [],
+ 'events': [
+ {
+ 'type': 'restock',
+ 'url': 'https://www.pokemoncenter.com/product/test-123',
+ 'name': 'Test Product',
+ 'price': '$29.99',
+ 'timestamp': '2024-01-15T10:00:00Z'
+ },
+ {
+ 'type': 'out_of_stock',
+ 'url': 'https://www.pokemoncenter.com/product/test-123',
+ 'name': 'Test Product',
+ 'timestamp': '2024-01-15T10:30:00Z'
+ }
+ ]
+ }
+
+ response = client.post('/api/extension/sync',
+ data=json.dumps(event_data),
+ content_type='application/json')
+
+ assert response.status_code == 200
+ result = json.loads(response.data)
+
+ assert result['success'] is True
+ assert result['events_processed'] == 2
+
+ def test_sync_price_change_event(self, client):
+ """Test syncing price change events"""
+ # First create the product
+ client.post('/api/extension/sync',
+ data=json.dumps({
+ 'products': [{
+ 'url': 'https://www.pokemoncenter.com/product/price-test',
+ 'name': 'Price Test Product',
+ 'price': '$39.99',
+ 'inStock': True
+ }]
+ }),
+ content_type='application/json')
+
+ # Now send price change event
+ event_data = {
+ 'events': [
+ {
+ 'type': 'price_change',
+ 'url': 'https://www.pokemoncenter.com/product/price-test',
+ 'oldPrice': '$39.99',
+ 'newPrice': '$34.99',
+ 'timestamp': '2024-01-15T12:00:00Z'
+ }
+ ]
+ }
+
+ response = client.post('/api/extension/sync',
+ data=json.dumps(event_data),
+ content_type='application/json')
+
+ assert response.status_code == 200
+ result = json.loads(response.data)
+ assert result['events_processed'] == 1
+
+
+class TestExtensionDataEndpoints:
+ """Tests for extension data retrieval endpoints"""
+
+ def test_get_extension_skus(self, client):
+ """Test getting extension SKUs"""
+ response = client.get('/api/extension/skus')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'skus' in data
+ assert 'total' in data
+
+ def test_get_extension_products(self, client):
+ """Test getting extension products"""
+ response = client.get('/api/extension/products')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'products' in data
+ assert 'total' in data
+
+ def test_get_extension_stats(self, client):
+ """Test getting extension stats"""
+ response = client.get('/api/extension/stats')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'total_skus' in data
+ assert 'total_products' in data
+ assert 'source' in data
+ assert data['source'] == 'chrome_extension'
+
+ def test_clear_extension_data(self, client):
+ """Test clearing extension data"""
+ response = client.post('/api/extension/clear')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert data['success'] is True
+
+
+class TestFavoritesEndpoints:
+ """Tests for favorites CRUD endpoints"""
+
+ def test_add_favorite(self, client):
+ """Test adding a favorite"""
+ data = {
+ 'type': 'product',
+ 'value': 'https://example.com/fav-product',
+ 'display_name': 'My Favorite ETB',
+ 'priority': 'high'
+ }
+
+ response = client.post('/api/favorites',
+ data=json.dumps(data),
+ content_type='application/json')
+
+ assert response.status_code == 201
+ result = json.loads(response.data)
+
+ assert result['success'] is True
+ assert 'id' in result
+
+ def test_get_favorites(self, client):
+ """Test getting all favorites"""
+ response = client.get('/api/favorites')
+
+ assert response.status_code == 200
+ data = json.loads(response.data)
+
+ assert 'favorites' in data
+
+ def test_delete_favorite(self, client):
+ """Test deleting a favorite"""
+ # First add one
+ add_response = client.post('/api/favorites',
+ data=json.dumps({
+ 'type': 'category',
+ 'value': 'ETB'
+ }),
+ content_type='application/json')
+
+ fav_id = json.loads(add_response.data)['id']
+
+ # Now delete it
+ response = client.delete(f'/api/favorites/{fav_id}')
+
+ assert response.status_code == 200
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_database.py b/tests/test_database.py
new file mode 100644
index 0000000..346ec50
--- /dev/null
+++ b/tests/test_database.py
@@ -0,0 +1,300 @@
+"""
+Unit tests for database operations.
+"""
+
+import pytest
+import tempfile
+import os
+from datetime import datetime, timedelta
+
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.database import Database
+
+
+@pytest.fixture
+def db():
+ """Create a temporary database for testing"""
+ with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
+ db_path = f.name
+
+ database = Database(db_path)
+ yield database
+
+ # Cleanup
+ os.unlink(db_path)
+
+
+class TestProductOperations:
+ """Tests for product CRUD operations"""
+
+ def test_create_product(self, db):
+ """Test creating a new product"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/product/123",
+ name="Test Pokemon Card",
+ site="pokemoncenter",
+ product_id="123",
+ price="$19.99",
+ in_stock=True
+ )
+
+ assert product_id is not None
+ assert product_id > 0
+
+ def test_get_product_by_url(self, db):
+ """Test retrieving product by URL"""
+ url = "https://example.com/product/456"
+ db.get_or_create_product(
+ url=url,
+ name="Pikachu ETB",
+ site="target",
+ in_stock=True
+ )
+
+ product = db.get_product_by_url(url)
+
+ assert product is not None
+ assert product['name'] == "Pikachu ETB"
+ assert product['site'] == "target"
+
+ def test_update_product_price(self, db):
+ """Test updating product price"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/product/789",
+ name="Booster Box",
+ site="bestbuy",
+ price="$99.99",
+ in_stock=True
+ )
+
+ # Update price
+ db.update_product_price(product_id, "$89.99")
+
+ product = db.get_product(product_id)
+ assert product['current_price'] == "$89.99"
+
+ # Check price history was recorded
+ history = db.get_price_history(product_id, days=1)
+ assert len(history) >= 1
+
+ def test_update_product_stock(self, db):
+ """Test updating product stock status"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/product/stock-test",
+ name="Limited Edition",
+ site="pokemoncenter",
+ in_stock=True
+ )
+
+ # Mark out of stock
+ db.update_product_stock(product_id, False)
+
+ product = db.get_product(product_id)
+ assert product['in_stock'] == 0 # SQLite stores as 0/1
+
+ def test_category_detection(self, db):
+ """Test automatic category detection from product name"""
+ test_cases = [
+ ("Pokemon Scarlet Elite Trainer Box", "ETB"),
+ ("Charizard Booster Bundle", "Booster Bundle"),
+ ("Display Booster Box 36 Packs", "Booster Box"),
+ ("Sleeved Booster Pack", "Booster Pack"),
+ ("Premium Collection Box", "Collection Box"),
+ ("Pokemon Trading Card Tin", "Tin"),
+ ("Card Binder Album", "Accessories"),
+ ("Random Pokemon Item", "Other"),
+ ]
+
+ for name, expected_category in test_cases:
+ product_id = db.get_or_create_product(
+ url=f"https://example.com/{name.replace(' ', '-')}",
+ name=name,
+ site="test",
+ in_stock=True
+ )
+ product = db.get_product(product_id)
+ assert product['category'] == expected_category, f"Failed for: {name}"
+
+
+class TestStockEvents:
+ """Tests for stock event tracking"""
+
+ def test_record_stock_event(self, db):
+ """Test recording stock events"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/event-test",
+ name="Event Test Product",
+ site="test",
+ in_stock=True
+ )
+
+ db.record_stock_event(product_id, "new_drop")
+ db.record_stock_event(product_id, "out_of_stock")
+ db.record_stock_event(product_id, "restock")
+
+ events = db.get_recent_events(limit=10)
+
+ # Should have 3 events
+ product_events = [e for e in events if e['product_id'] == product_id]
+ assert len(product_events) == 3
+
+ def test_events_today_count(self, db):
+ """Test counting events from today"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/today-test",
+ name="Today Test",
+ site="test",
+ in_stock=True
+ )
+
+ db.record_stock_event(product_id, "new_drop")
+ db.record_stock_event(product_id, "restock")
+
+ new_drops = db.get_events_today("new_drop")
+ restocks = db.get_events_today("restock")
+
+ assert new_drops >= 1
+ assert restocks >= 1
+
+ def test_selling_rate_calculation(self, db):
+ """Test that selling rate can be calculated from events"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/sellrate-test",
+ name="Fast Seller",
+ site="pokemoncenter",
+ in_stock=True
+ )
+
+ # Simulate: drop -> sold out
+ db.record_stock_event(product_id, "new_drop")
+ db.record_stock_event(product_id, "out_of_stock")
+
+ # Get stock duration stats
+ stats = db.get_stock_duration_stats()
+ # Should return at least an empty list (stats depend on timing)
+ assert isinstance(stats, list)
+
+
+class TestPriceHistory:
+ """Tests for price history tracking"""
+
+ def test_record_price_change(self, db):
+ """Test that price changes are recorded"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/price-test",
+ name="Price Test",
+ site="test",
+ price="$49.99",
+ in_stock=True
+ )
+
+ # Record initial price
+ db.record_price(product_id, "$49.99")
+
+ # Change price
+ db.record_price(product_id, "$39.99")
+
+ history = db.get_price_history(product_id, days=1)
+
+ # Should have at least 2 price points
+ assert len(history) >= 1
+
+ def test_duplicate_price_not_recorded(self, db):
+ """Test that same price is not recorded multiple times"""
+ product_id = db.get_or_create_product(
+ url="https://example.com/dup-price-test",
+ name="Dup Price Test",
+ site="test",
+ in_stock=True
+ )
+
+ # Record same price multiple times
+ db.record_price(product_id, "$29.99")
+ db.record_price(product_id, "$29.99")
+ db.record_price(product_id, "$29.99")
+
+ history = db.get_price_history(product_id, days=1)
+
+ # Should only have 1 entry
+ assert len(history) == 1
+
+
+class TestDashboardStats:
+ """Tests for dashboard statistics"""
+
+ def test_get_dashboard_stats(self, db):
+ """Test getting dashboard stats summary"""
+ # Add some test data
+ for i in range(5):
+ product_id = db.get_or_create_product(
+ url=f"https://example.com/stats-{i}",
+ name=f"Stats Product {i}",
+ site="pokemoncenter",
+ in_stock=(i % 2 == 0)
+ )
+ db.record_stock_event(product_id, "new_drop")
+
+ stats = db.get_dashboard_stats()
+
+ assert 'total_products' in stats
+ assert 'in_stock_count' in stats
+ assert 'new_drops_today' in stats
+ assert stats['total_products'] >= 5
+
+ def test_get_site_stats(self, db):
+ """Test per-site statistics"""
+ sites = ['target', 'bestbuy', 'pokemoncenter']
+
+ for site in sites:
+ db.get_or_create_product(
+ url=f"https://{site}.com/test",
+ name=f"{site} Product",
+ site=site,
+ in_stock=True
+ )
+
+ stats = db.get_site_stats()
+
+ assert len(stats) == 3
+ site_names = [s['site'] for s in stats]
+ assert 'target' in site_names
+ assert 'bestbuy' in site_names
+
+
+class TestFavorites:
+ """Tests for favorites functionality"""
+
+ def test_add_favorite(self, db):
+ """Test adding a favorite"""
+ fav_id = db.add_favorite(
+ fav_type="product",
+ value="https://example.com/fav-product",
+ display_name="My Favorite Card",
+ priority="high"
+ )
+
+ assert fav_id is not None
+
+ favorite = db.get_favorite(fav_id)
+ assert favorite['display_name'] == "My Favorite Card"
+
+ def test_check_is_favorite(self, db):
+ """Test checking if product is favorited"""
+ url = "https://example.com/check-fav"
+
+ # Not a favorite yet
+ result = db.check_is_favorite(url=url)
+ assert result is None
+
+ # Add as favorite
+ db.add_favorite(fav_type="product", value=url)
+
+ # Now should be found
+ result = db.check_is_favorite(url=url)
+ assert result is not None
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_product_filter.py b/tests/test_product_filter.py
new file mode 100644
index 0000000..a0155b2
--- /dev/null
+++ b/tests/test_product_filter.py
@@ -0,0 +1,175 @@
+"""
+Unit tests for Pokemon product validation filter.
+"""
+
+import pytest
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from scrapers.base import BaseScraper, Product
+
+
+def make_product(name: str, url: str = "", site: str = "test") -> Product:
+ """Helper to create Product with required fields"""
+ return Product(name=name, url=url, price=None, in_stock=True, site=site)
+
+
+class MockScraper(BaseScraper):
+ """Mock scraper for testing base class methods"""
+
+ site_name = "test"
+
+ def scrape_category_page(self, url: str):
+ return []
+
+ def check_product_stock(self, product):
+ return True, "$19.99"
+
+
+@pytest.fixture
+def scraper():
+ return MockScraper()
+
+
+class TestPokemonProductFilter:
+ """Tests for is_pokemon_product validation"""
+
+ def test_valid_pokemon_products(self, scraper):
+ """Test that valid Pokemon products are accepted"""
+ valid_products = [
+ make_product("Pokemon Scarlet & Violet Elite Trainer Box"),
+ make_product("Pikachu V Collection Box"),
+ make_product("TCG Booster Pack Prismatic Evolutions"),
+ make_product("Charizard Premium Collection"),
+ make_product("Pokemon ETB Paldean Fates"),
+ make_product("Pokémon Trading Card Game Tin"), # Unicode é
+ make_product("Mewtwo VMAX Box Set"),
+ make_product("Eevee Heroes Booster Box"),
+ ]
+
+ for product in valid_products:
+ assert scraper.is_pokemon_product(product), f"Should accept: {product.name}"
+
+ def test_invalid_non_pokemon_products(self, scraper):
+ """Test that non-Pokemon products are rejected"""
+ invalid_products = [
+ make_product("Ice Cube Tray Silicone Mold"),
+ make_product("Barbie Dream House Playset"),
+ make_product("Hot Wheels Track Builder"),
+ make_product("LEGO Star Wars Set"),
+ make_product("Kitchen Appliance Blender"),
+ make_product("Transformers Action Figure"),
+ make_product("Room Essentials Bedding Set"),
+ make_product("Threshold Furniture Table"),
+ make_product("Random Gaming Accessory"),
+ ]
+
+ for product in invalid_products:
+ assert not scraper.is_pokemon_product(product), f"Should reject: {product.name}"
+
+ def test_exclusion_takes_priority(self, scraper):
+ """Test that exclusion terms override Pokemon terms"""
+ # This has "pokemon" but also "ice cube" - should be rejected
+ product = make_product("Pokemon Ice Cube Tray Silicone")
+ assert not scraper.is_pokemon_product(product)
+
+ def test_case_insensitive(self, scraper):
+ """Test that matching is case insensitive"""
+ products = [
+ make_product("POKEMON ELITE TRAINER BOX"),
+ make_product("pokemon scarlet booster"),
+ make_product("PoKeMoN ChArIzArD"),
+ ]
+
+ for product in products:
+ assert scraper.is_pokemon_product(product), f"Should accept: {product.name}"
+
+
+class TestFilterPokemonProducts:
+ """Tests for filter_pokemon_products batch filtering"""
+
+ def test_filter_mixed_products(self, scraper):
+ """Test filtering a mix of valid and invalid products"""
+ products = [
+ make_product("Pokemon Booster Pack", url="1"),
+ make_product("Ice Cube Tray", url="2"),
+ make_product("Charizard Collection", url="3"),
+ make_product("Barbie Doll", url="4"),
+ make_product("Pikachu Tin", url="5"),
+ ]
+
+ filtered = scraper.filter_pokemon_products(products)
+
+ assert len(filtered) == 3
+ urls = [p.url for p in filtered]
+ assert "1" in urls
+ assert "3" in urls
+ assert "5" in urls
+ assert "2" not in urls
+ assert "4" not in urls
+
+ def test_filter_empty_list(self, scraper):
+ """Test filtering empty list returns empty"""
+ filtered = scraper.filter_pokemon_products([])
+ assert filtered == []
+
+ def test_filter_all_valid(self, scraper):
+ """Test filtering when all products are valid"""
+ products = [
+ make_product("Pokemon ETB", url="1"),
+ make_product("Pokemon Booster", url="2"),
+ ]
+
+ filtered = scraper.filter_pokemon_products(products)
+ assert len(filtered) == 2
+
+ def test_filter_all_invalid(self, scraper):
+ """Test filtering when all products are invalid"""
+ products = [
+ make_product("Random Item", url="1"),
+ make_product("Another Thing", url="2"),
+ ]
+
+ filtered = scraper.filter_pokemon_products(products)
+ assert len(filtered) == 0
+
+
+class TestProductDataclass:
+ """Tests for Product dataclass"""
+
+ def test_product_creation(self):
+ """Test creating a Product with all fields"""
+ product = Product(
+ name="Test Product",
+ url="https://example.com/test",
+ price="$29.99",
+ in_stock=True,
+ site="pokemoncenter",
+ image_url="https://example.com/image.jpg",
+ product_id="123"
+ )
+
+ assert product.name == "Test Product"
+ assert product.price == "$29.99"
+ assert product.in_stock is True
+
+ def test_product_minimal(self):
+ """Test Product with only required fields"""
+ product = Product(
+ name="Minimal Product",
+ url="https://example.com",
+ price=None,
+ in_stock=False
+ )
+
+ assert product.name == "Minimal Product"
+ assert product.price is None
+ assert product.in_stock is False
+ assert product.image_url is None
+ assert product.product_id is None
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/tests/test_stealth.py b/tests/test_stealth.py
new file mode 100644
index 0000000..707af6c
--- /dev/null
+++ b/tests/test_stealth.py
@@ -0,0 +1,111 @@
+"""
+Test script for the stealth browser module
+Run this to verify undetected-chromedriver is working
+"""
+
+import sys
+import time
+
+print("=" * 60)
+print("Stealth Browser Test")
+print("=" * 60)
+print()
+
+# Check dependencies
+print("1. Checking dependencies...")
+try:
+ import undetected_chromedriver as uc
+ print(" [OK] undetected-chromedriver installed")
+except ImportError:
+ print(" [ERROR] undetected-chromedriver not installed!")
+ print(" Run: pip install undetected-chromedriver")
+ sys.exit(1)
+
+try:
+ from selenium.webdriver.common.action_chains import ActionChains
+ print(" [OK] selenium installed")
+except ImportError:
+ print(" [ERROR] selenium not installed!")
+ print(" Run: pip install selenium")
+ sys.exit(1)
+
+# Import our stealth browser
+print()
+print("2. Importing stealth browser module...")
+try:
+ from stealth_browser import StealthBrowser, get_stealth_browser
+ print(" [OK] stealth_browser module loaded")
+except Exception as e:
+ print(f" [ERROR] Failed to import: {e}")
+ sys.exit(1)
+
+# Test browser launch
+print()
+print("3. Launching stealth browser...")
+print(" (A Chrome window should open)")
+print()
+
+browser = None
+try:
+ browser = StealthBrowser(
+ headless=False,
+ session_name="test_session"
+ )
+ browser.start()
+ print(" [OK] Browser started successfully!")
+
+ # Test navigation
+ print()
+ print("4. Testing navigation to Pokemon Center...")
+ print(" (Watch the browser window)")
+
+ url = "https://www.pokemoncenter.com/category/tcg-cards"
+ html = browser.get_page(url, wait_time=5)
+
+ print(f" [OK] Page loaded - {len(html)} bytes")
+
+ # Check for CAPTCHA
+ print()
+ print("5. Checking for bot detection...")
+ if browser.check_for_captcha():
+ print(" [!] CAPTCHA/Challenge detected!")
+ print(" The browser window is open - solve it manually if needed")
+ print(" Waiting up to 60 seconds...")
+ browser.wait_for_captcha_solve(timeout=60)
+ else:
+ print(" [OK] No CAPTCHA detected! Stealth mode working.")
+
+ # Show some page info
+ print()
+ print("6. Page analysis...")
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(html, 'html.parser')
+ title = soup.title.string if soup.title else "No title"
+ print(f" Page title: {title}")
+
+ # Count potential product elements
+ products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']")
+ print(f" Potential product elements: {len(products)}")
+
+ print()
+ print("=" * 60)
+ print("TEST COMPLETE")
+ print("=" * 60)
+ print()
+ print("The browser window will stay open for 10 seconds so you can inspect it.")
+ print("Press Ctrl+C to close early.")
+
+ time.sleep(10)
+
+except KeyboardInterrupt:
+ print("\n Interrupted by user")
+except Exception as e:
+ print(f" [ERROR] {e}")
+ import traceback
+ traceback.print_exc()
+finally:
+ if browser:
+ print()
+ print("Closing browser...")
+ browser.stop()
+ print("Done!")
diff --git a/tools/__init__.py b/tools/__init__.py
new file mode 100644
index 0000000..fa05096
--- /dev/null
+++ b/tools/__init__.py
@@ -0,0 +1 @@
+# Utility and experimental scripts
diff --git a/tools/analyze_har.py b/tools/analyze_har.py
new file mode 100644
index 0000000..95ec100
--- /dev/null
+++ b/tools/analyze_har.py
@@ -0,0 +1,336 @@
+"""
+HAR File Analyzer for Pokemon Center API Discovery
+
+Analyzes a HAR (HTTP Archive) file exported from Chrome DevTools
+to find API endpoints used by Pokemon Center.
+
+Usage:
+ 1. Export HAR from Chrome DevTools Network tab
+ 2. Save as pokemon_network.har in this folder
+ 3. Run: python analyze_har.py
+
+Or specify a different file:
+ python analyze_har.py my_capture.har
+"""
+
+import json
+import sys
+import re
+from pathlib import Path
+from urllib.parse import urlparse, parse_qs
+from collections import defaultdict
+
+# Patterns that indicate interesting API endpoints
+INTERESTING_PATTERNS = [
+ r'/api/',
+ r'/graphql',
+ r'/v\d+/',
+ r'product',
+ r'catalog',
+ r'search',
+ r'inventory',
+ r'stock',
+ r'\.json$',
+ r'algolia',
+ r'contentful',
+ r'commercetools',
+ r'demandware',
+ r'sfcc',
+]
+
+# Skip these resource types
+SKIP_TYPES = [
+ 'image', 'stylesheet', 'font', 'script', 'media',
+ '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
+ '.css', '.woff', '.woff2', '.ttf',
+ '.js', # Usually not APIs, but we'll catch fetch calls
+]
+
+def should_skip(url, mime_type=''):
+ """Check if we should skip this request"""
+ url_lower = url.lower()
+ mime_lower = mime_type.lower()
+
+ for skip in SKIP_TYPES:
+ if skip in url_lower or skip in mime_lower:
+ return True
+ return False
+
+def is_interesting(url):
+ """Check if URL matches patterns we care about"""
+ url_lower = url.lower()
+ for pattern in INTERESTING_PATTERNS:
+ if re.search(pattern, url_lower):
+ return True
+ return False
+
+def extract_json_preview(content, max_length=500):
+ """Try to extract and preview JSON content"""
+ if not content:
+ return None
+
+ try:
+ # HAR stores content as text or base64
+ text = content.get('text', '')
+ if not text:
+ return None
+
+ # Try to parse as JSON
+ data = json.loads(text)
+
+ # Return a preview
+ preview = json.dumps(data, indent=2)
+ if len(preview) > max_length:
+ return preview[:max_length] + "\n... (truncated)"
+ return preview
+ except:
+ return None
+
+def analyze_har(har_path):
+ """Analyze a HAR file for API endpoints"""
+
+ print(f"\nLoading {har_path}...")
+
+ with open(har_path, 'r', encoding='utf-8') as f:
+ har_data = json.load(f)
+
+ entries = har_data.get('log', {}).get('entries', [])
+ print(f"Found {len(entries)} network requests")
+
+ # Categorize requests
+ api_calls = []
+ json_responses = []
+ graphql_calls = []
+ third_party_apis = []
+
+ domains = defaultdict(int)
+
+ for entry in entries:
+ request = entry.get('request', {})
+ response = entry.get('response', {})
+
+ url = request.get('url', '')
+ method = request.get('method', 'GET')
+ status = response.get('status', 0)
+ mime_type = response.get('content', {}).get('mimeType', '')
+
+ # Track domains
+ parsed = urlparse(url)
+ domains[parsed.netloc] += 1
+
+ # Skip resources we don't care about
+ if should_skip(url, mime_type):
+ continue
+
+ # Check for JSON responses
+ if 'json' in mime_type.lower():
+ content_preview = extract_json_preview(response.get('content', {}))
+ json_responses.append({
+ 'url': url,
+ 'method': method,
+ 'status': status,
+ 'preview': content_preview,
+ 'headers': {h['name']: h['value'] for h in request.get('headers', [])},
+ 'response_headers': {h['name']: h['value'] for h in response.get('headers', [])}
+ })
+
+ # Check for GraphQL
+ if 'graphql' in url.lower():
+ post_data = request.get('postData', {})
+ graphql_calls.append({
+ 'url': url,
+ 'method': method,
+ 'body': post_data.get('text', ''),
+ 'status': status
+ })
+
+ # Check for interesting patterns
+ if is_interesting(url):
+ api_calls.append({
+ 'url': url,
+ 'method': method,
+ 'status': status,
+ 'mime': mime_type
+ })
+
+ # Check for third-party APIs
+ if any(tp in url.lower() for tp in ['algolia', 'contentful', 'commercetools', 'sfcc']):
+ third_party_apis.append({
+ 'url': url,
+ 'method': method,
+ 'status': status
+ })
+
+ # Print analysis
+ print("\n" + "=" * 70)
+ print("HAR ANALYSIS RESULTS")
+ print("=" * 70)
+
+ # Domains summary
+ print("\n[DOMAINS CONTACTED]")
+ print("-" * 40)
+ for domain, count in sorted(domains.items(), key=lambda x: -x[1])[:15]:
+ print(f" {count:4d} requests {domain}")
+
+ # GraphQL calls
+ if graphql_calls:
+ print("\n[GRAPHQL ENDPOINTS]")
+ print("-" * 40)
+ for call in graphql_calls:
+ print(f"\n [{call['method']}] {call['url']}")
+ print(f" Status: {call['status']}")
+ if call['body']:
+ try:
+ body = json.loads(call['body'])
+ if 'query' in body:
+ query_preview = body['query'][:200].replace('\n', ' ')
+ print(f" Query: {query_preview}...")
+ if 'operationName' in body:
+ print(f" Operation: {body['operationName']}")
+ except:
+ print(f" Body: {call['body'][:200]}...")
+
+ # JSON responses (potential APIs)
+ if json_responses:
+ print("\n[JSON RESPONSES - Potential APIs]")
+ print("-" * 40)
+
+ # Filter to most interesting ones
+ product_related = [r for r in json_responses if 'product' in r['url'].lower()]
+ other_json = [r for r in json_responses if 'product' not in r['url'].lower()][:10]
+
+ for resp in product_related + other_json:
+ print(f"\n [{resp['method']}] {resp['url'][:100]}")
+ print(f" Status: {resp['status']}")
+
+ if resp['preview']:
+ print(f" Preview:")
+ for line in resp['preview'].split('\n')[:10]:
+ print(f" {line}")
+
+ # Third-party APIs
+ if third_party_apis:
+ print("\n[THIRD-PARTY APIs]")
+ print("-" * 40)
+ for api in third_party_apis:
+ print(f" [{api['method']}] {api['url'][:100]}")
+
+ # Interesting endpoints
+ if api_calls:
+ print("\n[OTHER INTERESTING ENDPOINTS]")
+ print("-" * 40)
+ seen = set()
+ for call in api_calls:
+ # Dedupe by base URL
+ parsed = urlparse(call['url'])
+ base = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+ if base in seen:
+ continue
+ seen.add(base)
+
+ print(f" [{call['method']}] {call['url'][:100]}")
+ print(f" Status: {call['status']}, Type: {call['mime']}")
+
+ # Save detailed results
+ output_file = Path(har_path).stem + "_analysis.json"
+ output_path = Path(har_path).parent / output_file
+
+ with open(output_path, 'w') as f:
+ json.dump({
+ 'summary': {
+ 'total_requests': len(entries),
+ 'json_responses': len(json_responses),
+ 'graphql_calls': len(graphql_calls),
+ 'third_party_apis': len(third_party_apis),
+ 'interesting_endpoints': len(api_calls)
+ },
+ 'domains': dict(domains),
+ 'json_responses': json_responses,
+ 'graphql_calls': graphql_calls,
+ 'third_party_apis': third_party_apis,
+ 'api_calls': api_calls
+ }, f, indent=2)
+
+ print(f"\n[SAVED] Detailed results saved to: {output_path}")
+
+ # Recommendations
+ print("\n" + "=" * 70)
+ print("RECOMMENDATIONS")
+ print("=" * 70)
+
+ if graphql_calls:
+ print("""
+[OK] GraphQL endpoint found! This is likely the main data source.
+ Next steps:
+ 1. Examine the query structure in the analysis JSON
+ 2. Test if the endpoint works without cookies/auth
+ 3. Build a monitor that polls this endpoint
+""")
+
+ if any('algolia' in api['url'].lower() for api in third_party_apis):
+ print("""
+[OK] Algolia search detected! This is often used for product search.
+ Next steps:
+ 1. Find the Algolia App ID and Search API Key (usually in page source)
+ 2. Query Algolia directly - very fast and low detection risk
+ 3. Search for "apiKey" or "applicationId" in the page source
+""")
+
+ if json_responses and not graphql_calls:
+ print("""
+[INFO] JSON responses found but no GraphQL. Check the analysis JSON for:
+ - Responses containing product arrays
+ - URLs with /api/ or /v1/, /v2/ patterns
+ - Look for pagination parameters (page, limit, offset)
+""")
+
+ if not json_responses and not graphql_calls:
+ print("""
+[WARNING] No obvious API endpoints found. Possible reasons:
+ - Server-side rendering (data embedded in HTML)
+ - API calls blocked by bot protection
+ - Need to scroll/interact more to trigger lazy loading
+
+Try:
+ - Scrolling more on the page before exporting HAR
+ - Clicking on product filters
+ - Looking at individual product pages
+""")
+
+def main():
+ # Default HAR file location
+ default_har = Path(__file__).parent / "pokemon_network.har"
+
+ # Check command line args
+ if len(sys.argv) > 1:
+ har_path = Path(sys.argv[1])
+ else:
+ har_path = default_har
+
+ if not har_path.exists():
+ print("=" * 70)
+ print("HAR File Analyzer")
+ print("=" * 70)
+ print(f"""
+No HAR file found at: {har_path}
+
+To capture a HAR file:
+1. Open Chrome and go to: https://www.pokemoncenter.com/category/tcg-cards
+2. Press F12 to open DevTools
+3. Go to the Network tab
+4. Check "Preserve log" checkbox
+5. Refresh the page (F5)
+6. Scroll down to load more products
+7. Right-click in Network panel → "Save all as HAR with content"
+8. Save as: {default_har}
+9. Run this script again
+
+Or specify a HAR file:
+ python analyze_har.py path/to/your/file.har
+""")
+ return
+
+ analyze_har(har_path)
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/api_discovery.py b/tools/api_discovery.py
new file mode 100644
index 0000000..9aec25a
--- /dev/null
+++ b/tools/api_discovery.py
@@ -0,0 +1,267 @@
+"""
+API Discovery Tool for Pokemon Center
+Attempts to find and test API endpoints that could be used instead of browser scraping.
+"""
+
+import requests
+import json
+from urllib.parse import urljoin
+
+# Common API patterns to try
+BASE_URL = "https://www.pokemoncenter.com"
+
+# Headers to mimic a real browser
+HEADERS = {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+ "Accept": "application/json, text/plain, */*",
+ "Accept-Language": "en-US,en;q=0.9",
+ "Referer": "https://www.pokemoncenter.com/",
+ "Origin": "https://www.pokemoncenter.com",
+}
+
+# Common API endpoint patterns for e-commerce sites
+API_PATTERNS = [
+ # REST API patterns
+ "/api/products",
+ "/api/v1/products",
+ "/api/v2/products",
+ "/api/catalog/products",
+ "/api/search",
+ "/api/inventory",
+
+ # GraphQL
+ "/graphql",
+ "/api/graphql",
+
+ # Common e-commerce platforms
+ "/rest/V1/products", # Magento
+ "/_api/products", # Wix
+ "/cdn/shop/products.json", # Shopify pattern
+ "/products.json", # Shopify
+
+ # Search APIs
+ "/api/search/products",
+ "/search/suggest",
+ "/api/autocomplete",
+
+ # Algolia (very common for e-commerce search)
+ # Note: Algolia requires app ID and API key from the page
+]
+
+# Pokemon TCG specific search terms
+SEARCH_TERMS = ["pokemon", "tcg", "cards", "booster", "etb"]
+
+
+def test_endpoint(url: str, method: str = "GET", data: dict = None) -> dict:
+ """Test an API endpoint"""
+ try:
+ if method == "GET":
+ response = requests.get(url, headers=HEADERS, timeout=10)
+ else:
+ response = requests.post(url, headers=HEADERS, json=data, timeout=10)
+
+ return {
+ "url": url,
+ "status": response.status_code,
+ "content_type": response.headers.get("content-type", ""),
+ "size": len(response.content),
+ "sample": response.text[:500] if response.status_code == 200 else None
+ }
+ except Exception as e:
+ return {
+ "url": url,
+ "status": "error",
+ "error": str(e)
+ }
+
+
+def discover_apis():
+ """Attempt to discover API endpoints"""
+ print("=" * 60)
+ print("Pokemon Center API Discovery")
+ print("=" * 60)
+ print()
+
+ results = []
+
+ # Test common patterns
+ print("Testing common API patterns...")
+ for pattern in API_PATTERNS:
+ url = urljoin(BASE_URL, pattern)
+ result = test_endpoint(url)
+ results.append(result)
+
+ if result["status"] == 200:
+ print(f" [OK] {url}")
+ print(f" Content-Type: {result['content_type']}")
+ print(f" Size: {result['size']} bytes")
+ elif result["status"] != "error" and result["status"] < 500:
+ print(f" [{result['status']}] {url}")
+
+ # Test search with query params
+ print()
+ print("Testing search endpoints...")
+ search_patterns = [
+ "/api/search?q=pokemon",
+ "/api/products?search=tcg",
+ "/api/catalog?category=tcg-cards",
+ "/search?q=pokemon+tcg",
+ ]
+
+ for pattern in search_patterns:
+ url = urljoin(BASE_URL, pattern)
+ result = test_endpoint(url)
+ results.append(result)
+
+ if result["status"] == 200:
+ print(f" [OK] {url}")
+
+ # Look for Algolia configuration
+ print()
+ print("Checking for Algolia search...")
+ # Algolia is often exposed in page source
+ try:
+ response = requests.get(BASE_URL, headers=HEADERS, timeout=15)
+ if "algolia" in response.text.lower():
+ print(" [!] Algolia detected in page source")
+ # Try to extract app ID and search key
+ import re
+ app_id = re.search(r'["\']?algolia[_-]?app[_-]?id["\']?\s*[:=]\s*["\']([A-Z0-9]+)["\']', response.text, re.I)
+ api_key = re.search(r'["\']?algolia[_-]?(?:search[_-]?)?(?:api[_-]?)?key["\']?\s*[:=]\s*["\']([a-f0-9]+)["\']', response.text, re.I)
+
+ if app_id:
+ print(f" App ID: {app_id.group(1)}")
+ if api_key:
+ print(f" Search Key: {api_key.group(1)}")
+
+ # Check for other API clues
+ if "graphql" in response.text.lower():
+ print(" [!] GraphQL detected in page source")
+
+ if "__NEXT_DATA__" in response.text:
+ print(" [!] Next.js detected - may have API routes at /api/*")
+
+ if "window.__INITIAL_STATE__" in response.text or "window.__PRELOADED_STATE__" in response.text:
+ print(" [!] Pre-rendered state detected - data may be in page source")
+
+ except Exception as e:
+ print(f" Error checking main page: {e}")
+
+ # Summary
+ print()
+ print("=" * 60)
+ print("Summary")
+ print("=" * 60)
+
+ working = [r for r in results if r.get("status") == 200]
+ if working:
+ print(f"Found {len(working)} potentially working endpoints:")
+ for r in working:
+ print(f" - {r['url']}")
+ else:
+ print("No direct API endpoints found.")
+ print()
+ print("Alternative approaches to consider:")
+ print(" 1. Monitor sitemap.xml for new products")
+ print(" 2. Use Google Shopping API or similar aggregators")
+ print(" 3. Check if they have an RSS feed")
+ print(" 4. Use a service like Distill.io for change detection")
+ print(" 5. Proxy rotation with residential IPs")
+ print(" 6. Lower check frequency + add human-like delays")
+
+ return results
+
+
+def check_sitemap():
+ """Check sitemap for product URLs"""
+ print()
+ print("Checking sitemap...")
+
+ sitemap_urls = [
+ "/sitemap.xml",
+ "/sitemap_index.xml",
+ "/sitemaps/sitemap.xml",
+ "/robots.txt", # Often contains sitemap location
+ ]
+
+ for pattern in sitemap_urls:
+ url = urljoin(BASE_URL, pattern)
+ result = test_endpoint(url)
+
+ if result["status"] == 200:
+ print(f" [OK] {url}")
+ if "sitemap" in result.get("sample", "").lower():
+ print(f" Contains sitemap references")
+ if "product" in result.get("sample", "").lower():
+ print(f" Contains product references")
+
+
+def check_rss():
+ """Check for RSS feeds"""
+ print()
+ print("Checking for RSS/Atom feeds...")
+
+ feed_urls = [
+ "/feed",
+ "/rss",
+ "/feed.xml",
+ "/rss.xml",
+ "/atom.xml",
+ "/blog/feed",
+ "/news/feed",
+ ]
+
+ for pattern in feed_urls:
+ url = urljoin(BASE_URL, pattern)
+ result = test_endpoint(url)
+
+ if result["status"] == 200 and ("xml" in result.get("content_type", "") or "rss" in result.get("content_type", "")):
+ print(f" [OK] {url}")
+
+
+if __name__ == "__main__":
+ discover_apis()
+ check_sitemap()
+ check_rss()
+
+ print()
+ print("=" * 60)
+ print("Next Steps")
+ print("=" * 60)
+ print("""
+To avoid bot detection, consider these strategies:
+
+1. API-BASED MONITORING (if endpoints found):
+ - Call API endpoints directly with requests
+ - Much faster and less detectable than browser
+ - Can check more frequently
+
+2. SITEMAP MONITORING:
+ - Parse sitemap.xml periodically
+ - Detect new product URLs without visiting pages
+ - Very low detection risk
+
+3. HASH-BASED CHANGE DETECTION:
+ - Fetch page, hash content
+ - Only alert when hash changes
+ - Reduces unnecessary processing
+
+4. RESIDENTIAL PROXY ROTATION:
+ - Use services like Bright Data, Oxylabs
+ - Rotate IPs to avoid blocks
+ - More expensive but reliable
+
+5. HUMAN-LIKE BEHAVIOR:
+ - Random delays between 60-180 seconds
+ - Vary user agent strings
+ - Add mouse movements and scrolling
+ - Use real browser cookies
+
+6. THIRD-PARTY ALERTS:
+ - Discord servers that track Pokemon Center
+ - Stock alert services (NowInStock, etc.)
+ - Browser extensions like Distill.io
+
+Run this script to see what APIs are available:
+ python api_discovery.py
+""")
diff --git a/tools/api_monitor.py b/tools/api_monitor.py
new file mode 100644
index 0000000..cda2b12
--- /dev/null
+++ b/tools/api_monitor.py
@@ -0,0 +1,239 @@
+"""
+Pokemon Center API Monitor
+
+Uses cookies from a browser session to make API calls directly.
+Much lighter than full browser scraping once cookies are established.
+
+Workflow:
+1. Run warmup_session.py first to get valid cookies
+2. This script uses those cookies to call APIs directly
+3. Falls back to browser if cookies expire
+"""
+
+import json
+import time
+import pickle
+import logging
+import requests
+from pathlib import Path
+from datetime import datetime
+from typing import Optional, Dict, List, Any
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Session/cookie storage
+SESSION_DIR = Path(__file__).parent / "sessions"
+COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
+
+# API endpoints discovered from HAR analysis
+API_BASE = "https://www.pokemoncenter.com"
+ENDPOINTS = {
+ "product": "/tpci-ecommweb-api/product/{sku}",
+ "status": "/tpci-ecommweb-api/product/status/{encoded_id}",
+ "category": "/site/resourceapi/category/{category}",
+ "reviews": "/tpci-ecommweb-api/review/get-product-scores",
+}
+
+# Required headers from HAR capture
+BASE_HEADERS = {
+ "Accept": "application/json",
+ "Accept-Version": "1",
+ "Content-Type": "application/json",
+ "X-Store-Locale": "en-us",
+ "X-Store-Scope": "pokemon",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+ "Referer": "https://www.pokemoncenter.com/category/tcg-cards",
+ "Sec-Fetch-Dest": "empty",
+ "Sec-Fetch-Mode": "cors",
+ "Sec-Fetch-Site": "same-origin",
+}
+
+# Known product SKUs for TCG (build this list over time)
+KNOWN_TCG_SKUS_FILE = Path(__file__).parent / "known_tcg_skus.json"
+
+
+class PokemonCenterAPI:
+ """Direct API client for Pokemon Center using session cookies"""
+
+ def __init__(self):
+ self.session = requests.Session()
+ self.session.headers.update(BASE_HEADERS)
+ self.cookies_loaded = False
+ self.last_cookie_refresh = None
+
+ def load_cookies(self) -> bool:
+ """Load cookies from the saved session file"""
+ if not COOKIE_FILE.exists():
+ logger.warning(f"No cookie file found at {COOKIE_FILE}")
+ logger.info("Run warmup_session.py first to create a valid session")
+ return False
+
+ try:
+ with open(COOKIE_FILE, 'rb') as f:
+ cookies = pickle.load(f)
+
+ # Add cookies to session
+ for cookie in cookies:
+ self.session.cookies.set(
+ cookie['name'],
+ cookie['value'],
+ domain=cookie.get('domain', '.pokemoncenter.com'),
+ path=cookie.get('path', '/')
+ )
+
+ logger.info(f"Loaded {len(cookies)} cookies from session file")
+ self.cookies_loaded = True
+ self.last_cookie_refresh = datetime.now()
+ return True
+
+ except Exception as e:
+ logger.error(f"Failed to load cookies: {e}")
+ return False
+
+ def _make_request(self, method: str, url: str, **kwargs) -> Optional[requests.Response]:
+ """Make a request with error handling"""
+ try:
+ if method.upper() == "GET":
+ response = self.session.get(url, timeout=15, **kwargs)
+ else:
+ response = self.session.post(url, timeout=15, **kwargs)
+
+ # Check for blocking
+ if response.status_code == 403:
+ if "captcha" in response.text.lower() or "blocked" in response.text.lower():
+ logger.warning("Request blocked - cookies may have expired")
+ self.cookies_loaded = False
+ return None
+
+ return response
+
+ except requests.RequestException as e:
+ logger.error(f"Request failed: {e}")
+ return None
+
+ def get_product(self, sku: str) -> Optional[Dict]:
+ """Get product details by SKU"""
+ if not self.cookies_loaded:
+ if not self.load_cookies():
+ return None
+
+ url = f"{API_BASE}{ENDPOINTS['product'].format(sku=sku)}"
+ response = self._make_request("GET", url)
+
+ if response and response.status_code == 200:
+ return response.json()
+ return None
+
+ def get_product_status(self, encoded_id: str) -> Optional[Dict]:
+ """Get product availability status"""
+ if not self.cookies_loaded:
+ if not self.load_cookies():
+ return None
+
+ url = f"{API_BASE}{ENDPOINTS['status'].format(encoded_id=encoded_id)}"
+ response = self._make_request("GET", url)
+
+ if response and response.status_code == 200:
+ return response.json()
+ return None
+
+ def get_category(self, category: str = "new-releases") -> Optional[Dict]:
+ """Get category listing (potential goldmine for new drops!)"""
+ if not self.cookies_loaded:
+ if not self.load_cookies():
+ return None
+
+ url = f"{API_BASE}{ENDPOINTS['category'].format(category=category)}"
+ response = self._make_request("GET", url)
+
+ if response and response.status_code == 200:
+ return response.json()
+ return None
+
+ def get_review_scores(self, sku_list: List[str]) -> Optional[Dict]:
+ """Get review scores for multiple SKUs"""
+ if not self.cookies_loaded:
+ if not self.load_cookies():
+ return None
+
+ skus = ",".join(sku_list)
+ url = f"{API_BASE}{ENDPOINTS['reviews']}?skuList={skus}"
+ response = self._make_request("GET", url)
+
+ if response and response.status_code == 200:
+ return response.json()
+ return None
+
+
+def load_known_skus() -> List[str]:
+ """Load list of known TCG SKUs"""
+ if KNOWN_TCG_SKUS_FILE.exists():
+ with open(KNOWN_TCG_SKUS_FILE, 'r') as f:
+ return json.load(f)
+ return []
+
+
+def save_known_skus(skus: List[str]):
+ """Save list of known TCG SKUs"""
+ with open(KNOWN_TCG_SKUS_FILE, 'w') as f:
+ json.dump(skus, f, indent=2)
+
+
+def test_api():
+ """Test the API client"""
+ print("=" * 70)
+ print("Pokemon Center API Monitor Test")
+ print("=" * 70)
+ print()
+
+ api = PokemonCenterAPI()
+
+ if not api.load_cookies():
+ print("\nNo valid cookies found!")
+ print("Please run: python warmup_session.py")
+ print("Then try again.")
+ return
+
+ print("\n[1] Testing category endpoint (new-releases)...")
+ category_data = api.get_category("new-releases")
+ if category_data:
+ print(" SUCCESS! Category data retrieved.")
+ print(f" Keys: {list(category_data.keys())[:5]}")
+
+ # Save for analysis
+ with open("category_response.json", "w") as f:
+ json.dump(category_data, f, indent=2)
+ print(" Saved to category_response.json")
+ else:
+ print(" FAILED - cookies may have expired")
+
+ print("\n[2] Testing product endpoint...")
+ # Try a known SKU from our HAR capture
+ product_data = api.get_product("699-17113")
+ if product_data:
+ print(" SUCCESS! Product data retrieved.")
+ print(f" Keys: {list(product_data.keys())[:5]}")
+ else:
+ print(" FAILED - cookies may have expired")
+
+ print("\n[3] Testing review scores endpoint...")
+ reviews = api.get_review_scores(["699-17113", "191-85953"])
+ if reviews:
+ print(" SUCCESS! Review scores retrieved.")
+ print(f" Data: {reviews}")
+ else:
+ print(" FAILED - cookies may have expired")
+
+ print()
+ print("=" * 70)
+ if category_data or product_data:
+ print("API access working! Can monitor without full browser scraping.")
+ print("Cookies will expire eventually - re-run warmup when needed.")
+ else:
+ print("API blocked. Need fresh cookies from browser session.")
+ print("=" * 70)
+
+
+if __name__ == "__main__":
+ test_api()
diff --git a/tools/backend_monitor.py b/tools/backend_monitor.py
new file mode 100644
index 0000000..26bdbd2
--- /dev/null
+++ b/tools/backend_monitor.py
@@ -0,0 +1,370 @@
+"""
+Pokemon Center Backend Monitor
+
+Detects NEW products added to the API before they're publicly announced.
+This is how accounts like @pokepullzhq detect drops early.
+
+Strategy:
+1. Maintain a list of all known product SKUs
+2. Periodically check the API for current products
+3. Compare: Any new SKUs = potential silent drop
+4. Alert immediately on new detections
+
+Usage:
+1. First run warmup_session.py to get valid cookies
+2. Then run: python backend_monitor.py
+"""
+
+import json
+import time
+import pickle
+import logging
+import requests
+from pathlib import Path
+from datetime import datetime
+from typing import Optional, Dict, List, Set
+
+# Setup logging
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# File paths
+DATA_DIR = Path(__file__).parent.parent / "data"
+SESSION_DIR = DATA_DIR / "sessions"
+COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
+KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
+DETECTIONS_LOG = DATA_DIR / "detections.json"
+
+# API configuration
+API_BASE = "https://www.pokemoncenter.com"
+
+# Categories to monitor for new products
+MONITOR_CATEGORIES = [
+ "new-releases",
+ "tcg-cards",
+ # Add more as needed
+]
+
+# Headers required for API calls
+HEADERS = {
+ "Accept": "application/json",
+ "Accept-Version": "1",
+ "Content-Type": "application/json",
+ "X-Store-Locale": "en-us",
+ "X-Store-Scope": "pokemon",
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+ "Referer": "https://www.pokemoncenter.com/",
+}
+
+# Check interval (seconds)
+CHECK_INTERVAL = 60
+
+
+class BackendMonitor:
+ """Monitors Pokemon Center API for new product drops"""
+
+ def __init__(self):
+ self.session = requests.Session()
+ self.session.headers.update(HEADERS)
+ self.known_skus: Set[str] = set()
+ self.cookies_valid = False
+ self.detections: List[Dict] = []
+
+ def load_cookies(self) -> bool:
+ """Load cookies from warmup session"""
+ if not COOKIE_FILE.exists():
+ logger.error(f"No cookie file found at {COOKIE_FILE}")
+ logger.info("Run warmup_session.py first!")
+ return False
+
+ try:
+ with open(COOKIE_FILE, 'rb') as f:
+ cookies = pickle.load(f)
+
+ for cookie in cookies:
+ self.session.cookies.set(
+ cookie['name'],
+ cookie['value'],
+ domain=cookie.get('domain', '.pokemoncenter.com'),
+ path=cookie.get('path', '/')
+ )
+
+ logger.info(f"Loaded {len(cookies)} cookies")
+ self.cookies_valid = True
+ return True
+
+ except Exception as e:
+ logger.error(f"Failed to load cookies: {e}")
+ return False
+
+ def load_known_skus(self):
+ """Load previously seen SKUs"""
+ if KNOWN_SKUS_FILE.exists():
+ with open(KNOWN_SKUS_FILE, 'r') as f:
+ data = json.load(f)
+ self.known_skus = set(data.get('skus', []))
+ logger.info(f"Loaded {len(self.known_skus)} known SKUs")
+ else:
+ logger.info("No known SKUs file - will create on first run")
+ self.known_skus = set()
+
+ def save_known_skus(self):
+ """Save known SKUs to file"""
+ with open(KNOWN_SKUS_FILE, 'w') as f:
+ json.dump({
+ 'skus': list(self.known_skus),
+ 'last_updated': datetime.now().isoformat(),
+ 'count': len(self.known_skus)
+ }, f, indent=2)
+
+ def log_detection(self, sku: str, product_info: Dict):
+ """Log a new product detection"""
+ detection = {
+ 'sku': sku,
+ 'detected_at': datetime.now().isoformat(),
+ 'product_info': product_info
+ }
+ self.detections.append(detection)
+
+ # Append to detections log file
+ detections = []
+ if DETECTIONS_LOG.exists():
+ with open(DETECTIONS_LOG, 'r') as f:
+ detections = json.load(f)
+
+ detections.append(detection)
+
+ with open(DETECTIONS_LOG, 'w') as f:
+ json.dump(detections, f, indent=2)
+
+ def get_category_products(self, category: str) -> Optional[Dict]:
+ """Fetch products from a category endpoint"""
+ url = f"{API_BASE}/site/resourceapi/category/{category}"
+
+ try:
+ response = self.session.get(url, timeout=15)
+
+ if response.status_code == 403:
+ logger.warning("API blocked - cookies may have expired")
+ self.cookies_valid = False
+ return None
+
+ if response.status_code == 200:
+ return response.json()
+
+ logger.warning(f"Unexpected status {response.status_code} for {category}")
+ return None
+
+ except Exception as e:
+ logger.error(f"Error fetching {category}: {e}")
+ return None
+
+ def get_product_details(self, sku: str) -> Optional[Dict]:
+ """Get full details for a specific product"""
+ url = f"{API_BASE}/tpci-ecommweb-api/product/{sku}"
+
+ try:
+ response = self.session.get(url, timeout=15)
+
+ if response.status_code == 200:
+ return response.json()
+
+ return None
+
+ except Exception as e:
+ logger.error(f"Error fetching product {sku}: {e}")
+ return None
+
+ def extract_skus_from_response(self, data: Dict) -> Set[str]:
+ """Extract product SKUs from API response"""
+ skus = set()
+
+ # The response structure varies - try multiple approaches
+ # This will need adjustment based on actual API response
+
+ def find_skus(obj, depth=0):
+ """Recursively find SKU-like values"""
+ if depth > 10: # Prevent infinite recursion
+ return
+
+ if isinstance(obj, dict):
+ # Look for SKU fields
+ for key in ['sku', 'skuCode', 'productId', 'id', 'code']:
+ if key in obj:
+ value = obj[key]
+ if isinstance(value, str) and self._looks_like_sku(value):
+ skus.add(value)
+
+ # Look in nested objects
+ for value in obj.values():
+ find_skus(value, depth + 1)
+
+ elif isinstance(obj, list):
+ for item in obj:
+ find_skus(item, depth + 1)
+
+ find_skus(data)
+ return skus
+
+ def _looks_like_sku(self, value: str) -> bool:
+ """Check if a string looks like a Pokemon Center SKU"""
+ # SKUs we've seen: 699-17113, 191-85953, 10-10191-109
+ if not value:
+ return False
+
+ # Must contain digits and possibly hyphens
+ has_digit = any(c.isdigit() for c in value)
+ reasonable_length = 5 <= len(value) <= 20
+
+ return has_digit and reasonable_length
+
+ def check_for_new_products(self) -> List[Dict]:
+ """Main check - look for new SKUs across all categories"""
+ if not self.cookies_valid:
+ if not self.load_cookies():
+ return []
+
+ new_products = []
+ current_skus = set()
+
+ for category in MONITOR_CATEGORIES:
+ logger.debug(f"Checking category: {category}")
+
+ data = self.get_category_products(category)
+ if data:
+ skus = self.extract_skus_from_response(data)
+ current_skus.update(skus)
+ logger.debug(f" Found {len(skus)} SKUs in {category}")
+
+ if not current_skus:
+ logger.warning("No SKUs found - API may not be returning data")
+ return []
+
+ # Find new SKUs
+ new_skus = current_skus - self.known_skus
+
+ if new_skus:
+ logger.info(f"!!! DETECTED {len(new_skus)} NEW SKU(s) !!!")
+
+ for sku in new_skus:
+ # Get full product details
+ details = self.get_product_details(sku)
+
+ product_info = {
+ 'sku': sku,
+ 'details': details,
+ 'detected_at': datetime.now().isoformat()
+ }
+
+ # Try to extract name from details
+ name = "Unknown Product"
+ if details:
+ # Look for name in various places
+ name = (
+ details.get('name') or
+ details.get('displayName') or
+ details.get('definition', {}).get('display-name') or
+ sku
+ )
+
+ logger.info(f" NEW: {sku} - {name}")
+ self.log_detection(sku, product_info)
+ new_products.append(product_info)
+
+ # Add to known SKUs
+ self.known_skus.add(sku)
+
+ # Save updated known SKUs
+ self.save_known_skus()
+
+ else:
+ logger.info(f"Check complete - {len(current_skus)} products, no new drops")
+
+ return new_products
+
+ def send_discord_alert(self, product: Dict):
+ """Send Discord notification for new product"""
+ # Import from your existing discord_notifier
+ try:
+ from src.discord_notifier import send_notification
+ # You'd format and send the alert here
+ logger.info(f"Discord alert sent for {product['sku']}")
+ except ImportError:
+ logger.warning("Discord notifier not available")
+
+ def run(self):
+ """Main monitoring loop"""
+ print("=" * 60)
+ print("Pokemon Center Backend Monitor")
+ print("=" * 60)
+ print()
+ print("This monitors for NEW products added to the API.")
+ print("New SKUs = potential silent drops before announcement!")
+ print()
+ print(f"Check interval: {CHECK_INTERVAL} seconds")
+ print(f"Monitoring categories: {', '.join(MONITOR_CATEGORIES)}")
+ print()
+ print("Press Ctrl+C to stop")
+ print("=" * 60)
+ print()
+
+ # Load known SKUs
+ self.load_known_skus()
+
+ # Load cookies
+ if not self.load_cookies():
+ print("\nERROR: No valid cookies!")
+ print("Run: python warmup_session.py")
+ return
+
+ # First check - populate known SKUs if empty
+ if not self.known_skus:
+ logger.info("First run - building initial SKU database...")
+ self.check_for_new_products()
+ logger.info(f"Baseline established with {len(self.known_skus)} products")
+ print()
+
+ # Main loop
+ check_count = 0
+ while True:
+ try:
+ check_count += 1
+ logger.info(f"--- Check #{check_count} ---")
+
+ new_products = self.check_for_new_products()
+
+ if new_products:
+ print()
+ print("!" * 60)
+ print("!!! NEW PRODUCT DETECTED !!!")
+ print("!" * 60)
+ for p in new_products:
+ print(f" SKU: {p['sku']}")
+ # Send Discord alert
+ self.send_discord_alert(p)
+ print("!" * 60)
+ print()
+
+ # Wait for next check
+ time.sleep(CHECK_INTERVAL)
+
+ except KeyboardInterrupt:
+ print("\nStopping monitor...")
+ self.save_known_skus()
+ break
+
+ except Exception as e:
+ logger.error(f"Error in main loop: {e}")
+ time.sleep(CHECK_INTERVAL)
+
+
+def main():
+ monitor = BackendMonitor()
+ monitor.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/capture_api_calls.py b/tools/capture_api_calls.py
new file mode 100644
index 0000000..dd1700d
--- /dev/null
+++ b/tools/capture_api_calls.py
@@ -0,0 +1,242 @@
+"""
+API Call Capture Tool for Pokemon Center
+
+This script opens Pokemon Center and logs ALL network requests,
+helping identify backend APIs used for product data.
+
+Usage:
+ python capture_api_calls.py
+
+Output:
+ - api_captures.json: All captured API calls
+ - Console output with interesting endpoints
+"""
+
+import json
+import time
+import re
+from datetime import datetime
+from pathlib import Path
+
+print("=" * 70)
+print("Pokemon Center API Capture Tool")
+print("=" * 70)
+print()
+
+# Check for selenium
+try:
+ from selenium import webdriver
+ from selenium.webdriver.chrome.options import Options
+ from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+except ImportError:
+ print("ERROR: selenium not installed")
+ print("Run: pip install selenium")
+ exit(1)
+
+# Storage for captured requests
+captured_requests = []
+interesting_patterns = [
+ r'api',
+ r'graphql',
+ r'product',
+ r'catalog',
+ r'search',
+ r'inventory',
+ r'stock',
+ r'algolia',
+ r'contentful',
+ r'commercetools',
+ r'\.json',
+]
+
+def is_interesting(url):
+ """Check if URL matches patterns we care about"""
+ url_lower = url.lower()
+ for pattern in interesting_patterns:
+ if re.search(pattern, url_lower):
+ return True
+ return False
+
+def setup_browser():
+ """Setup Chrome with network logging enabled"""
+ options = Options()
+ options.add_argument("--disable-blink-features=AutomationControlled")
+ options.add_argument("--window-size=1920,1080")
+
+ # Enable performance logging to capture network requests
+ options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
+
+ driver = webdriver.Chrome(options=options)
+ return driver
+
+def extract_network_requests(driver):
+ """Extract network requests from Chrome performance logs"""
+ logs = driver.get_log('performance')
+ requests = []
+
+ for entry in logs:
+ try:
+ log = json.loads(entry['message'])['message']
+
+ # We want Network.requestWillBeSent and Network.responseReceived
+ if log['method'] == 'Network.requestWillBeSent':
+ request = log['params']['request']
+ requests.append({
+ 'type': 'request',
+ 'url': request.get('url', ''),
+ 'method': request.get('method', ''),
+ 'headers': request.get('headers', {}),
+ 'postData': request.get('postData', None),
+ 'timestamp': entry['timestamp']
+ })
+
+ elif log['method'] == 'Network.responseReceived':
+ response = log['params']['response']
+ requests.append({
+ 'type': 'response',
+ 'url': response.get('url', ''),
+ 'status': response.get('status', 0),
+ 'mimeType': response.get('mimeType', ''),
+ 'headers': response.get('headers', {}),
+ 'timestamp': entry['timestamp']
+ })
+
+ except Exception:
+ pass
+
+ return requests
+
+def analyze_requests(requests):
+ """Analyze captured requests for interesting patterns"""
+ api_calls = []
+ seen_urls = set()
+
+ for req in requests:
+ url = req.get('url', '')
+
+ # Skip if already seen or not interesting
+ if url in seen_urls:
+ continue
+ if not is_interesting(url):
+ continue
+
+ # Skip common non-API resources
+ if any(ext in url for ext in ['.png', '.jpg', '.gif', '.css', '.woff', '.svg', '.ico']):
+ continue
+
+ seen_urls.add(url)
+ api_calls.append(req)
+
+ return api_calls
+
+def main():
+ driver = None
+
+ try:
+ print("Starting Chrome with network logging...")
+ driver = setup_browser()
+
+ print("Navigating to Pokemon Center TCG page...")
+ print("(This may trigger a CAPTCHA - solve it if needed)")
+ print()
+
+ # Navigate to the TCG category
+ driver.get("https://www.pokemoncenter.com/category/tcg-cards")
+
+ print("Waiting for page to load...")
+ time.sleep(10)
+
+ # Scroll down to trigger lazy loading
+ print("Scrolling to load more content...")
+ for i in range(3):
+ driver.execute_script("window.scrollBy(0, 800);")
+ time.sleep(2)
+
+ # Wait a bit more
+ time.sleep(5)
+
+ print()
+ print("Extracting network requests...")
+ requests = extract_network_requests(driver)
+ print(f"Captured {len(requests)} total network events")
+
+ # Analyze for interesting APIs
+ api_calls = analyze_requests(requests)
+ print(f"Found {len(api_calls)} potentially interesting API calls")
+
+ # Save all captures
+ output_file = Path(__file__).parent / "api_captures.json"
+ with open(output_file, 'w') as f:
+ json.dump({
+ 'captured_at': datetime.now().isoformat(),
+ 'page_url': driver.current_url,
+ 'total_requests': len(requests),
+ 'interesting_calls': api_calls,
+ 'all_requests': requests
+ }, f, indent=2)
+
+ print(f"\nSaved full capture to: {output_file}")
+
+ # Display interesting findings
+ print()
+ print("=" * 70)
+ print("INTERESTING API CALLS FOUND")
+ print("=" * 70)
+
+ if not api_calls:
+ print("No obvious API calls detected.")
+ print("This could mean:")
+ print(" 1. Data is server-rendered (no client API)")
+ print(" 2. API calls use non-standard paths")
+ print(" 3. Bot protection blocked the content")
+ else:
+ for call in api_calls:
+ url = call.get('url', '')
+ method = call.get('method', 'GET')
+ status = call.get('status', '-')
+ mime = call.get('mimeType', '')
+
+ # Truncate long URLs
+ display_url = url[:100] + '...' if len(url) > 100 else url
+
+ print(f"\n[{method}] {display_url}")
+ if status != '-':
+ print(f" Status: {status}, Type: {mime}")
+
+ # Check for key patterns
+ if 'graphql' in url.lower():
+ print(" ⚡ GraphQL endpoint!")
+ if 'algolia' in url.lower():
+ print(" 🔍 Algolia search!")
+ if 'product' in url.lower():
+ print(" 📦 Product data!")
+
+ print()
+ print("=" * 70)
+ print("NEXT STEPS")
+ print("=" * 70)
+ print("""
+1. Review api_captures.json for the full data
+2. Look for JSON responses containing product info
+3. Test promising endpoints with curl/requests
+4. Note any required headers (auth tokens, API keys)
+
+To test an endpoint manually:
+ curl -H "User-Agent: Mozilla/5.0..." "https://api.example.com/endpoint"
+""")
+
+ # Keep browser open for manual inspection
+ print()
+ input("Browser is still open. Press ENTER to close and exit...")
+
+ except Exception as e:
+ print(f"Error: {e}")
+ import traceback
+ traceback.print_exc()
+ finally:
+ if driver:
+ driver.quit()
+ print("Browser closed.")
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/smart_monitor.py b/tools/smart_monitor.py
new file mode 100644
index 0000000..ea91c3a
--- /dev/null
+++ b/tools/smart_monitor.py
@@ -0,0 +1,415 @@
+"""
+Smart Pokemon Center Monitor
+
+Uses stealth browser with adaptive modes and human-like behavior.
+Detects new products before they're announced.
+
+Modes:
+- STEALTH: Normal monitoring (~1 min intervals, human-like)
+- ALERT: Fast checking when new product detected (~20 sec)
+- COOLDOWN: Gradual return to stealth after alert
+
+Usage:
+ python smart_monitor.py
+"""
+
+import json
+import time
+import random
+import logging
+from pathlib import Path
+from datetime import datetime, timedelta
+from typing import Optional, Set, List, Dict
+from enum import Enum
+
+# Setup logging
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s [%(levelname)s] %(message)s',
+ datefmt='%H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+# File paths
+DATA_DIR = Path(__file__).parent.parent / "data"
+KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
+DETECTIONS_FILE = DATA_DIR / "detections.json"
+
+# URLs to monitor
+MONITOR_URLS = [
+ "https://www.pokemoncenter.com/category/tcg-cards?sort=newest",
+ "https://www.pokemoncenter.com/category/new-releases",
+]
+
+
+class MonitorMode(Enum):
+ STEALTH = "stealth"
+ ALERT = "alert"
+ COOLDOWN = "cooldown"
+
+
+class SmartMonitor:
+ """Adaptive monitor with human-like behavior"""
+
+ def __init__(self):
+ self.browser = None
+ self.known_skus: Set[str] = set()
+ self.mode = MonitorMode.STEALTH
+ self.alert_triggered_at: Optional[datetime] = None
+ self.check_count = 0
+ self.last_detection: Optional[Dict] = None
+
+ # Mode timing settings
+ self.timing = {
+ MonitorMode.STEALTH: (45, 90), # 45-90 seconds
+ MonitorMode.ALERT: (15, 30), # 15-30 seconds
+ MonitorMode.COOLDOWN: (60, 120), # 60-120 seconds
+ }
+
+ # Alert mode duration
+ self.alert_duration = timedelta(minutes=30)
+ self.cooldown_duration = timedelta(minutes=15)
+
+ def start_browser(self):
+ """Start the stealth browser"""
+ if self.browser:
+ return
+
+ logger.info("Starting stealth browser...")
+
+ try:
+ from stealth_browser import StealthBrowser
+ self.browser = StealthBrowser(
+ headless=False,
+ session_name="smart_monitor"
+ )
+ self.browser.start()
+ logger.info("Browser started successfully")
+ except Exception as e:
+ logger.error(f"Failed to start browser: {e}")
+ raise
+
+ def stop_browser(self):
+ """Stop the browser"""
+ if self.browser:
+ logger.info("Stopping browser...")
+ self.browser.stop()
+ self.browser = None
+
+ def load_known_skus(self):
+ """Load previously seen SKUs"""
+ if KNOWN_SKUS_FILE.exists():
+ with open(KNOWN_SKUS_FILE, 'r') as f:
+ data = json.load(f)
+ self.known_skus = set(data.get('skus', []))
+ logger.info(f"Loaded {len(self.known_skus)} known SKUs")
+ else:
+ self.known_skus = set()
+ logger.info("No known SKUs file - starting fresh")
+
+ def save_known_skus(self):
+ """Save known SKUs"""
+ with open(KNOWN_SKUS_FILE, 'w') as f:
+ json.dump({
+ 'skus': list(self.known_skus),
+ 'updated': datetime.now().isoformat(),
+ 'count': len(self.known_skus)
+ }, f, indent=2)
+
+ def log_detection(self, sku: str, name: str, url: str):
+ """Log a new product detection"""
+ detection = {
+ 'sku': sku,
+ 'name': name,
+ 'url': url,
+ 'detected_at': datetime.now().isoformat(),
+ 'mode': self.mode.value
+ }
+
+ self.last_detection = detection
+
+ # Load existing detections
+ detections = []
+ if DETECTIONS_FILE.exists():
+ with open(DETECTIONS_FILE, 'r') as f:
+ detections = json.load(f)
+
+ detections.append(detection)
+
+ with open(DETECTIONS_FILE, 'w') as f:
+ json.dump(detections, f, indent=2)
+
+ logger.info(f"Detection logged: {sku}")
+
+ def get_interval(self) -> float:
+ """Get randomized check interval based on current mode"""
+ min_sec, max_sec = self.timing[self.mode]
+
+ # Add gaussian jitter for more natural timing
+ base = (min_sec + max_sec) / 2
+ jitter = random.gauss(0, (max_sec - min_sec) / 4)
+ interval = base + jitter
+
+ # Clamp to bounds
+ return max(min_sec * 0.8, min(max_sec * 1.2, interval))
+
+ def maybe_do_human_action(self):
+ """Occasionally do something human-like"""
+ if not self.browser or not self.browser.driver:
+ return
+
+ action = random.random()
+
+ if action < 0.3:
+ # Scroll randomly
+ self.browser.human_scroll()
+ elif action < 0.4:
+ # Small mouse movement
+ self.browser.human_mouse_move()
+ elif action < 0.45:
+ # Longer pause (human distraction)
+ pause = random.uniform(3, 8)
+ logger.debug(f"Human pause: {pause:.1f}s")
+ time.sleep(pause)
+
+ def maybe_take_break(self) -> bool:
+ """Occasionally take a longer break"""
+ # 3% chance of a break
+ if random.random() < 0.03:
+ break_time = random.randint(120, 300) # 2-5 minutes
+ logger.info(f"Taking a break for {break_time}s (human-like pause)")
+ time.sleep(break_time)
+ return True
+ return False
+
+ def extract_skus_from_page(self) -> Set[str]:
+ """Extract product SKUs from the current page"""
+ if not self.browser or not self.browser.driver:
+ return set()
+
+ skus = set()
+
+ try:
+ # Get page source and parse
+ from bs4 import BeautifulSoup
+ html = self.browser.driver.page_source
+ soup = BeautifulSoup(html, 'html.parser')
+
+ # Look for product links - Pokemon Center format: /product/SKU/name
+ import re
+ product_links = soup.select('a[href*="/product/"]')
+
+ for link in product_links:
+ href = link.get('href', '')
+ # Extract SKU from URL like /product/699-17113/product-name
+ match = re.search(r'/product/([0-9]+-?[0-9]+)', href)
+ if match:
+ skus.add(match.group(1))
+
+ # Also look for data attributes
+ for elem in soup.select('[data-sku], [data-product-id]'):
+ sku = elem.get('data-sku') or elem.get('data-product-id')
+ if sku and re.match(r'^[0-9]+-?[0-9]+', sku):
+ skus.add(sku)
+
+ except Exception as e:
+ logger.error(f"Error extracting SKUs: {e}")
+
+ return skus
+
+ def check_page(self, url: str) -> Set[str]:
+ """Load a page and extract SKUs"""
+ if not self.browser:
+ self.start_browser()
+
+ try:
+ # Human-like delay before navigation
+ self.browser.human_delay(0.5, 2.0)
+
+ # Navigate
+ logger.debug(f"Loading: {url}")
+ self.browser.driver.get(url)
+
+ # Wait for page load
+ time.sleep(random.uniform(3, 6))
+
+ # Check for CAPTCHA
+ if self.browser.check_for_captcha():
+ logger.warning("CAPTCHA detected!")
+ print("\n" + "!" * 50)
+ print("CAPTCHA DETECTED - Please solve it in the browser")
+ print("!" * 50 + "\n")
+ self.browser.wait_for_captcha_solve(timeout=120)
+
+ # Human actions
+ self.maybe_do_human_action()
+
+ # Extract SKUs
+ skus = self.extract_skus_from_page()
+
+ return skus
+
+ except Exception as e:
+ logger.error(f"Error checking page: {e}")
+ return set()
+
+ def update_mode(self):
+ """Update monitoring mode based on state"""
+ now = datetime.now()
+
+ if self.mode == MonitorMode.ALERT:
+ # Check if alert period is over
+ if self.alert_triggered_at:
+ elapsed = now - self.alert_triggered_at
+ if elapsed > self.alert_duration:
+ logger.info("Alert period over, entering cooldown")
+ self.mode = MonitorMode.COOLDOWN
+
+ elif self.mode == MonitorMode.COOLDOWN:
+ # Check if cooldown is over
+ if self.alert_triggered_at:
+ elapsed = now - self.alert_triggered_at
+ if elapsed > (self.alert_duration + self.cooldown_duration):
+ logger.info("Cooldown over, returning to stealth mode")
+ self.mode = MonitorMode.STEALTH
+ self.alert_triggered_at = None
+
+ def trigger_alert_mode(self):
+ """Switch to alert mode"""
+ logger.info("!!! ENTERING ALERT MODE - Faster checks !!!")
+ self.mode = MonitorMode.ALERT
+ self.alert_triggered_at = datetime.now()
+
+ def send_discord_alert(self, sku: str, name: str, url: str):
+ """Send Discord notification"""
+ try:
+ from src.discord_notifier import DiscordNotifier
+ from config import DISCORD_WEBHOOK_URL
+
+ if DISCORD_WEBHOOK_URL and DISCORD_WEBHOOK_URL != "YOUR_WEBHOOK_URL_HERE":
+ notifier = DiscordNotifier(DISCORD_WEBHOOK_URL)
+ # Create a simple product dict
+ product = {
+ 'name': name,
+ 'url': f"https://www.pokemoncenter.com/product/{sku}",
+ 'price': 'Check site',
+ 'site': 'pokemoncenter',
+ }
+ notifier.send_stock_alert(product, "NEW BACKEND DETECTION")
+ logger.info("Discord alert sent!")
+ except Exception as e:
+ logger.warning(f"Could not send Discord alert: {e}")
+
+ def run_check(self) -> List[str]:
+ """Run a single check across all monitored URLs"""
+ self.check_count += 1
+ all_skus = set()
+ new_skus = []
+
+ logger.info(f"Check #{self.check_count} | Mode: {self.mode.value.upper()}")
+
+ for url in MONITOR_URLS:
+ skus = self.check_page(url)
+ all_skus.update(skus)
+
+ # Brief pause between pages
+ if url != MONITOR_URLS[-1]:
+ time.sleep(random.uniform(2, 5))
+
+ logger.info(f"Found {len(all_skus)} total SKUs")
+
+ # Find new SKUs
+ if self.known_skus: # Only check if we have a baseline
+ new = all_skus - self.known_skus
+ if new:
+ for sku in new:
+ logger.info(f"!!! NEW SKU DETECTED: {sku}")
+ new_skus.append(sku)
+
+ # Log and alert
+ url = f"https://www.pokemoncenter.com/product/{sku}"
+ self.log_detection(sku, f"New Product {sku}", url)
+ self.send_discord_alert(sku, f"New Product {sku}", url)
+
+ # Trigger alert mode
+ self.trigger_alert_mode()
+
+ # Update known SKUs
+ self.known_skus.update(all_skus)
+ self.save_known_skus()
+
+ return new_skus
+
+ def run(self):
+ """Main monitoring loop"""
+ print()
+ print("=" * 60)
+ print(" SMART POKEMON CENTER MONITOR")
+ print("=" * 60)
+ print()
+ print("Modes:")
+ print(f" STEALTH: {self.timing[MonitorMode.STEALTH]} sec (normal)")
+ print(f" ALERT: {self.timing[MonitorMode.ALERT]} sec (after detection)")
+ print(f" COOLDOWN: {self.timing[MonitorMode.COOLDOWN]} sec (transition)")
+ print()
+ print("Press Ctrl+C to stop")
+ print("=" * 60)
+ print()
+
+ try:
+ # Initialize
+ self.load_known_skus()
+ self.start_browser()
+
+ # First check - build baseline if needed
+ if not self.known_skus:
+ logger.info("First run - building SKU baseline...")
+ self.run_check()
+ logger.info(f"Baseline: {len(self.known_skus)} products")
+ print()
+
+ # Main loop
+ while True:
+ # Maybe take a break
+ if self.maybe_take_break():
+ continue
+
+ # Run check
+ new_skus = self.run_check()
+
+ if new_skus:
+ print()
+ print("!" * 60)
+ print("!!! NEW PRODUCT(S) DETECTED !!!")
+ for sku in new_skus:
+ print(f" -> {sku}")
+ print("!" * 60)
+ print()
+
+ # Update mode
+ self.update_mode()
+
+ # Get interval and wait
+ interval = self.get_interval()
+ logger.info(f"Next check in {interval:.0f}s")
+ time.sleep(interval)
+
+ except KeyboardInterrupt:
+ print("\nStopping monitor...")
+ except Exception as e:
+ logger.error(f"Monitor error: {e}")
+ import traceback
+ traceback.print_exc()
+ finally:
+ self.save_known_skus()
+ self.stop_browser()
+ print("Monitor stopped.")
+
+
+def main():
+ monitor = SmartMonitor()
+ monitor.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/stealth_browser.py b/tools/stealth_browser.py
new file mode 100644
index 0000000..728b683
--- /dev/null
+++ b/tools/stealth_browser.py
@@ -0,0 +1,539 @@
+"""
+Stealth Browser Module
+Uses undetected-chromedriver to bypass bot detection (Imperva, Cloudflare, etc.)
+Includes session persistence, proxy rotation, and human-like behavior.
+"""
+
+import os
+import json
+import time
+import random
+import logging
+import pickle
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+# Data directory
+DATA_DIR = Path(__file__).parent.parent / "data"
+
+# Session storage directory
+SESSION_DIR = DATA_DIR / "sessions"
+SESSION_DIR.mkdir(parents=True, exist_ok=True)
+
+# Proxy configuration file
+PROXY_FILE = DATA_DIR / "proxies.json"
+
+
+class ProxyRotator:
+ """Manages proxy rotation for requests"""
+
+ def __init__(self, proxy_file: Path = PROXY_FILE):
+ self.proxies: List[Dict] = []
+ self.current_index = 0
+ self.failed_proxies: Dict[str, datetime] = {}
+ self.cooldown_minutes = 30
+
+ self._load_proxies(proxy_file)
+
+ def _load_proxies(self, proxy_file: Path):
+ """Load proxies from configuration file"""
+ if proxy_file.exists():
+ try:
+ with open(proxy_file, 'r') as f:
+ data = json.load(f)
+ self.proxies = data.get("proxies", [])
+ logger.info(f"Loaded {len(self.proxies)} proxies")
+ except Exception as e:
+ logger.warning(f"Failed to load proxies: {e}")
+
+ def get_proxy(self) -> Optional[Dict]:
+ """Get next available proxy"""
+ if not self.proxies:
+ return None
+
+ # Clean up expired cooldowns
+ now = datetime.now()
+ self.failed_proxies = {
+ k: v for k, v in self.failed_proxies.items()
+ if now - v < timedelta(minutes=self.cooldown_minutes)
+ }
+
+ # Find next working proxy
+ attempts = 0
+ while attempts < len(self.proxies):
+ proxy = self.proxies[self.current_index]
+ proxy_key = f"{proxy.get('host')}:{proxy.get('port')}"
+
+ self.current_index = (self.current_index + 1) % len(self.proxies)
+
+ if proxy_key not in self.failed_proxies:
+ return proxy
+
+ attempts += 1
+
+ # All proxies in cooldown, return first one anyway
+ return self.proxies[0] if self.proxies else None
+
+ def mark_failed(self, proxy: Dict):
+ """Mark a proxy as failed (temporary cooldown)"""
+ if proxy:
+ proxy_key = f"{proxy.get('host')}:{proxy.get('port')}"
+ self.failed_proxies[proxy_key] = datetime.now()
+ logger.warning(f"Proxy {proxy_key} marked as failed")
+
+
+class StealthBrowser:
+ """
+ Stealth browser using undetected-chromedriver.
+ Designed to bypass Imperva/Incapsula and similar bot protection.
+ """
+
+ def __init__(
+ self,
+ headless: bool = False,
+ proxy: Optional[Dict] = None,
+ user_data_dir: Optional[str] = None,
+ session_name: str = "default"
+ ):
+ self.headless = headless
+ self.proxy = proxy
+ self.user_data_dir = user_data_dir
+ self.session_name = session_name
+ self.driver = None
+ self._setup_complete = False
+
+ def _get_chrome_options(self):
+ """Configure Chrome options for stealth"""
+ import undetected_chromedriver as uc
+
+ options = uc.ChromeOptions()
+
+ # Basic stealth settings - compatible with newer Chrome versions
+ options.add_argument("--disable-blink-features=AutomationControlled")
+ options.add_argument("--disable-dev-shm-usage")
+ options.add_argument("--no-sandbox")
+ options.add_argument("--disable-infobars")
+
+ # Window size (realistic resolution)
+ options.add_argument("--window-size=1920,1080")
+
+ # User data directory for session persistence
+ if self.user_data_dir:
+ options.add_argument(f"--user-data-dir={self.user_data_dir}")
+
+ # Proxy configuration
+ if self.proxy:
+ proxy_str = self._format_proxy(self.proxy)
+ if proxy_str:
+ options.add_argument(f"--proxy-server={proxy_str}")
+
+ # Headless mode (note: more detectable)
+ if self.headless:
+ options.add_argument("--headless=new")
+
+ return options
+
+ def _format_proxy(self, proxy: Dict) -> Optional[str]:
+ """Format proxy dict into Chrome proxy string"""
+ if not proxy:
+ return None
+
+ host = proxy.get("host")
+ port = proxy.get("port")
+
+ if not host or not port:
+ return None
+
+ protocol = proxy.get("protocol", "http")
+ return f"{protocol}://{host}:{port}"
+
+ def start(self):
+ """Start the browser"""
+ if self.driver:
+ return
+
+ try:
+ import undetected_chromedriver as uc
+
+ options = self._get_chrome_options()
+
+ # Create driver with version_main to match installed Chrome version
+ # This prevents "ChromeDriver only supports Chrome version X" errors
+ self.driver = uc.Chrome(
+ options=options,
+ use_subprocess=True,
+ version_main=146, # Match user's Chrome version
+ )
+
+ # Set realistic viewport
+ self.driver.set_window_size(1920, 1080)
+
+ # Load saved cookies if they exist
+ self._load_cookies()
+
+ self._setup_complete = True
+ logger.info("Stealth browser started successfully")
+
+ except Exception as e:
+ logger.error(f"Failed to start stealth browser: {e}")
+ raise
+
+ def stop(self):
+ """Stop the browser and save session"""
+ if self.driver:
+ try:
+ self._save_cookies()
+ self.driver.quit()
+ except Exception as e:
+ logger.warning(f"Error stopping browser: {e}")
+ finally:
+ self.driver = None
+ self._setup_complete = False
+
+ def _get_cookie_file(self) -> Path:
+ """Get path to cookie file for this session"""
+ return SESSION_DIR / f"{self.session_name}_cookies.pkl"
+
+ def _save_cookies(self):
+ """Save cookies to file for session persistence"""
+ if not self.driver:
+ return
+
+ try:
+ cookies = self.driver.get_cookies()
+ cookie_file = self._get_cookie_file()
+
+ with open(cookie_file, 'wb') as f:
+ pickle.dump(cookies, f)
+
+ logger.debug(f"Saved {len(cookies)} cookies to {cookie_file}")
+ except Exception as e:
+ logger.warning(f"Failed to save cookies: {e}")
+
+ def _load_cookies(self):
+ """Load cookies from file"""
+ cookie_file = self._get_cookie_file()
+
+ if not cookie_file.exists():
+ return
+
+ try:
+ with open(cookie_file, 'rb') as f:
+ cookies = pickle.load(f)
+
+ # Need to visit domain first before adding cookies
+ # This will be done when navigating to the actual page
+
+ self._pending_cookies = cookies
+ logger.debug(f"Loaded {len(cookies)} cookies from {cookie_file}")
+ except Exception as e:
+ logger.warning(f"Failed to load cookies: {e}")
+ self._pending_cookies = []
+
+ def _apply_pending_cookies(self, domain: str):
+ """Apply loaded cookies after visiting domain"""
+ if not hasattr(self, '_pending_cookies') or not self._pending_cookies:
+ return
+
+ for cookie in self._pending_cookies:
+ try:
+ # Only add cookies for matching domain
+ if domain in cookie.get('domain', ''):
+ self.driver.add_cookie(cookie)
+ except Exception:
+ pass # Some cookies may fail, that's ok
+
+ self._pending_cookies = []
+
+ def human_delay(self, min_seconds: float = 1.0, max_seconds: float = 3.0):
+ """Add human-like random delay"""
+ delay = random.uniform(min_seconds, max_seconds)
+ time.sleep(delay)
+
+ def human_scroll(self):
+ """Scroll like a human would"""
+ if not self.driver:
+ return
+
+ # Random scroll amount
+ scroll_amount = random.randint(200, 600)
+
+ # Smooth scroll
+ self.driver.execute_script(f"""
+ window.scrollBy({{
+ top: {scroll_amount},
+ behavior: 'smooth'
+ }});
+ """)
+
+ self.human_delay(0.5, 1.5)
+
+ def human_mouse_move(self):
+ """Simulate mouse movement (basic)"""
+ if not self.driver:
+ return
+
+ # Move mouse to random position
+ try:
+ from selenium.webdriver.common.action_chains import ActionChains
+
+ actions = ActionChains(self.driver)
+
+ # Random coordinates within viewport
+ x = random.randint(100, 800)
+ y = random.randint(100, 600)
+
+ # Move by offset from current position
+ actions.move_by_offset(x, y).perform()
+
+ # Reset position
+ actions.move_by_offset(-x, -y).perform()
+ except Exception:
+ pass # Mouse movement is optional
+
+ def get_page(self, url: str, wait_time: float = 5.0) -> str:
+ """
+ Navigate to URL with human-like behavior.
+
+ Args:
+ url: URL to navigate to
+ wait_time: Time to wait for page load
+
+ Returns:
+ Page HTML content
+ """
+ if not self.driver:
+ self.start()
+
+ try:
+ # Pre-navigation delay
+ self.human_delay(0.5, 1.5)
+
+ # Navigate
+ self.driver.get(url)
+
+ # Apply any pending cookies
+ from urllib.parse import urlparse
+ domain = urlparse(url).netloc
+ self._apply_pending_cookies(domain)
+
+ # Wait for page load
+ time.sleep(wait_time)
+
+ # Human-like behavior
+ self.human_scroll()
+ self.human_delay(1, 2)
+ self.human_mouse_move()
+
+ # Get page content
+ html = self.driver.page_source
+
+ # Save cookies after successful page load
+ self._save_cookies()
+
+ return html
+
+ except Exception as e:
+ logger.error(f"Error getting page {url}: {e}")
+ raise
+
+ def check_for_captcha(self) -> bool:
+ """Check if page has a CAPTCHA challenge blocking content"""
+ if not self.driver:
+ return False
+
+ page_source = self.driver.page_source.lower()
+
+ # First check: Did the page load actual content?
+ # If we see product elements, it's NOT a CAPTCHA page
+ content_loaded_indicators = [
+ 'class="product-card',
+ 'data-product-id',
+ 'data-sku',
+ '/product/',
+ 'add to cart',
+ 'product-grid',
+ 'product-list',
+ ]
+
+ for indicator in content_loaded_indicators:
+ if indicator in page_source:
+ # Page has actual content - no CAPTCHA
+ return False
+
+ # Only flag as CAPTCHA if we see blocking indicators AND no content
+ captcha_indicators = [
+ "verify you are human",
+ "press & hold",
+ "press and hold",
+ "checking your browser",
+ "just a moment",
+ "enable javascript and cookies",
+ "access denied",
+ "blocked",
+ "challenge-running",
+ "cf-browser-verification",
+ "ddos-guard",
+ ]
+
+ for indicator in captcha_indicators:
+ if indicator in page_source:
+ return True
+
+ # Also check if page is suspiciously empty (might be blocked)
+ if len(page_source) < 5000 and "pokemoncenter" not in page_source:
+ return True
+
+ return False
+
+ def wait_for_captcha_solve(self, timeout: int = 120):
+ """
+ Wait for user to solve CAPTCHA manually.
+ Only works in non-headless mode.
+ """
+ if self.headless:
+ logger.warning("Cannot solve CAPTCHA in headless mode")
+ return False
+
+ logger.info("CAPTCHA detected! Please solve it manually...")
+ print("\n" + "=" * 50)
+ print("CAPTCHA DETECTED!")
+ print("Please solve the CAPTCHA in the browser window.")
+ print("=" * 50 + "\n")
+
+ start_time = time.time()
+
+ while time.time() - start_time < timeout:
+ if not self.check_for_captcha():
+ logger.info("CAPTCHA solved!")
+ self._save_cookies() # Save session after solving
+ return True
+
+ time.sleep(2)
+
+ logger.warning("CAPTCHA solve timeout")
+ return False
+
+ def screenshot(self, filename: str = "screenshot.png"):
+ """Take a screenshot for debugging"""
+ if self.driver:
+ try:
+ self.driver.save_screenshot(filename)
+ logger.info(f"Screenshot saved to {filename}")
+ except Exception as e:
+ logger.warning(f"Failed to save screenshot: {e}")
+
+
+class StealthBrowserPool:
+ """
+ Manages multiple stealth browser instances with proxy rotation.
+ """
+
+ def __init__(
+ self,
+ pool_size: int = 1,
+ use_proxies: bool = False,
+ headless: bool = False
+ ):
+ self.pool_size = pool_size
+ self.use_proxies = use_proxies
+ self.headless = headless
+ self.browsers: List[StealthBrowser] = []
+ self.proxy_rotator = ProxyRotator() if use_proxies else None
+ self.current_index = 0
+
+ def get_browser(self) -> StealthBrowser:
+ """Get a browser from the pool"""
+ # Create browser if pool is empty
+ if not self.browsers:
+ proxy = self.proxy_rotator.get_proxy() if self.proxy_rotator else None
+ browser = StealthBrowser(
+ headless=self.headless,
+ proxy=proxy,
+ session_name=f"pool_{self.current_index}"
+ )
+ browser.start()
+ self.browsers.append(browser)
+ return browser
+
+ # Rotate through browsers
+ browser = self.browsers[self.current_index]
+ self.current_index = (self.current_index + 1) % len(self.browsers)
+ return browser
+
+ def shutdown_all(self):
+ """Shutdown all browsers in pool"""
+ for browser in self.browsers:
+ try:
+ browser.stop()
+ except Exception:
+ pass
+ self.browsers = []
+
+
+# Global instances
+_stealth_browser: Optional[StealthBrowser] = None
+_browser_pool: Optional[StealthBrowserPool] = None
+
+
+def get_stealth_browser(
+ headless: bool = False,
+ session_name: str = "pokemoncenter"
+) -> StealthBrowser:
+ """Get or create the global stealth browser instance"""
+ global _stealth_browser
+
+ if _stealth_browser is None:
+ _stealth_browser = StealthBrowser(
+ headless=headless,
+ session_name=session_name
+ )
+
+ if not _stealth_browser._setup_complete:
+ _stealth_browser.start()
+
+ return _stealth_browser
+
+
+def shutdown_stealth_browser():
+ """Shutdown the global stealth browser"""
+ global _stealth_browser
+
+ if _stealth_browser:
+ _stealth_browser.stop()
+ _stealth_browser = None
+
+
+def create_proxy_config_template():
+ """Create a template proxies.json file"""
+ template = {
+ "proxies": [
+ {
+ "host": "proxy1.example.com",
+ "port": 8080,
+ "protocol": "http",
+ "username": "user",
+ "password": "pass"
+ },
+ {
+ "host": "proxy2.example.com",
+ "port": 8080,
+ "protocol": "http",
+ "username": "user",
+ "password": "pass"
+ }
+ ],
+ "_comment": "Add your residential proxies here. Recommended providers: Bright Data, Oxylabs, Smartproxy"
+ }
+
+ if not PROXY_FILE.exists():
+ with open(PROXY_FILE, 'w') as f:
+ json.dump(template, f, indent=2)
+ logger.info(f"Created proxy template at {PROXY_FILE}")
+
+
+# Create template on import
+create_proxy_config_template()
diff --git a/tools/warmup.py b/tools/warmup.py
new file mode 100644
index 0000000..fba06fe
--- /dev/null
+++ b/tools/warmup.py
@@ -0,0 +1,74 @@
+"""
+Warmup tool for sites with CAPTCHA/bot protection.
+Run this before the main monitor to solve CAPTCHAs manually.
+
+Usage:
+ python tools/warmup.py gamestop
+ python tools/warmup.py all
+"""
+
+import sys
+import os
+
+# Add parent directory to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import logging
+from scrapers.gamestop import warmup_gamestop
+
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def warmup_all():
+ """Warm up all sites that need it"""
+ results = {}
+
+ logger.info("=" * 50)
+ logger.info("Starting warmup for GameStop...")
+ logger.info("=" * 50)
+ results["gamestop"] = warmup_gamestop()
+
+ # Add more sites here as needed
+ # results["pokemoncenter"] = warmup_pokemoncenter()
+
+ logger.info("=" * 50)
+ logger.info("Warmup Results:")
+ for site, success in results.items():
+ status = "OK" if success else "FAILED"
+ logger.info(f" {site}: {status}")
+ logger.info("=" * 50)
+
+ return all(results.values())
+
+
+def main():
+ if len(sys.argv) < 2:
+ print(__doc__)
+ print("\nAvailable sites: gamestop, all")
+ sys.exit(1)
+
+ site = sys.argv[1].lower()
+
+ if site == "gamestop":
+ success = warmup_gamestop()
+ elif site == "all":
+ success = warmup_all()
+ else:
+ print(f"Unknown site: {site}")
+ print("Available sites: gamestop, all")
+ sys.exit(1)
+
+ if success:
+ logger.info("Warmup completed successfully!")
+ sys.exit(0)
+ else:
+ logger.error("Warmup failed or timed out")
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/warmup_session.py b/tools/warmup_session.py
new file mode 100644
index 0000000..271269e
--- /dev/null
+++ b/tools/warmup_session.py
@@ -0,0 +1,119 @@
+"""
+Session Warmup Tool for Pokemon Center
+
+Run this BEFORE starting the monitor to:
+1. Open a stealth browser
+2. Let you manually browse and solve any CAPTCHAs
+3. Save cookies/session for the monitor to reuse
+
+Usage:
+ python warmup_session.py
+
+After running:
+1. Browse Pokemon Center normally for a few minutes
+2. Add items to cart, look at products, etc.
+3. Solve any CAPTCHAs that appear
+4. Press Enter in this terminal when done
+5. The session will be saved and reused by the monitor
+"""
+
+import sys
+import time
+
+print("=" * 60)
+print("Pokemon Center Session Warmup")
+print("=" * 60)
+print()
+
+# Import stealth browser
+try:
+ from .stealth_browser import StealthBrowser
+except ImportError as e:
+ print(f"Error importing stealth browser: {e}")
+ sys.exit(1)
+
+print("Starting stealth browser...")
+print("This will open a Chrome window.")
+print()
+
+browser = StealthBrowser(
+ headless=False,
+ session_name="pokemoncenter" # Same name used by monitor
+)
+
+try:
+ browser.start()
+ print("[OK] Browser started!")
+ print()
+
+ # Navigate to Pokemon Center
+ print("Navigating to Pokemon Center...")
+ browser.driver.get("https://www.pokemoncenter.com")
+
+ time.sleep(3)
+
+ print()
+ print("=" * 60)
+ print("WARMUP INSTRUCTIONS")
+ print("=" * 60)
+ print("""
+1. If you see a CAPTCHA or "Pardon Our Interruption":
+ - Solve it in the browser window
+ - Wait for the page to load
+
+2. Browse naturally for 2-3 minutes:
+ - Click on some products
+ - Look at different categories
+ - Add something to cart (you don't have to buy)
+ - This builds a legitimate browsing profile
+
+3. Navigate to the TCG section:
+ - https://www.pokemoncenter.com/category/tcg-cards
+
+4. Once you're browsing normally without issues:
+ - Come back here
+ - Press ENTER to save the session
+""")
+ print("=" * 60)
+ print()
+
+ input("Press ENTER when you're done browsing to save the session...")
+
+ print()
+ print("Saving session cookies...")
+ browser._save_cookies()
+
+ # Check how many cookies we got
+ cookies = browser.driver.get_cookies()
+ print(f"[OK] Saved {len(cookies)} cookies")
+
+ # Get current URL for reference
+ current_url = browser.driver.current_url
+ print(f"[OK] Final URL: {current_url}")
+
+ print()
+ print("=" * 60)
+ print("SESSION SAVED!")
+ print("=" * 60)
+ print("""
+Your session has been saved. The monitor will now use these
+cookies when checking Pokemon Center.
+
+Tips for best results:
+- Run the monitor with 3-5 minute check intervals
+- Keep USE_STEALTH_BROWSER = True in config.py
+- If you get blocked again, run this warmup again
+
+Session file: data/sessions/pokemoncenter_cookies.pkl
+""")
+
+except KeyboardInterrupt:
+ print("\nInterrupted by user")
+except Exception as e:
+ print(f"Error: {e}")
+ import traceback
+ traceback.print_exc()
+finally:
+ print("Closing browser...")
+ browser.stop()
+ print("Done!")