From 8d382e723fbe83f1bf032fedc5c0ab58441131f9 Mon Sep 17 00:00:00 2001 From: Mike McGhen Date: Fri, 27 Mar 2026 23:08:09 -0400 Subject: [PATCH] feat: Implement Walmart scraper and integrate with existing architecture - Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability. --- .gitignore | 17 +- CLAUDE.md | 137 +++ chrome-extension/api-interceptor.js | 206 ++++ chrome-extension/background.js | 235 +++- chrome-extension/content.js | 25 +- chrome-extension/manifest.json | 12 +- chrome-extension/popup.html | 14 + chrome-extension/popup.js | 16 +- config.py | 112 +- dashboard/__init__.py | 4 + dashboard/api.py | 1082 ++++++++++++++++++ dashboard/app.py | 77 ++ dashboard/static/app.js | 1280 +++++++++++++++++++++ dashboard/static/style.css | 1613 +++++++++++++++++++++++++++ dashboard/templates/index.html | 371 ++++++ data/__init__.py | 1 + debug_bestbuy.html | 688 ++++++++++++ debug_gamestop.html | 1 + discord_notifier.py | 183 --- docs/BOT_DETECTION_STRATEGIES.md | 189 ++++ docs/NOTES.md | 13 + docs/POKEMON_CENTER_DROPS_NOTES.md | 46 + docs/SMART_MONITOR_DESIGN.md | 199 ++++ docs/backend-api-monitor.md | 285 +++++ main.py | 268 ++++- product_tracker.py | 136 --- pytest.ini | 9 + requirements.txt | 10 + scrapers/__init__.py | 12 +- scrapers/base.py | 44 + scrapers/bestbuy.py | 505 +++++++++ scrapers/gamestop.py | 478 ++++++++ scrapers/pokemoncenter.py | 106 +- scrapers/target.py | 130 ++- scrapers/walmart.py | 444 ++++++++ src/__init__.py | 1 + browser.py => src/browser.py | 12 + src/database.py | 929 +++++++++++++++ src/discord_bot.py | 353 ++++++ src/discord_notifier.py | 340 ++++++ src/favorites.py | 192 ++++ src/news/__init__.py | 9 + src/news/base.py | 96 ++ src/news/pokemon_fetcher.py | 222 ++++ src/news/sentiment.py | 218 ++++ src/product_tracker.py | 246 ++++ src/scraper_state.py | 264 +++++ src/store_locator.py | 382 +++++++ tests/__init__.py | 1 + tests/test_api.py | 68 ++ tests/test_dashboard_api.py | 407 +++++++ tests/test_database.py | 300 +++++ tests/test_product_filter.py | 175 +++ tests/test_stealth.py | 111 ++ tools/__init__.py | 1 + tools/analyze_har.py | 336 ++++++ tools/api_discovery.py | 267 +++++ tools/api_monitor.py | 239 ++++ tools/backend_monitor.py | 370 ++++++ tools/capture_api_calls.py | 242 ++++ tools/smart_monitor.py | 415 +++++++ tools/stealth_browser.py | 539 +++++++++ tools/warmup.py | 74 ++ tools/warmup_session.py | 119 ++ 64 files changed, 15433 insertions(+), 443 deletions(-) create mode 100644 CLAUDE.md create mode 100644 chrome-extension/api-interceptor.js create mode 100644 dashboard/__init__.py create mode 100644 dashboard/api.py create mode 100644 dashboard/app.py create mode 100644 dashboard/static/app.js create mode 100644 dashboard/static/style.css create mode 100644 dashboard/templates/index.html create mode 100644 data/__init__.py create mode 100644 debug_bestbuy.html create mode 100644 debug_gamestop.html delete mode 100644 discord_notifier.py create mode 100644 docs/BOT_DETECTION_STRATEGIES.md create mode 100644 docs/NOTES.md create mode 100644 docs/POKEMON_CENTER_DROPS_NOTES.md create mode 100644 docs/SMART_MONITOR_DESIGN.md create mode 100644 docs/backend-api-monitor.md delete mode 100644 product_tracker.py create mode 100644 pytest.ini create mode 100644 scrapers/bestbuy.py create mode 100644 scrapers/gamestop.py create mode 100644 scrapers/walmart.py create mode 100644 src/__init__.py rename browser.py => src/browser.py (94%) create mode 100644 src/database.py create mode 100644 src/discord_bot.py create mode 100644 src/discord_notifier.py create mode 100644 src/favorites.py create mode 100644 src/news/__init__.py create mode 100644 src/news/base.py create mode 100644 src/news/pokemon_fetcher.py create mode 100644 src/news/sentiment.py create mode 100644 src/product_tracker.py create mode 100644 src/scraper_state.py create mode 100644 src/store_locator.py create mode 100644 tests/__init__.py create mode 100644 tests/test_api.py create mode 100644 tests/test_dashboard_api.py create mode 100644 tests/test_database.py create mode 100644 tests/test_product_filter.py create mode 100644 tests/test_stealth.py create mode 100644 tools/__init__.py create mode 100644 tools/analyze_har.py create mode 100644 tools/api_discovery.py create mode 100644 tools/api_monitor.py create mode 100644 tools/backend_monitor.py create mode 100644 tools/capture_api_calls.py create mode 100644 tools/smart_monitor.py create mode 100644 tools/stealth_browser.py create mode 100644 tools/warmup.py create mode 100644 tools/warmup_session.py diff --git a/.gitignore b/.gitignore index 6f0ecc3..b319d77 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,26 @@ __pycache__/ venv/ .venv/ -# Logs and data +# Logs *.log + +# Data files (runtime/generated) products.json +data/sessions/ +data/*.json +data/*.pkl +*.pkl + +# Debug artifacts debug_*.png +# Large analysis files +*.har +*_analysis.json + +# Database +*.db + # IDE .vscode/ .idea/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e287d64 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# Pokemon Stock Monitor - Project Guide + +## Overview +A Pokemon TCG stock monitoring system that tracks retail sites (Target, PokemonCenter, GameStop, Best Buy, Walmart) for restocks and new drops, sending Discord notifications when products become available. + +## Architecture + +``` +User -> Chrome Extension (PokemonCenter API interception) + -> Python Monitor (main.py) -> Scrapers -> ProductTracker -> Discord Notifications + -> Flask Dashboard (localhost:5000) -> REST API +``` + +## Directory Structure + +### Core Application +- `main.py` - Entry point, scheduling loop, orchestrates scraping +- `config.py` - All configuration (intervals, URLs, Discord webhook, browser settings) + +### Source Code (`src/`) +- `browser.py` - Playwright/Chrome browser management +- `product_tracker.py` - Detects new products and restocks, manages state +- `discord_notifier.py` - Sends Discord webhook notifications +- `discord_bot.py` - Interactive Discord bot commands +- `database.py` - SQLite database for historical tracking +- `favorites.py` - User favorites/watchlist management +- `scraper_state.py` - Persists scraper state between runs +- `store_locator.py` - Target store location lookup + +### Scrapers (`scrapers/`) +- `base.py` - `Product` dataclass, `BaseScraper` abstract class +- `pokemoncenter.py` - PokemonCenter scraper (needs stealth browser) +- `target.py` - Target.com scraper +- `gamestop.py` - GameStop.com scraper (light bot protection) +- `bestbuy.py` - BestBuy.com scraper (moderate bot protection) +- `walmart.py` - Walmart.com scraper (aggressive bot protection, may need stealth) + +### Chrome Extension (`chrome-extension/`) +- `manifest.json` - Extension config (MV3) +- `background.js` - Service worker, manages alarms and notifications +- `content.js` - Injected into PokemonCenter pages +- `api-interceptor.js` - Intercepts PokemonCenter API responses +- `popup.html/js` - Extension popup UI + +### Dashboard (`dashboard/`) +- `app.py` - Flask app, routes +- `api.py` - REST API endpoints (`/api/*`) +- `templates/index.html` - Main template +- `static/app.js` - Frontend JavaScript + +### Data (`data/`) +- `products.json` - Tracked products state +- `scraper_state.json` - Persisted scraper state +- `known_skus.json` - Known product SKUs +- `proxies.json` - Proxy configuration +- `user_locations.json` - User store locations + +### Tools (`tools/`) +Development and debugging utilities: +- `api_monitor.py`, `backend_monitor.py` - API monitoring +- `stealth_browser.py` - Anti-detection browser +- `capture_api_calls.py` - HAR capture for debugging + +## Key Classes + +### `Product` (scrapers/base.py) +```python +@dataclass +class Product: + name: str + url: str + price: Optional[str] + in_stock: bool + image_url: Optional[str] + site: str + product_id: Optional[str] +``` + +### `ProductTracker` (src/product_tracker.py) +- `process_products(products)` -> `(new_products, restocked_products)` +- `get_stats()` -> dashboard statistics + +### `BaseScraper` (scrapers/base.py) +- `scrape_category_page(url)` -> `List[Product]` +- `check_product_stock(url)` -> `(in_stock, price)` + +## Common Tasks + +### Add a new retailer scraper +1. Create `scrapers/newsite.py` extending `BaseScraper` +2. Implement `scrape_category_page()` and `check_product_stock()` +3. Add to `scrapers/__init__.py` +4. Add config in `config.py` (URLs, enable flag) +5. Add check function in `main.py` + +### Modify Discord notifications +- Webhook format: `src/discord_notifier.py` +- Bot commands: `src/discord_bot.py` + +### Change check intervals or URLs +- Edit `config.py` + +### Dashboard changes +- Routes: `dashboard/app.py` +- API: `dashboard/api.py` +- Frontend: `dashboard/static/app.js` + +### Chrome extension changes +- Popup UI: `chrome-extension/popup.html`, `popup.js` +- API interception: `chrome-extension/api-interceptor.js` +- Background logic: `chrome-extension/background.js` + +## Conventions +- Python 3.13+ +- Logging via `logging` module (configured in main.py) +- Dataclasses for data structures +- Flask for web/API +- Playwright for browser automation +- SQLite for persistence (`stats.db`) + +## Running + +```bash +# Main monitor +python main.py + +# Dashboard only +python dashboard/app.py + +# Discord bot +python src/discord_bot.py +``` + +## Testing +```bash +python -m pytest tests/ +``` diff --git a/chrome-extension/api-interceptor.js b/chrome-extension/api-interceptor.js new file mode 100644 index 0000000..4ca5f5f --- /dev/null +++ b/chrome-extension/api-interceptor.js @@ -0,0 +1,206 @@ +// API Interceptor - Captures Pokemon Center API responses +// Injected into the page context to intercept fetch/XHR + +(function() { + 'use strict'; + + // Avoid double injection + if (window.__pokemonMonitorInjected) return; + window.__pokemonMonitorInjected = true; + + const API_PATTERNS = [ + '/tpci-ecommweb-api/product/', + '/site/resourceapi/category/', + '/tpci-ecommweb-api/review/', + '/graphql', + ]; + + function isApiUrl(url) { + return API_PATTERNS.some(pattern => url.includes(pattern)); + } + + function extractSkusFromData(data, skus = new Set()) { + if (!data) return skus; + + if (typeof data === 'object') { + // Look for SKU-like fields + for (const key of ['sku', 'skuCode', 'productId', 'id', 'code']) { + if (data[key] && typeof data[key] === 'string') { + const value = data[key]; + // Pokemon Center SKUs: 699-17113, 191-85953, etc. + if (/^\d{1,3}-\d{4,6}$/.test(value) || /^\d{5,}$/.test(value)) { + skus.add(value); + } + } + } + + // Recurse into nested objects/arrays + for (const value of Object.values(data)) { + if (typeof value === 'object') { + extractSkusFromData(value, skus); + } + } + } + + if (Array.isArray(data)) { + for (const item of data) { + extractSkusFromData(item, skus); + } + } + + return skus; + } + + function sendToExtension(type, data) { + window.postMessage({ + source: 'pokemon-api-interceptor', + type: type, + data: data + }, '*'); + } + + // Intercept fetch + const originalFetch = window.fetch; + window.fetch = async function(...args) { + const response = await originalFetch.apply(this, args); + + try { + const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || ''; + + if (isApiUrl(url) && response.ok) { + // Clone response so we can read it without consuming + const clone = response.clone(); + const contentType = clone.headers.get('content-type') || ''; + + if (contentType.includes('application/json')) { + clone.json().then(data => { + const skus = extractSkusFromData(data); + + sendToExtension('api-response', { + url: url, + timestamp: Date.now(), + skuCount: skus.size, + skus: Array.from(skus), + preview: JSON.stringify(data).slice(0, 500) + }); + }).catch(() => {}); + } + } + } catch (e) { + // Silently ignore errors + } + + return response; + }; + + // Intercept XMLHttpRequest + const originalXhrOpen = XMLHttpRequest.prototype.open; + const originalXhrSend = XMLHttpRequest.prototype.send; + + XMLHttpRequest.prototype.open = function(method, url, ...rest) { + this._monitorUrl = url; + return originalXhrOpen.apply(this, [method, url, ...rest]); + }; + + XMLHttpRequest.prototype.send = function(...args) { + const xhr = this; + const url = xhr._monitorUrl || ''; + + if (isApiUrl(url)) { + xhr.addEventListener('load', function() { + try { + if (xhr.status === 200) { + const contentType = xhr.getResponseHeader('content-type') || ''; + if (contentType.includes('application/json')) { + const data = JSON.parse(xhr.responseText); + const skus = extractSkusFromData(data); + + sendToExtension('api-response', { + url: url, + timestamp: Date.now(), + skuCount: skus.size, + skus: Array.from(skus), + preview: JSON.stringify(data).slice(0, 500) + }); + } + } + } catch (e) { + // Silently ignore + } + }); + } + + return originalXhrSend.apply(this, args); + }; + + console.log('[Pokemon Monitor] API interceptor active'); + + // Also scan embedded page data for SKUs (handles server-side rendered pages) + function scanPageForSkus() { + const skus = new Set(); + + // Method 1: Extract from product URLs in the page + const productLinks = document.querySelectorAll('a[href*="/product/"]'); + productLinks.forEach(link => { + const match = link.href.match(/\/product\/(\d{1,3}-\d{4,6})/); + if (match) { + skus.add(match[1]); + } + }); + + // Method 2: Look for __NEXT_DATA__ (Next.js embedded data) + const nextDataScript = document.getElementById('__NEXT_DATA__'); + if (nextDataScript) { + try { + const data = JSON.parse(nextDataScript.textContent); + extractSkusFromData(data, skus); + } catch (e) {} + } + + // Method 3: Look for any script tags with JSON containing product data + document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]').forEach(script => { + try { + const data = JSON.parse(script.textContent); + extractSkusFromData(data, skus); + } catch (e) {} + }); + + // Method 4: Look for data attributes on product elements + document.querySelectorAll('[data-sku], [data-product-id], [data-product-sku]').forEach(el => { + const sku = el.dataset.sku || el.dataset.productId || el.dataset.productSku; + if (sku && /^\d{1,3}-\d{4,6}$/.test(sku)) { + skus.add(sku); + } + }); + + if (skus.size > 0) { + console.log(`[Pokemon Monitor] Found ${skus.size} SKUs in page`); + sendToExtension('api-response', { + url: window.location.href, + timestamp: Date.now(), + skuCount: skus.size, + skus: Array.from(skus), + source: 'page-scan' + }); + } + } + + // Scan page after it loads + if (document.readyState === 'complete') { + setTimeout(scanPageForSkus, 1000); + } else { + window.addEventListener('load', () => setTimeout(scanPageForSkus, 1000)); + } + + // Re-scan when page content changes (for infinite scroll, etc.) + let scanTimeout; + const observer = new MutationObserver(() => { + clearTimeout(scanTimeout); + scanTimeout = setTimeout(scanPageForSkus, 2000); + }); + + // Start observing after initial load + setTimeout(() => { + observer.observe(document.body, { childList: true, subtree: true }); + }, 3000); +})(); diff --git a/chrome-extension/background.js b/chrome-extension/background.js index e44e0a5..aaa9c1d 100644 --- a/chrome-extension/background.js +++ b/chrome-extension/background.js @@ -9,7 +9,9 @@ const DEFAULT_CONFIG = { ], keywords: [], // Empty = all products, or ["chaos rising", "booster"] etc notifyNewProducts: true, - notifyRestocks: true + notifyRestocks: true, + dashboardUrl: "http://localhost:5000", // Local dashboard URL + syncToDashboard: true // Enable syncing to local dashboard }; // Store known products @@ -20,11 +22,21 @@ let checkLoopRunning = false; // Prevent multiple loops let isChecking = false; // Prevent overlapping checks let lastNotificationTime = 0; // Rate limiting for Discord +// API monitoring - track SKUs seen from backend API calls +let knownSkus = new Set(); +let apiStats = { + lastApiResponse: null, + totalSkusTracked: 0, + newSkusDetected: 0, + apiResponseCount: 0 +}; + // Initialize chrome.runtime.onInstalled.addListener(() => { console.log("Pokemon Stock Monitor installed"); loadConfig().then(() => { loadProducts(); + loadKnownSkus(); startCheckLoop(); }); }); @@ -33,6 +45,7 @@ chrome.runtime.onInstalled.addListener(() => { chrome.runtime.onStartup.addListener(() => { loadConfig().then(() => { loadProducts(); + loadKnownSkus(); startCheckLoop(); }); }); @@ -108,6 +121,97 @@ async function saveProducts() { await chrome.storage.local.set({ knownProducts }); } +// Load known SKUs from storage (for API monitoring) +async function loadKnownSkus() { + const stored = await chrome.storage.local.get("knownSkus"); + if (stored.knownSkus) { + knownSkus = new Set(stored.knownSkus); + } + console.log(`[API Monitor] Loaded ${knownSkus.size} known SKUs`); +} + +// Save known SKUs to storage +async function saveKnownSkus() { + await chrome.storage.local.set({ knownSkus: Array.from(knownSkus) }); +} + +// Handle API data from content script interceptor +async function handleApiData(data) { + apiStats.apiResponseCount++; + apiStats.lastApiResponse = new Date().toISOString(); + + if (!data.skus || data.skus.length === 0) return; + + const newSkus = []; + + for (const sku of data.skus) { + if (!knownSkus.has(sku)) { + knownSkus.add(sku); + newSkus.push(sku); + apiStats.newSkusDetected++; + } + } + + apiStats.totalSkusTracked = knownSkus.size; + + if (newSkus.length > 0) { + console.log(`[API Monitor] NEW SKUs detected: ${newSkus.join(', ')}`); + + // Send Discord alert for new SKUs + if (config.discordWebhook && config.notifyNewProducts) { + for (const sku of newSkus.slice(0, 3)) { // Limit to 3 to avoid spam + await sendSkuAlert(sku, data.url); + await new Promise(r => setTimeout(r, 1000)); + } + } + + await saveKnownSkus(); + } +} + +// Send alert for new SKU detected via API +async function sendSkuAlert(sku, sourceUrl) { + const productUrl = `https://www.pokemoncenter.com/product/${sku}`; + + const embed = { + title: "NEW SKU DETECTED (API)", + description: `**SKU: ${sku}**\n\nDetected in backend API before public listing!`, + url: productUrl, + color: 0xFF00FF, // Magenta for API detections + fields: [ + { name: "SKU", value: sku, inline: true }, + { name: "Source", value: "API Intercept", inline: true }, + { name: "Link", value: `[VIEW PRODUCT](${productUrl})`, inline: false } + ], + footer: { text: "Pokemon Monitor - API Detection" }, + timestamp: new Date().toISOString() + }; + + try { + const response = await fetch(config.discordWebhook, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + content: "@everyone NEW SKU FROM API!", + embeds: [embed] + }) + }); + + if (response.ok) { + console.log(`[API Monitor] Discord alert sent for SKU: ${sku}`); + + chrome.notifications.create({ + type: "basic", + iconUrl: "icon128.png", + title: "NEW SKU DETECTED!", + message: `SKU: ${sku} - Check Discord!` + }); + } + } catch (error) { + console.error("[API Monitor] Error sending alert:", error); + } +} + // Main stock check function async function runStockCheck() { if (!config.enabled) { @@ -287,11 +391,55 @@ function processProducts(products) { lastSeen: now, lastInStock: product.inStock ? now : null }; + + // Queue new_drop event for dashboard sync + pendingEvents.push({ + type: 'new_drop', + url: product.url, + name: product.name, + price: product.price, + inStock: product.inStock, + timestamp: now + }); } else { // Existing product - check for restock if (product.inStock && !existing.inStock) { restockedProducts.push(product); existing.lastInStock = now; + + // Queue restock event for dashboard sync + pendingEvents.push({ + type: 'restock', + url: product.url, + name: product.name, + price: product.price, + previousPrice: existing.price, + timestamp: now + }); + } + + // Track when items go OUT of stock (for selling rate calculation) + if (!product.inStock && existing.inStock) { + pendingEvents.push({ + type: 'out_of_stock', + url: product.url, + name: product.name, + price: product.price, + lastInStock: existing.lastInStock, // When it came in stock + timestamp: now // When it sold out + }); + } + + // Track price changes + if (product.price && existing.price && product.price !== existing.price) { + pendingEvents.push({ + type: 'price_change', + url: product.url, + name: product.name, + oldPrice: existing.price, + newPrice: product.price, + timestamp: now + }); } // Update @@ -393,7 +541,69 @@ async function sendDiscordNotification(product, alertType) { } } -// Listen for messages from popup +// Track pending events to sync (restocks, new drops) +let pendingEvents = []; + +// Sync data to local dashboard +async function syncToDashboard() { + if (!config.syncToDashboard || !config.dashboardUrl) { + return { success: false, reason: 'sync disabled' }; + } + + // Build products array with all tracking data + const productsToSync = Object.values(knownProducts).map(p => ({ + ...p, + site: 'pokemoncenter', + sku: p.productId || extractSkuFromUrl(p.url) + })); + + const syncData = { + skus: Array.from(knownSkus), + products: productsToSync, + apiStats: apiStats, + events: pendingEvents // Include pending events (restocks, new drops) + }; + + try { + const response = await fetch(`${config.dashboardUrl}/api/extension/sync`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(syncData) + }); + + if (response.ok) { + const result = await response.json(); + console.log(`[Dashboard Sync] Success - ${result.total_skus} SKUs, ${result.total_products} products`); + // Clear pending events after successful sync + pendingEvents = []; + return { success: true, ...result }; + } else { + console.warn(`[Dashboard Sync] Failed: ${response.status}`); + return { success: false, status: response.status }; + } + } catch (error) { + // Dashboard might not be running - silently fail + console.debug(`[Dashboard Sync] Dashboard not available: ${error.message}`); + return { success: false, error: error.message }; + } +} + +// Helper to extract SKU from Pokemon Center URL +function extractSkuFromUrl(url) { + if (!url) return null; + const match = url.match(/\/product\/([^\/\?]+)/); + return match ? match[1] : null; +} + +// Auto-sync to dashboard periodically (every 5 minutes) +setInterval(() => { + if (config.syncToDashboard) { + syncToDashboard(); + } +}, 5 * 60 * 1000); + + +// Listen for messages from popup and content scripts chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { if (message.type === "getConfig") { sendResponse(config); @@ -407,12 +617,27 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { } else if (message.type === "getStats") { sendResponse({ totalProducts: Object.keys(knownProducts).length, - enabled: config.enabled + enabled: config.enabled, + apiStats: apiStats, + knownSkuCount: knownSkus.size }); } else if (message.type === "clearProducts") { knownProducts = {}; saveProducts(); sendResponse({ success: true }); + } else if (message.type === "clearSkus") { + knownSkus = new Set(); + apiStats.totalSkusTracked = 0; + saveKnownSkus(); + sendResponse({ success: true }); + } else if (message.type === "apiData") { + // Handle API data from content script interceptor + handleApiData(message.data); + sendResponse({ success: true }); + } else if (message.type === "syncToDashboard") { + // Manual sync to local dashboard + syncToDashboard().then(result => sendResponse(result)); + return true; // Keep channel open for async response } return true; }); @@ -420,7 +645,9 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { // Run initial check after a short delay setTimeout(() => { loadConfig().then(() => { - loadProducts().then(() => { + loadProducts(); + loadKnownSkus().then(() => { + console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center"); if (config.enabled) { console.log(`Check interval: ${config.checkIntervalSeconds} seconds`); runStockCheck(); diff --git a/chrome-extension/content.js b/chrome-extension/content.js index 276f4ee..f2b9415 100644 --- a/chrome-extension/content.js +++ b/chrome-extension/content.js @@ -1,9 +1,32 @@ // Pokemon Stock Monitor - Content Script -// Runs on PokemonCenter pages to extract product data +// Runs on PokemonCenter pages to extract product data and intercept APIs (function() { console.log("[Pokemon Monitor] Content script loaded on:", window.location.href); + // Inject the API interceptor into the page context + function injectApiInterceptor() { + const script = document.createElement('script'); + script.src = chrome.runtime.getURL('api-interceptor.js'); + script.onload = () => script.remove(); + (document.head || document.documentElement).appendChild(script); + } + + // Listen for messages from the injected interceptor + window.addEventListener('message', (event) => { + if (event.source !== window) return; + if (event.data?.source !== 'pokemon-api-interceptor') return; + + // Forward API data to background script + chrome.runtime.sendMessage({ + type: 'apiData', + data: event.data.data + }).catch(() => {}); + }); + + // Inject the interceptor + injectApiInterceptor(); + // Listen for messages from background script chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { if (message.type === "extractProducts") { diff --git a/chrome-extension/manifest.json b/chrome-extension/manifest.json index e0f03e8..5c75ae5 100644 --- a/chrome-extension/manifest.json +++ b/chrome-extension/manifest.json @@ -1,8 +1,8 @@ { "manifest_version": 3, "name": "Pokemon Stock Monitor", - "version": "1.0.1", - "description": "Monitors PokemonCenter for restocks and new drops, sends Discord notifications", + "version": "1.1.0", + "description": "Monitors PokemonCenter for restocks and new drops via API interception, sends Discord notifications", "permissions": [ "alarms", "storage", @@ -24,7 +24,13 @@ { "matches": ["https://www.pokemoncenter.com/*"], "js": ["content.js"], - "run_at": "document_idle" + "run_at": "document_start" + } + ], + "web_accessible_resources": [ + { + "resources": ["api-interceptor.js"], + "matches": ["https://www.pokemoncenter.com/*"] } ] } diff --git a/chrome-extension/popup.html b/chrome-extension/popup.html index 150c93a..4a31f5e 100644 --- a/chrome-extension/popup.html +++ b/chrome-extension/popup.html @@ -242,10 +242,24 @@ +
+ Sync to Dashboard + +
+ + +
+ + + Local dashboard for viewing data
+
Settings saved!
diff --git a/chrome-extension/popup.js b/chrome-extension/popup.js index 47eed7f..a73d3aa 100644 --- a/chrome-extension/popup.js +++ b/chrome-extension/popup.js @@ -13,6 +13,8 @@ document.addEventListener("DOMContentLoaded", async () => { document.getElementById("enabled").checked = config.enabled !== false; document.getElementById("notifyNew").checked = config.notifyNewProducts !== false; document.getElementById("notifyRestock").checked = config.notifyRestocks !== false; + document.getElementById("syncDashboard").checked = config.syncToDashboard !== false; + document.getElementById("dashboardUrl").value = config.dashboardUrl || "http://localhost:5000"; // Update stats document.getElementById("productCount").textContent = stats.totalProducts || 0; @@ -30,7 +32,9 @@ document.addEventListener("DOMContentLoaded", async () => { checkIntervalSeconds: parseInt(document.getElementById("interval").value) || 15, enabled: document.getElementById("enabled").checked, notifyNewProducts: document.getElementById("notifyNew").checked, - notifyRestocks: document.getElementById("notifyRestock").checked + notifyRestocks: document.getElementById("notifyRestock").checked, + syncToDashboard: document.getElementById("syncDashboard").checked, + dashboardUrl: document.getElementById("dashboardUrl").value.trim() || "http://localhost:5000" }; await chrome.runtime.sendMessage({ type: "saveConfig", config: newConfig }); @@ -52,6 +56,16 @@ document.addEventListener("DOMContentLoaded", async () => { showSaved("History cleared!"); } }); + + // Sync to Dashboard button + document.getElementById("syncBtn").addEventListener("click", async () => { + const result = await chrome.runtime.sendMessage({ type: "syncToDashboard" }); + if (result.success) { + showSaved(`Synced! ${result.total_skus || 0} SKUs, ${result.total_products || 0} products`); + } else { + showSaved("Sync failed - is dashboard running?"); + } + }); }); function updateStatus(enabled) { diff --git a/config.py b/config.py index a201149..f340cc4 100644 --- a/config.py +++ b/config.py @@ -5,14 +5,38 @@ Update DISCORD_WEBHOOK_URL with your actual webhook URL # Discord webhook URL - GET THIS FROM YOUR DISCORD SERVER # Server Settings -> Integrations -> Webhooks -> New Webhook -> Copy Webhook URL -DISCORD_WEBHOOK_URL = "YOUR_WEBHOOK_URL_HERE" +DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1471927173353963570/8Xh4Y_D8zRDi5MAP6ZlX0rf2Fc5wEtRKCup74kBZltk2qaatxArFp-yrk7ZWh5EOZbHp" + +# Enable/disable Discord notifications +NOTIFICATIONS_ENABLED = True # Set to False to disable all Discord alerts +NEW_DROP_NOTIFICATIONS = True # Set to False to silence new drop notifications +SKIP_DUPLICATE_SKUS = True # Skip notifications for products we've already notified about + +# Discord Bot Token (for interactive bot commands) +# Create a bot at https://discord.com/developers/applications +# Required for !setlocation, !stores, !stock commands +DISCORD_BOT_TOKEN = "MTQ4NjQ2ODM2NDkyMTczNzM3OQ.Gxl8pq.wIvK2eVbp1G0SJhW4XTYD3zopR1gohaNv5sxlQ" # How often to check each site (in seconds) CHECK_INTERVAL_SECONDS = 60 +# Pagination settings - how many pages to scrape per site +# Set to 1 for fastest checks, higher for more thorough scraping +MAX_PAGES = { + "target": 1, # Each page has ~24 products + "bestbuy": 1, + "gamestop": 1, + "walmart": 1, + "pokemoncenter": 1, +} + +# Sort by newest - ensures new drops appear first +SORT_BY_NEWEST = True + # Which sites to monitor # Note: PokemonCenter has strong bot protection (Imperva) - may need manual workarounds -SITES_ENABLED = ["target"] # Options: "pokemoncenter", "target", "walmart", "bestbuy" +# Note: Walmart has aggressive bot protection (PerimeterX) - may need stealth mode +SITES_ENABLED = ["target", "bestbuy"] # Options: "pokemoncenter", "target", "gamestop", "bestbuy" (disabled - needs rework), "walmart" # PokemonCenter URLs to monitor POKEMON_CENTER_URLS = [ @@ -24,6 +48,21 @@ TARGET_URLS = [ "https://www.target.com/s?searchTerm=pokemon+tcg", ] +# GameStop URLs to monitor +GAMESTOP_URLS = [ + "https://www.gamestop.com/search/?q=pokemon+tcg&lang=en_US", +] + +# Best Buy URLs to monitor +BESTBUY_URLS = [ + "https://www.bestbuy.com/site/searchpage.jsp?st=pokemon+tcg", +] + +# Walmart URLs to monitor +WALMART_URLS = [ + "https://www.walmart.com/search?q=pokemon+tcg", +] + # Keyword filtering (disabled by default) KEYWORD_FILTER_ENABLED = False KEYWORDS = [ @@ -46,10 +85,77 @@ SLOW_MO = 0 # Milliseconds to slow down browser actions (for debugging) # Leave as None to use a fresh browser profile # Set to None to use a fresh profile, or a path to use an existing profile # Note: Chrome must be closed when using an existing profile -CHROME_USER_DATA_DIR = None +CHROME_USER_DATA_DIR = None # Disabled - causes issues with Playwright persistent context # Use real Chrome instead of Playwright's Chromium (helps with bot detection) USE_REAL_CHROME = True +# ============================================================================= +# STEALTH MODE SETTINGS (for Pokemon Center and other protected sites) +# ============================================================================= + +# Use undetected-chromedriver instead of Playwright (better for Imperva/Cloudflare) +USE_STEALTH_BROWSER = True + +# Stealth browser settings +STEALTH_HEADLESS = False # Keep False - headless is more detectable +STEALTH_SESSION_NAME = "pokemoncenter" # Name for cookie/session persistence + +# Human-like behavior settings +MIN_CHECK_INTERVAL = 180 # Minimum seconds between checks for protected sites +MAX_CHECK_INTERVAL = 300 # Maximum seconds (will randomize between min/max) +HUMAN_DELAY_MIN = 2.0 # Minimum delay between actions (seconds) +HUMAN_DELAY_MAX = 5.0 # Maximum delay between actions (seconds) + +# Proxy settings (optional but recommended for heavy use) +USE_PROXIES = False # Set to True to enable proxy rotation +# Configure proxies in proxies.json file + +# Auto-solve CAPTCHA settings +CAPTCHA_WAIT_TIMEOUT = 120 # Seconds to wait for manual CAPTCHA solve +PAUSE_ON_CAPTCHA = True # Pause monitoring when CAPTCHA detected (requires manual solve) + # Logging LOG_LEVEL = "INFO" + +# Dashboard settings +DASHBOARD_HOST = "0.0.0.0" # Listen on all interfaces +DASHBOARD_PORT = 5000 +DASHBOARD_DEBUG = False # Set to True for development + +# Database settings +DATABASE_PATH = None # None = use default (stats.db in project root) + +# ============================================================================= +# NEWS AGGREGATION SETTINGS +# ============================================================================= + +# News sources configuration +NEWS_CONFIG = { + 'twitter': { + 'enabled': False, # Enable when you have API access + 'bearer_token': 'YOUR_TWITTER_BEARER_TOKEN', # Get from https://developer.twitter.com + 'accounts': [ + 'pokepullzhq', # Poke Pullz + 'PokemonRestocks', # Pokemon TCG Restocks & News + 'PokemonDealsTCG', # Pokemon Deals, Alerts & News! + 'PokeNotifyX' # PokeNotify + ], + 'fetch_interval': 300 # 5 minutes (free tier: 1500 tweets/month) + }, + 'manual_sources': { + 'enabled': True, + 'sources': ['Poke Pullz Discord', 'PokePings Discord', 'Other'] + }, + 'pokemon_official': { + 'enabled': True, + 'url': 'https://www.pokemon.com/us/pokemon-tcg-news/', + 'fetch_interval': 3600 # 1 hour (official news updates less frequently) + } +} + +# Auto-fetch news on dashboard start +NEWS_AUTO_FETCH = True + +# Clean up old news articles after this many days +NEWS_RETENTION_DAYS = 30 diff --git a/dashboard/__init__.py b/dashboard/__init__.py new file mode 100644 index 0000000..33c608a --- /dev/null +++ b/dashboard/__init__.py @@ -0,0 +1,4 @@ +# Dashboard module +from .app import run_dashboard + +__all__ = ['run_dashboard'] diff --git a/dashboard/api.py b/dashboard/api.py new file mode 100644 index 0000000..1e87daf --- /dev/null +++ b/dashboard/api.py @@ -0,0 +1,1082 @@ +""" +REST API endpoints for Pokemon Stock Monitor dashboard. +""" + +import sys +import re +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from flask import Blueprint, jsonify, request +from src.database import get_database +from src.favorites import get_favorites_manager +from src.scraper_state import scraper_state + +api_bp = Blueprint('api', __name__) + + +def clean_product_name(name: str) -> str: + """Fix common encoding issues in product names""" + if not name: + return name + + # Fix Pokémon encoding issues + name = re.sub(r'Pok[éÃ\u00c3\u00a9]+mon', 'Pokémon', name, flags=re.IGNORECASE) + name = name.replace('Pokémon', 'Pokémon') + name = name.replace('Pokﾃｩmon', 'Pokémon') + + # Fix dashes + name = name.replace('â€"', '–') + name = name.replace('â€"', '-') + + # Fix quotes and apostrophes + name = name.replace('’', "'") + name = name.replace('‘', "'") + name = name.replace('“', '"') + name = name.replace(''', "'") + name = name.replace(''', "'") + + # Fix accented characters + name = name.replace('é', 'é') + name = name.replace('è', 'è') + name = name.replace('à ', 'à') + + # Fix trademark + name = name.replace('â„¢', '™') + name = name.replace('®', '®') + name = name.replace('Â', '') + + return name.strip() + + +def clean_product_dict(product: dict) -> dict: + """Clean encoding in a product dictionary""" + if product and 'name' in product: + product['name'] = clean_product_name(product['name']) + return product + + +# ==================== Stats Endpoints ==================== + +@api_bp.route('/stats') +def get_stats(): + """Get dashboard stats summary""" + db = get_database() + stats = db.get_dashboard_stats() + return jsonify(stats) + + +@api_bp.route('/health') +def get_health(): + """Get monitor health status""" + db = get_database() + last_check = db.get_last_check() + return jsonify({ + 'status': 'ok', + 'last_check': last_check + }) + + +# ==================== Products Endpoints ==================== + +@api_bp.route('/products') +def get_products(): + """Get products with optional filters""" + db = get_database() + + # Parse query params + site = request.args.get('site') + category = request.args.get('category') + in_stock = request.args.get('in_stock') + favorites_only = request.args.get('favorites_only', 'false').lower() == 'true' + event_type = request.args.get('event_type') # 'new_drop' or 'restock' + period = request.args.get('period') # 'today' or 'week' + limit = int(request.args.get('limit', 100)) + offset = int(request.args.get('offset', 0)) + + # Convert in_stock to bool if provided + if in_stock is not None: + in_stock = in_stock.lower() == 'true' + + products = db.get_products( + site=site, + category=category, + in_stock=in_stock, + favorites_only=favorites_only, + event_type=event_type, + period=period, + limit=limit, + offset=offset + ) + + # Clean names and add favorite status to each product + favorites = get_favorites_manager() + for product in products: + clean_product_dict(product) + fav = favorites.check_product_priority( + url=product['url'], + name=product['name'], + category=product.get('category') + ) + product['is_favorite'] = fav is not None + product['favorite_id'] = fav['id'] if fav else None + + return jsonify({ + 'products': products, + 'total': db.get_product_count(site=site, in_stock=in_stock) + }) + + +@api_bp.route('/products/') +def get_product(product_id): + """Get single product details""" + db = get_database() + product = db.get_product(product_id) + + if not product: + return jsonify({'error': 'Product not found'}), 404 + + clean_product_dict(product) + return jsonify(product) + + +@api_bp.route('/products//history') +def get_product_history(product_id): + """Get price and stock history for a product""" + db = get_database() + days = int(request.args.get('days', 30)) + + price_history = db.get_price_history(product_id, days=days) + + # Get stock events for this product + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT event_type, recorded_at FROM stock_events + WHERE product_id = ? AND recorded_at >= datetime('now', ?) + ORDER BY recorded_at ASC + """, (product_id, f"-{days} days")) + stock_events = [dict(row) for row in cursor.fetchall()] + + return jsonify({ + 'price_history': price_history, + 'stock_events': stock_events + }) + + +@api_bp.route('/products//timeline') +def get_product_timeline(product_id): + """Get complete timeline of all events for a product""" + db = get_database() + + product = db.get_product(product_id) + if not product: + return jsonify({'error': 'Product not found'}), 404 + + with db.get_connection() as conn: + cursor = conn.cursor() + + # Get all stock events + cursor.execute(""" + SELECT 'stock' as source, event_type as type, recorded_at as timestamp + FROM stock_events WHERE product_id = ? + UNION ALL + SELECT 'price' as source, 'price_change' as type, recorded_at as timestamp + FROM price_history WHERE product_id = ? + ORDER BY timestamp DESC + """, (product_id, product_id)) + events = [dict(row) for row in cursor.fetchall()] + + # Get price at each point + cursor.execute(""" + SELECT price, recorded_at FROM price_history + WHERE product_id = ? ORDER BY recorded_at ASC + """, (product_id,)) + prices = [dict(row) for row in cursor.fetchall()] + + # Calculate stats + stats = { + 'total_restocks': sum(1 for e in events if e['type'] == 'restock'), + 'total_sellouts': sum(1 for e in events if e['type'] == 'out_of_stock'), + 'price_changes': sum(1 for e in events if e['type'] == 'price_change'), + 'first_seen': product.get('first_seen'), + 'last_seen': product.get('last_seen') + } + + return jsonify({ + 'product': product, + 'timeline': events, + 'price_history': prices, + 'stats': stats + }) + + +# ==================== Events Endpoints ==================== + +@api_bp.route('/events') +def get_events(): + """Get recent events feed""" + db = get_database() + limit = int(request.args.get('limit', 20)) + event_types = request.args.getlist('type') + + events = db.get_recent_events( + limit=limit, + event_types=event_types if event_types else None + ) + + # Clean product names in events + for event in events: + clean_product_dict(event) + + return jsonify({'events': events}) + + +# ==================== Analytics Endpoints ==================== + +@api_bp.route('/analytics/drops') +def get_drop_timing(): + """Get drop timing analytics""" + db = get_database() + days = int(request.args.get('days', 30)) + stats = db.get_drop_timing_stats(days=days) + return jsonify({'drop_timing': stats}) + + +@api_bp.route('/analytics/stock') +def get_stock_duration(): + """Get stock duration analytics""" + db = get_database() + stats = db.get_stock_duration_stats() + return jsonify({'stock_duration': stats}) + + +@api_bp.route('/analytics/sites') +def get_site_stats(): + """Get per-site statistics""" + db = get_database() + stats = db.get_site_stats() + return jsonify({'sites': stats}) + + +@api_bp.route('/analytics/checks') +def get_check_history(): + """Get check history""" + db = get_database() + site = request.args.get('site') + limit = int(request.args.get('limit', 100)) + + history = db.get_check_history(site=site, limit=limit) + return jsonify({'checks': history}) + + +@api_bp.route('/analytics/selling-rates') +def get_selling_rates(): + """Get detailed selling rate analytics per product""" + db = get_database() + limit = int(request.args.get('limit', 50)) + category = request.args.get('category') + + with db.get_connection() as conn: + cursor = conn.cursor() + # Get products with their sell-through times + query = """ + SELECT + p.id, p.name, p.url, p.site, p.category, p.image_url, + COUNT(CASE WHEN e.event_type = 'restock' THEN 1 END) as restock_count, + COUNT(CASE WHEN e.event_type = 'out_of_stock' THEN 1 END) as sellout_count, + COUNT(CASE WHEN e.event_type = 'new_drop' THEN 1 END) as drop_count + FROM products p + LEFT JOIN stock_events e ON p.id = e.product_id + WHERE 1=1 + """ + params = [] + + if category: + query += " AND p.category = ?" + params.append(category) + + query += """ + GROUP BY p.id + HAVING sellout_count > 0 OR restock_count > 0 + ORDER BY (restock_count + sellout_count) DESC + LIMIT ? + """ + params.append(limit) + + cursor.execute(query, params) + products = [dict(row) for row in cursor.fetchall()] + + # For each product, calculate average time to sell out + for product in products: + cursor.execute(""" + SELECT + in_event.recorded_at as in_stock_time, + out_event.recorded_at as out_stock_time, + CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER) as minutes_in_stock + FROM stock_events in_event + JOIN stock_events out_event ON in_event.product_id = out_event.product_id + AND out_event.event_type = 'out_of_stock' + AND out_event.recorded_at > in_event.recorded_at + AND NOT EXISTS ( + SELECT 1 FROM stock_events middle + WHERE middle.product_id = in_event.product_id + AND middle.event_type IN ('restock', 'new_drop', 'out_of_stock') + AND middle.recorded_at > in_event.recorded_at + AND middle.recorded_at < out_event.recorded_at + ) + WHERE in_event.product_id = ? + AND in_event.event_type IN ('restock', 'new_drop') + ORDER BY in_event.recorded_at DESC + LIMIT 10 + """, (product['id'],)) + + sellthrough_times = [dict(row) for row in cursor.fetchall()] + product['sellthrough_history'] = sellthrough_times + + if sellthrough_times: + avg_minutes = sum(s['minutes_in_stock'] for s in sellthrough_times) / len(sellthrough_times) + product['avg_minutes_to_sellout'] = round(avg_minutes, 1) + else: + product['avg_minutes_to_sellout'] = None + + return jsonify({'products': products}) + + +# ==================== Favorites Endpoints ==================== + +@api_bp.route('/favorites', methods=['GET']) +def get_favorites(): + """Get all favorites""" + favorites = get_favorites_manager() + fav_type = request.args.get('type') + + if fav_type: + favs = favorites.db.get_favorites(fav_type=fav_type) + else: + favs = favorites.get_all_favorites() + + return jsonify({'favorites': favs}) + + +@api_bp.route('/favorites', methods=['POST']) +def add_favorite(): + """Add a new favorite""" + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + fav_type = data.get('type') + value = data.get('value') + + if not fav_type or not value: + return jsonify({'error': 'type and value are required'}), 400 + + favorites = get_favorites_manager() + + if fav_type == 'product': + fav_id = favorites.add_product_favorite( + url=value, + display_name=data.get('display_name'), + priority=data.get('priority', 'high'), + notify_discord=data.get('notify_discord', True), + notify_sound=data.get('notify_sound', False), + custom_webhook=data.get('custom_webhook') + ) + elif fav_type == 'category': + fav_id = favorites.add_category_favorite( + category=value, + display_name=data.get('display_name'), + priority=data.get('priority', 'high'), + notify_discord=data.get('notify_discord', True), + notify_sound=data.get('notify_sound', False), + custom_webhook=data.get('custom_webhook') + ) + else: + return jsonify({'error': 'Invalid type. Must be "product" or "category"'}), 400 + + return jsonify({'id': fav_id, 'success': True}), 201 + + +@api_bp.route('/favorites/', methods=['PUT']) +def update_favorite(favorite_id): + """Update a favorite""" + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + favorites = get_favorites_manager() + favorites.update_favorite(favorite_id, **data) + + return jsonify({'success': True}) + + +@api_bp.route('/favorites/', methods=['DELETE']) +def delete_favorite(favorite_id): + """Delete a favorite""" + favorites = get_favorites_manager() + favorites.remove_favorite(favorite_id) + return jsonify({'success': True}) + + +@api_bp.route('/favorites/suggestions') +def get_favorite_suggestions(): + """Get suggested categories for favorites""" + favorites = get_favorites_manager() + suggestions = favorites.get_suggested_categories() + return jsonify({'suggestions': suggestions}) + + +# ==================== Scraper Control Endpoints ==================== + +@api_bp.route('/scrapers/state') +def get_scraper_state(): + """Get current scraper state""" + state = scraper_state.get_state() + return jsonify(state) + + +@api_bp.route('/scrapers//toggle', methods=['POST']) +def toggle_scraper(scraper): + """Enable or disable a scraper""" + data = request.get_json() or {} + enabled = data.get('enabled') + + if enabled is None: + # Toggle current state + current = scraper_state.get_state() + if scraper not in current['scrapers']: + return jsonify({'error': 'Unknown scraper'}), 404 + enabled = not current['scrapers'][scraper]['enabled'] + + success = scraper_state.set_scraper_enabled(scraper, enabled) + + if success: + return jsonify({'success': True, 'enabled': enabled}) + return jsonify({'error': 'Failed to update scraper state'}), 400 + + +@api_bp.route('/scrapers/interval', methods=['POST']) +def set_check_interval(): + """Set the check interval""" + data = request.get_json() or {} + interval = data.get('interval') + + if not interval or not isinstance(interval, int): + return jsonify({'error': 'interval (int) required'}), 400 + + success = scraper_state.set_check_interval(interval) + + if success: + return jsonify({'success': True, 'interval': interval}) + return jsonify({'error': 'Interval must be at least 10 seconds'}), 400 + + +@api_bp.route('/scrapers/start', methods=['POST']) +def start_monitor(): + """Start the stock monitor""" + result = scraper_state.start_monitor() + if result['success']: + return jsonify(result) + return jsonify(result), 400 + + +@api_bp.route('/scrapers/stop', methods=['POST']) +def stop_monitor(): + """Stop the stock monitor""" + result = scraper_state.stop_monitor() + if result['success']: + return jsonify(result) + return jsonify(result), 400 + + +# ==================== News Endpoints ==================== + +@api_bp.route('/news') +def get_news(): + """Get news articles with optional filters""" + db = get_database() + + # Parse query params + source = request.args.get('source') + sentiment = request.args.get('sentiment') + drop_related = request.args.get('drop_related') + limit = int(request.args.get('limit', 50)) + offset = int(request.args.get('offset', 0)) + + # Convert drop_related to bool if provided + if drop_related is not None: + drop_related = drop_related.lower() == 'true' + + articles = db.get_news_articles( + source=source, + sentiment=sentiment, + drop_related=drop_related, + limit=limit, + offset=offset + ) + + return jsonify({ + 'articles': articles, + 'total': len(articles) # For now, just return count of fetched + }) + + +@api_bp.route('/news/stats') +def get_news_stats(): + """Get news statistics""" + db = get_database() + stats = db.get_news_stats() + return jsonify(stats) + + +@api_bp.route('/news/correlation') +def get_news_correlation(): + """Get news articles correlated with recent drops""" + db = get_database() + hours = int(request.args.get('hours', 24)) + articles = db.get_correlated_news(hours=hours) + return jsonify({'articles': articles}) + + +@api_bp.route('/news/refresh', methods=['POST']) +def refresh_news(): + """Manually trigger news fetch from all sources""" + from src.news.pokemon_fetcher import fetch_pokemon_news + from src.news.sentiment import analyze_article + + db = get_database() + fetched = 0 + errors = [] + + # Fetch Pokemon.com news + try: + articles = fetch_pokemon_news(limit=20) + for article in articles: + db.add_news_article( + source=article.source, + source_account=article.source_account, + external_id=article.external_id, + title=article.title, + content=article.content, + url=article.url, + author=article.author, + image_url=article.image_url, + published_at=article.published_at.isoformat() if article.published_at else None, + sentiment_score=article.sentiment_score, + sentiment_label=article.sentiment_label, + keywords=article.keywords, + is_drop_related=article.is_drop_related, + is_restock_related=article.is_restock_related + ) + fetched += 1 + except Exception as e: + errors.append(f"Pokemon.com: {str(e)}") + + return jsonify({ + 'success': True, + 'fetched': fetched, + 'errors': errors + }) + + +@api_bp.route('/news/manual', methods=['POST']) +def add_manual_news(): + """Add manually pasted news (Discord messages, etc.)""" + from src.news.sentiment import analyze_article + from datetime import datetime + + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + content = data.get('content') + if not content: + return jsonify({'error': 'content is required'}), 400 + + source_account = data.get('source_account', 'Manual Entry') + url = data.get('url') + title = data.get('title') + + # Analyze sentiment + analysis = analyze_article(content, title) + + db = get_database() + article_id = db.add_news_article( + source='discord_manual', + source_account=source_account, + external_id=f"manual_{datetime.now().timestamp()}", + title=title, + content=content, + url=url, + author=source_account, + published_at=datetime.now().isoformat(), + sentiment_score=analysis['sentiment_score'], + sentiment_label=analysis['sentiment_label'], + keywords=analysis['keywords'], + is_drop_related=analysis['is_drop_related'], + is_restock_related=analysis['is_restock_related'] + ) + + return jsonify({'id': article_id, 'success': True, 'analysis': analysis}), 201 + + +@api_bp.route('/news/', methods=['DELETE']) +def delete_news_article(article_id): + """Delete a news article""" + db = get_database() + + # Check if article exists + article = db.get_news_article(article_id) + if not article: + return jsonify({'error': 'Article not found'}), 404 + + # Delete from database + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM news_articles WHERE id = ?", (article_id,)) + + return jsonify({'success': True}) + + +# ==================== Chrome Extension Integration ==================== + +# In-memory storage for extension data (persisted to file) +import json +from datetime import datetime +from pathlib import Path + +EXTENSION_DATA_FILE = Path(__file__).parent.parent / "data" / "extension_data.json" + +def load_extension_data(): + """Load extension data from file""" + if EXTENSION_DATA_FILE.exists(): + try: + with open(EXTENSION_DATA_FILE, 'r') as f: + return json.load(f) + except: + pass + return { + 'skus': [], + 'products': [], + 'api_stats': {}, + 'last_sync': None + } + +def save_extension_data(data): + """Save extension data to file""" + EXTENSION_DATA_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(EXTENSION_DATA_FILE, 'w') as f: + json.dump(data, f, indent=2) + +extension_data = load_extension_data() + + +@api_bp.route('/extension/sync', methods=['POST']) +def sync_extension_data(): + """ + Receive data from Chrome extension. + The extension should POST its known SKUs, products, events, and stats. + """ + global extension_data + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + db = get_database() + new_skus = [] + events_processed = 0 + + # Merge SKUs (add new ones) + # NOTE: We only store SKUs for reference - we don't create products from bare SKUs + # because Pokemon Center URLs require a product slug that we don't have. + # Products are only created when we receive full product data with proper URLs. + if 'skus' in data: + existing_skus = set(extension_data.get('skus', [])) + new_skus = [s for s in data['skus'] if s not in existing_skus] + extension_data['skus'] = list(existing_skus | set(data['skus'])) + + # Merge products - update existing or add new + if 'products' in data: + existing_urls = {p['url']: p for p in extension_data.get('products', [])} + for product in data['products']: + url = product.get('url') + if not url: + continue + + # Update extension data store + if url in existing_urls: + # Update existing product data + existing_urls[url].update(product) + else: + extension_data['products'].append(product) + existing_urls[url] = product + + # Sync to main database + product_id = db.get_or_create_product( + url=url, + name=product.get('name', 'Unknown'), + site=product.get('site', 'pokemoncenter'), + product_id=product.get('sku') or product.get('productId'), + image_url=product.get('imageUrl'), + price=product.get('price'), + in_stock=product.get('inStock', True) + ) + + # Update price if changed + if product.get('price') and product_id: + db.update_product_price(product_id, product['price']) + + extension_data['products'] = list(existing_urls.values()) + + # Process events (restocks, new drops, price changes) + if 'events' in data: + for event in data['events']: + event_type = event.get('type') + url = event.get('url') + + if not event_type or not url: + continue + + product = db.get_product_by_url(url) + if not product: + # Create product if it doesn't exist + product_id = db.get_or_create_product( + url=url, + name=event.get('name', 'Unknown'), + site='pokemoncenter', + in_stock=event.get('inStock', True), + price=event.get('price') + ) + else: + product_id = product['id'] + + if product_id: + if event_type == 'restock': + db.record_stock_event(product_id, 'restock') + # Update stock status + db.update_product_stock(product_id, True) + events_processed += 1 + + elif event_type == 'new_drop': + db.record_stock_event(product_id, 'new_drop') + events_processed += 1 + + elif event_type == 'out_of_stock': + db.record_stock_event(product_id, 'out_of_stock') + # Update stock status + db.update_product_stock(product_id, False) + events_processed += 1 + + elif event_type == 'price_change': + new_price = event.get('newPrice') + if new_price: + db.update_product_price(product_id, new_price) + events_processed += 1 + + # Update API stats + if 'apiStats' in data: + extension_data['api_stats'] = data['apiStats'] + + extension_data['last_sync'] = datetime.now().isoformat() + save_extension_data(extension_data) + + return jsonify({ + 'success': True, + 'total_skus': len(extension_data['skus']), + 'total_products': len(extension_data['products']), + 'new_skus_added': len(new_skus), + 'events_processed': events_processed + }) + + +@api_bp.route('/extension/skus', methods=['GET']) +def get_extension_skus(): + """Get all SKUs tracked by the extension""" + return jsonify({ + 'skus': extension_data.get('skus', []), + 'total': len(extension_data.get('skus', [])) + }) + + +@api_bp.route('/extension/skus', methods=['POST']) +def add_extension_skus(): + """Add new SKUs from extension""" + global extension_data + data = request.get_json() + + if not data or 'skus' not in data: + return jsonify({'error': 'skus array required'}), 400 + + existing = set(extension_data.get('skus', [])) + new_skus = [s for s in data['skus'] if s not in existing] + + extension_data['skus'] = list(existing | set(data['skus'])) + extension_data['last_sync'] = datetime.now().isoformat() + save_extension_data(extension_data) + + return jsonify({ + 'success': True, + 'added': len(new_skus), + 'total': len(extension_data['skus']) + }) + + +@api_bp.route('/extension/products', methods=['GET']) +def get_extension_products(): + """Get all products tracked by the extension""" + return jsonify({ + 'products': extension_data.get('products', []), + 'total': len(extension_data.get('products', [])) + }) + + +@api_bp.route('/extension/stats') +def get_extension_stats(): + """Get extension statistics for dashboard""" + return jsonify({ + 'total_skus': len(extension_data.get('skus', [])), + 'total_products': len(extension_data.get('products', [])), + 'api_stats': extension_data.get('api_stats', {}), + 'last_sync': extension_data.get('last_sync'), + 'source': 'chrome_extension' + }) + + +@api_bp.route('/extension/broken-urls', methods=['GET']) +def get_broken_pokemoncenter_urls(): + """ + Find Pokemon Center products with broken URLs. + Broken URLs are those without a product slug (just /product/{sku}). + """ + db = get_database() + + with db.get_connection() as conn: + cursor = conn.cursor() + # Find pokemoncenter products where URL ends with just the SKU (no slug) + # Valid URLs look like: /product/10-10022-102/product-name-here + # Invalid URLs look like: /product/10-10050 (no trailing slug) + cursor.execute(""" + SELECT id, url, name, product_id, first_seen + FROM products + WHERE site = 'pokemoncenter' + AND url LIKE '%/product/%' + AND url NOT LIKE '%/product/%/%' + """) + broken = [dict(row) for row in cursor.fetchall()] + + return jsonify({ + 'broken_urls': broken, + 'count': len(broken) + }) + + +@api_bp.route('/extension/broken-urls/cleanup', methods=['POST']) +def cleanup_broken_pokemoncenter_urls(): + """ + Delete Pokemon Center products with broken URLs. + """ + db = get_database() + + with db.get_connection() as conn: + cursor = conn.cursor() + # Delete products with broken URLs + cursor.execute(""" + DELETE FROM products + WHERE site = 'pokemoncenter' + AND url LIKE '%/product/%' + AND url NOT LIKE '%/product/%/%' + """) + deleted = cursor.rowcount + + return jsonify({ + 'success': True, + 'deleted': deleted + }) + + +@api_bp.route('/extension/clear', methods=['POST']) +def clear_extension_data(): + """Clear all extension data""" + global extension_data + extension_data = { + 'skus': [], + 'products': [], + 'api_stats': {}, + 'last_sync': None + } + save_extension_data(extension_data) + return jsonify({'success': True}) + + +# ==================== Check Trigger ==================== + +@api_bp.route('/check', methods=['POST']) +def trigger_check(): + """Trigger a manual stock check (placeholder - needs integration)""" + # This would need to communicate with the main monitor process + # For now, just return a message + return jsonify({ + 'message': 'Manual check triggered. Check monitor logs for results.', + 'success': True + }) + + +# ==================== User Management Endpoints ==================== + +@api_bp.route('/users', methods=['GET']) +def get_users(): + """Get all users""" + db = get_database() + users = db.get_all_users() + return jsonify({'users': users}) + + +@api_bp.route('/users', methods=['POST']) +def create_user(): + """Create a new user""" + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + name = data.get('name') + if not name: + return jsonify({'error': 'name is required'}), 400 + + db = get_database() + + # Check if user already exists + existing = db.get_user_by_name(name) + if existing: + return jsonify({'error': 'User with this name already exists'}), 400 + + user_id = db.create_user( + name=name, + zip_code=data.get('zip_code'), + radius_miles=data.get('radius_miles', 25), + discord_webhook=data.get('discord_webhook'), + notify_enabled=data.get('notify_enabled', True) + ) + + return jsonify({'id': user_id, 'success': True}), 201 + + +@api_bp.route('/users/', methods=['GET']) +def get_user(user_id): + """Get a specific user""" + db = get_database() + user = db.get_user(user_id) + + if not user: + return jsonify({'error': 'User not found'}), 404 + + return jsonify(user) + + +@api_bp.route('/users/', methods=['PUT']) +def update_user(user_id): + """Update a user""" + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + db = get_database() + + # Check user exists + user = db.get_user(user_id) + if not user: + return jsonify({'error': 'User not found'}), 404 + + # If changing name, check for conflicts + if 'name' in data and data['name'] != user['name']: + existing = db.get_user_by_name(data['name']) + if existing: + return jsonify({'error': 'User with this name already exists'}), 400 + + success = db.update_user(user_id, **data) + return jsonify({'success': success}) + + +@api_bp.route('/users/', methods=['DELETE']) +def delete_user(user_id): + """Delete a user""" + db = get_database() + + user = db.get_user(user_id) + if not user: + return jsonify({'error': 'User not found'}), 404 + + success = db.delete_user(user_id) + return jsonify({'success': success}) + + +@api_bp.route('/users//location', methods=['PUT']) +def update_user_location(user_id): + """Update user's location settings""" + data = request.get_json() + + if not data: + return jsonify({'error': 'No data provided'}), 400 + + zip_code = data.get('zip_code') + if not zip_code: + return jsonify({'error': 'zip_code is required'}), 400 + + db = get_database() + + user = db.get_user(user_id) + if not user: + return jsonify({'error': 'User not found'}), 404 + + radius_miles = data.get('radius_miles', 25) + success = db.update_user_location(user_id, zip_code, radius_miles) + + return jsonify({'success': success}) + + +@api_bp.route('/users//stores', methods=['GET']) +def get_user_stores(user_id): + """Get nearby stores for a user based on their location""" + from src.store_locator import find_all_nearby_stores + + db = get_database() + user = db.get_user(user_id) + + if not user: + return jsonify({'error': 'User not found'}), 404 + + if not user.get('zip_code'): + return jsonify({'error': 'User has no location set'}), 400 + + stores = find_all_nearby_stores(user['zip_code'], user.get('radius_miles', 25)) + + # Convert Store objects to dictionaries + result = {} + for retailer, store_list in stores.items(): + result[retailer] = [ + { + 'name': s.name, + 'address': s.address, + 'city': s.city, + 'state': s.state, + 'zip_code': s.zip_code, + 'distance_miles': s.distance_miles, + 'phone': s.phone, + 'store_id': s.store_id + } + for s in store_list + ] + + return jsonify({ + 'user': user['name'], + 'location': {'zip_code': user['zip_code'], 'radius_miles': user.get('radius_miles', 25)}, + 'stores': result + }) diff --git a/dashboard/app.py b/dashboard/app.py new file mode 100644 index 0000000..7bfef02 --- /dev/null +++ b/dashboard/app.py @@ -0,0 +1,77 @@ +""" +Flask dashboard for Pokemon Stock Monitor. +Provides web UI for stats visualization and favorites management. +""" + +import os +import sys +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from flask import Flask, render_template, jsonify, request +from flask_cors import CORS + +from src.database import get_database +from src.favorites import get_favorites_manager +from .api import api_bp + +# Create Flask app +app = Flask(__name__) +CORS(app) + +# Register API blueprint +app.register_blueprint(api_bp, url_prefix='/api') + + +@app.route('/') +def dashboard(): + """Main dashboard page""" + return render_template('index.html') + + +@app.route('/products') +def products_page(): + """Products listing page""" + return render_template('index.html', page='products') + + +@app.route('/news') +def news_page(): + """News aggregation page""" + return render_template('index.html', page='news') + + +@app.route('/analytics') +def analytics_page(): + """Analytics page""" + return render_template('index.html', page='analytics') + + +@app.route('/favorites') +def favorites_page(): + """Favorites management page""" + return render_template('index.html', page='favorites') + + +@app.route('/settings') +def settings_page(): + """Settings page""" + return render_template('index.html', page='settings') + + +@app.route('/users') +def users_page(): + """Users management page""" + return render_template('index.html', page='users') + + +def run_dashboard(host: str = '0.0.0.0', port: int = 5000, debug: bool = False): + """Run the dashboard server""" + print(f"Starting Pokemon Stock Monitor Dashboard on http://{host}:{port}") + app.run(host=host, port=port, debug=debug, threaded=True) + + +if __name__ == '__main__': + run_dashboard(debug=True) diff --git a/dashboard/static/app.js b/dashboard/static/app.js new file mode 100644 index 0000000..7671ad5 --- /dev/null +++ b/dashboard/static/app.js @@ -0,0 +1,1280 @@ +// Pokemon Stock Monitor Dashboard - Frontend JavaScript + +const API_BASE = '/api'; + +// State +let currentPage = 'dashboard'; +let charts = {}; +let currentEventFilter = null; // { type: 'new_drop'|'restock', period: 'today'|'week' } + +// Initialize +document.addEventListener('DOMContentLoaded', () => { + initNavigation(); + loadDashboard(); + loadSuggestions(); + initEventListeners(); + + // Auto-refresh dashboard every 30 seconds + setInterval(() => { + if (currentPage === 'dashboard') { + loadDashboard(); + } + }, 30000); +}); + +// Navigation +function initNavigation() { + document.querySelectorAll('.nav-links a').forEach(link => { + link.addEventListener('click', (e) => { + e.preventDefault(); + const page = e.target.dataset.page; + navigateTo(page); + }); + }); +} + +function navigateTo(page) { + // Update active nav + document.querySelectorAll('.nav-links a').forEach(a => a.classList.remove('active')); + document.querySelector(`[data-page="${page}"]`).classList.add('active'); + + // Show page + document.querySelectorAll('.page').forEach(p => p.classList.remove('active')); + document.getElementById(`page-${page}`).classList.add('active'); + + currentPage = page; + + // Load page content + switch (page) { + case 'dashboard': + loadDashboard(); + break; + case 'products': + loadProducts(); + break; + case 'news': + loadNews(); + break; + case 'analytics': + loadAnalytics(); + break; + case 'favorites': + loadFavorites(); + break; + case 'control': + loadControlPanel(); + break; + case 'users': + loadUsers(); + break; + case 'settings': + loadSettings(); + break; + } +} + +// Event Listeners +function initEventListeners() { + // Product filters - clear event filter when user changes regular filters + ['filterSite', 'filterCategory', 'filterStock', 'filterFavorites'].forEach(id => { + const el = document.getElementById(id); + if (el) { + el.addEventListener('change', () => { + currentEventFilter = null; // Clear event filter when using regular filters + loadProducts(); + }); + } + }); + + // Add favorite button + document.getElementById('addFavoriteBtn')?.addEventListener('click', addFavorite); + + // Modal close + document.querySelector('.modal-close')?.addEventListener('click', closeModal); + document.getElementById('productModal')?.addEventListener('click', (e) => { + if (e.target.id === 'productModal') closeModal(); + }); +} + +// API Helpers +async function api(endpoint, options = {}) { + try { + const response = await fetch(`${API_BASE}${endpoint}`, { + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers + } + }); + return await response.json(); + } catch (error) { + console.error('API Error:', error); + return null; + } +} + +// Store display names and branding +const STORE_INFO = { + pokemoncenter: { + name: 'Pokemon Center', + logo: 'https://www.google.com/s2/favicons?domain=pokemoncenter.com&sz=128', + fallback: 'PC', + color: '#ffcb05', + bgColor: '#1d1d42' + }, + target: { + name: 'Target', + logo: 'https://www.google.com/s2/favicons?domain=target.com&sz=128', + fallback: '◎', + color: '#cc0000', + bgColor: '#ffffff' + }, + walmart: { + name: 'Walmart', + logo: 'https://www.google.com/s2/favicons?domain=walmart.com&sz=128', + fallback: '✱', + color: '#0071ce', + bgColor: '#ffffff' + }, + bestbuy: { + name: 'Best Buy', + logo: 'https://www.google.com/s2/favicons?domain=bestbuy.com&sz=128', + fallback: 'BB', + color: '#fff200', + bgColor: '#0046be' + }, + gamestop: { + name: 'GameStop', + logo: 'https://www.google.com/s2/favicons?domain=gamestop.com&sz=128', + fallback: 'GS', + color: '#ff0000', + bgColor: '#000000' + } +}; + +// All stores to show (even without data) +const ALL_STORES = ['pokemoncenter', 'target', 'gamestop', 'walmart', 'bestbuy']; + +// Dashboard +async function loadDashboard() { + const stats = await api('/stats'); + if (!stats) return; + + // Update stats cards + document.getElementById('newDropsToday').textContent = stats.new_drops_today || 0; + document.getElementById('restocksToday').textContent = stats.restocks_today || 0; + document.getElementById('newDropsWeek').textContent = stats.new_drops_week || 0; + document.getElementById('restocksWeek').textContent = stats.restocks_week || 0; + + // Update store cards + loadStoreCards(stats.sites || []); + + // Update status + const statusDot = document.getElementById('monitorStatus'); + const statusText = document.getElementById('statusText'); + if (stats.last_check) { + statusDot.classList.add('active'); + const lastCheck = new Date(stats.last_check.checked_at); + statusText.textContent = `Last check: ${formatTime(lastCheck)}`; + } else { + statusDot.classList.remove('active'); + statusText.textContent = 'No checks yet'; + } + + // Load activity feed + loadActivityFeed(); +} + +function loadStoreCards(sites) { + const container = document.getElementById('storeCards'); + + // Create a map of site data for quick lookup + const siteDataMap = {}; + (sites || []).forEach(s => { + siteDataMap[s.site] = s; + }); + + // Show all stores, even without data + container.innerHTML = ALL_STORES.map(storeKey => { + const info = STORE_INFO[storeKey]; + const data = siteDataMap[storeKey] || { total_products: 0, in_stock_count: 0 }; + const outOfStock = data.total_products - data.in_stock_count; + const hasData = data.total_products > 0; + + return ` +
+
+ ${info.name} + +
+
+
${info.name}
+
+ ${hasData ? ` + + ● ${data.in_stock_count} in stock + + + ○ ${outOfStock} sold out + + ` : ` + Not tracking yet + `} +
+
+
›
+
+ `; + }).join(''); +} + +function goToStoreProducts(store) { + // Navigate to products page and filter by store + navigateTo('products'); + document.getElementById('filterSite').value = store; + loadProducts(); +} + +async function loadActivityFeed() { + const data = await api('/events?limit=20'); + if (!data || !data.events) return; + + const feed = document.getElementById('activityFeed'); + if (data.events.length === 0) { + feed.innerHTML = '
No recent activity
'; + return; + } + + feed.innerHTML = data.events.map(event => { + const storeInfo = STORE_INFO[event.site] || { name: event.site, color: '#333', bgColor: '#666', logo: '', fallback: '?' }; + const badgeClass = event.event_type === 'restock' ? 'restock' : 'new-drop'; + const badgeText = event.event_type === 'restock' ? 'Restock' : 'New'; + const hasProductImage = event.image_url && !event.image_url.includes('data:') && event.image_url.length > 10; + + // Build store logo HTML with fallback + const storeLogoHtml = ` +
+ + +
+ `; + + return ` +
+ ${hasProductImage + ? `` + : storeLogoHtml + } +
+
+ ${cleanProductName(event.name)} + ${badgeText} +
+
${storeInfo.name} • ${event.current_price || 'N/A'}
+
+
${formatTime(new Date(event.recorded_at))}
+
+ `; + }).join(''); +} + +// Navigate to products filtered by event type +function goToFilteredProducts(eventType, period) { + currentEventFilter = { type: eventType, period: period }; + navigateTo('products'); +} + +// Clear event filter +function clearEventFilter() { + currentEventFilter = null; + loadProducts(); +} + +// Products +async function loadProducts() { + const site = document.getElementById('filterSite')?.value || ''; + const category = document.getElementById('filterCategory')?.value || ''; + const inStock = document.getElementById('filterStock')?.value || ''; + const favoritesOnly = document.getElementById('filterFavorites')?.checked || false; + + let url = '/products?limit=50'; + if (site) url += `&site=${site}`; + if (category) url += `&category=${encodeURIComponent(category)}`; + if (inStock) url += `&in_stock=${inStock}`; + if (favoritesOnly) url += '&favorites_only=true'; + + // Apply event type filter if set + if (currentEventFilter) { + url += `&event_type=${currentEventFilter.type}&period=${currentEventFilter.period}`; + } + + // Update filter badge visibility + updateFilterBadge(); + + const data = await api(url); + if (!data || !data.products) return; + + const grid = document.getElementById('productsGrid'); + + if (data.products.length === 0) { + grid.innerHTML = '
No products found
'; + return; + } + + grid.innerHTML = data.products.map(product => { + const storeInfo = STORE_INFO[product.site] || { name: product.site, color: '#333', bgColor: '#666', logo: '', fallback: '?' }; + const cleanName = cleanProductName(product.name); + const hasImage = product.image_url && !product.image_url.includes('data:') && product.image_url.length > 10; + + // Build store logo fallback HTML + const storeLogoHtml = ` +
+ + +
+ `; + + return ` +
+
+ ${hasImage + ? `` + : storeLogoHtml + } +
+
${cleanName}
+
+ ${product.current_price || 'N/A'} + + ${product.in_stock ? 'IN STOCK' : 'OUT OF STOCK'} + +
+
+ ${storeInfo.name} + +
+
+ `; + }).join(''); +} + +async function showProductDetails(productId) { + const product = await api(`/products/${productId}`); + const history = await api(`/products/${productId}/history`); + + if (!product) return; + + document.getElementById('modalProductName').textContent = cleanProductName(product.name); + const storeInfo = STORE_INFO[product.site] || { name: product.site, color: '#333', bgColor: '#666', icon: '?' }; + + document.getElementById('modalProductDetails').innerHTML = ` +

Store: ${storeInfo.name}

+

Category: ${product.category || 'Unknown'}

+

Price: ${product.current_price || 'N/A'}

+

Status: ${product.in_stock ? 'In Stock' : 'Out of Stock'}

+

First Seen: ${formatDate(product.first_seen)}

+

Last Seen: ${formatDate(product.last_seen)}

+

View on ${storeInfo.name} →

+ ${history && history.price_history && history.price_history.length > 0 ? ` +

Price History

+
    + ${history.price_history.map(p => `
  • ${p.price} - ${formatDate(p.recorded_at)}
    • `).join('')} +
+ ` : ''} + `; + + document.getElementById('productModal').classList.add('show'); +} + +function closeModal() { + document.getElementById('productModal').classList.remove('show'); +} + +// Analytics +async function loadAnalytics() { + await Promise.all([ + loadDropTimingChart(), + loadStockDurationChart(), + loadSiteChart() + ]); +} + +async function loadDropTimingChart() { + const data = await api('/analytics/drops'); + if (!data || !data.drop_timing) return; + + const ctx = document.getElementById('dropTimingChart'); + if (!ctx) return; + + // Destroy existing chart + if (charts.dropTiming) charts.dropTiming.destroy(); + + // Fill in missing hours + const hourlyData = new Array(24).fill(0); + data.drop_timing.forEach(d => { + hourlyData[parseInt(d.hour)] = d.count; + }); + + charts.dropTiming = new Chart(ctx, { + type: 'bar', + data: { + labels: Array.from({length: 24}, (_, i) => `${i}:00`), + datasets: [{ + label: 'Drops/Restocks', + data: hourlyData, + backgroundColor: 'rgba(255, 203, 5, 0.6)', + borderColor: 'rgba(255, 203, 5, 1)', + borderWidth: 1 + }] + }, + options: { + responsive: true, + plugins: { + legend: { display: false } + }, + scales: { + y: { beginAtZero: true, ticks: { color: '#aaa' } }, + x: { ticks: { color: '#aaa' } } + } + } + }); +} + +async function loadStockDurationChart() { + const data = await api('/analytics/stock'); + if (!data || !data.stock_duration) return; + + const ctx = document.getElementById('stockDurationChart'); + if (!ctx) return; + + if (charts.stockDuration) charts.stockDuration.destroy(); + + const labels = data.stock_duration.map(d => d.category || 'Unknown'); + const values = data.stock_duration.map(d => d.avg_minutes_in_stock || 0); + + charts.stockDuration = new Chart(ctx, { + type: 'bar', + data: { + labels: labels, + datasets: [{ + label: 'Avg Minutes In Stock', + data: values, + backgroundColor: 'rgba(0, 200, 83, 0.6)', + borderColor: 'rgba(0, 200, 83, 1)', + borderWidth: 1 + }] + }, + options: { + responsive: true, + indexAxis: 'y', + plugins: { + legend: { display: false } + }, + scales: { + x: { beginAtZero: true, ticks: { color: '#aaa' } }, + y: { ticks: { color: '#aaa' } } + } + } + }); +} + +async function loadSiteChart() { + const data = await api('/analytics/sites'); + if (!data || !data.sites) return; + + const ctx = document.getElementById('siteChart'); + if (!ctx) return; + + if (charts.site) charts.site.destroy(); + + charts.site = new Chart(ctx, { + type: 'doughnut', + data: { + labels: data.sites.map(s => s.site), + datasets: [{ + data: data.sites.map(s => s.total_products), + backgroundColor: [ + 'rgba(255, 203, 5, 0.8)', + 'rgba(0, 153, 255, 0.8)', + 'rgba(0, 200, 83, 0.8)', + 'rgba(198, 40, 40, 0.8)' + ] + }] + }, + options: { + responsive: true, + plugins: { + legend: { + position: 'bottom', + labels: { color: '#eee' } + } + } + } + }); +} + +// Favorites +async function loadFavorites() { + const data = await api('/favorites'); + if (!data || !data.favorites) return; + + const list = document.getElementById('favoritesList'); + + if (data.favorites.length === 0) { + list.innerHTML = '
No favorites yet. Add some above!
'; + return; + } + + list.innerHTML = data.favorites.map(fav => ` +
+
${fav.type === 'category' ? '📁' : '🔗'}
+
+
${escapeHtml(fav.display_name || fav.value)}
+
${fav.type === 'category' ? 'Category' : 'Product'}
+
+ ${fav.priority.toUpperCase()} + +
+ `).join(''); +} + +async function loadSuggestions() { + const data = await api('/favorites/suggestions'); + if (!data || !data.suggestions) return; + + const container = document.getElementById('suggestions'); + container.innerHTML = data.suggestions.slice(0, 10).map(s => ` + ${s} + `).join(''); +} + +async function addFavorite() { + const type = document.getElementById('favType').value; + const value = document.getElementById('favValue').value.trim(); + const priority = document.getElementById('favPriority').value; + + if (!value) { + alert('Please enter a value'); + return; + } + + await api('/favorites', { + method: 'POST', + body: JSON.stringify({ type, value, priority }) + }); + + document.getElementById('favValue').value = ''; + loadFavorites(); +} + +async function deleteFavorite(id) { + if (!confirm('Remove this favorite?')) return; + + await api(`/favorites/${id}`, { method: 'DELETE' }); + loadFavorites(); +} + +async function toggleFavorite(url, name, favoriteId) { + if (favoriteId) { + await api(`/favorites/${favoriteId}`, { method: 'DELETE' }); + } else { + await api('/favorites', { + method: 'POST', + body: JSON.stringify({ + type: 'product', + value: url, + display_name: name, + priority: 'high' + }) + }); + } + loadProducts(); +} + +// Utilities +function updateFilterBadge() { + const badge = document.getElementById('eventFilterBadge'); + if (!badge) return; + + if (currentEventFilter) { + const typeLabel = currentEventFilter.type === 'new_drop' ? 'New Drops' : 'Restocks'; + const periodLabel = currentEventFilter.period === 'today' ? 'Today' : '7 Days'; + badge.innerHTML = ` + ${typeLabel} - ${periodLabel} + × + `; + badge.style.display = 'flex'; + } else { + badge.style.display = 'none'; + } +} + +function formatTime(date) { + const now = new Date(); + const diff = (now - date) / 1000; + + if (diff < 60) return 'Just now'; + if (diff < 3600) return `${Math.floor(diff / 60)}m ago`; + if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`; + return date.toLocaleDateString(); +} + +function formatDate(dateStr) { + if (!dateStr) return 'N/A'; + return new Date(dateStr).toLocaleString(); +} + +function escapeHtml(text) { + if (!text) return ''; + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +function cleanProductName(name) { + if (!name) return ''; + + // Fix common UTF-8 encoding issues (mojibake) + return name + // Fix Pokémon - multiple encoding patterns + .replace(/Pok[éÃ\u00c3\u00a9]+mon/gi, 'Pokémon') + .replace(/Pokémon/gi, 'Pokémon') + .replace(/Pokémon/gi, 'Pokémon') + .replace(/Pokﾃｩmon/gi, 'Pokémon') + // Keep "Pokemon" as-is for non-encoded versions + // Fix dashes - â€" is UTF-8 em-dash mangled + .replace(/â€"/g, '–') + .replace(/â€"/g, '-') + .replace(/â€""/g, '–') + // Fix apostrophes and quotes + .replace(/’/g, "'") + .replace(/‘/g, "'") + .replace(/“/g, '"') + .replace(/â€[^a-zA-Z0-9]/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/"/g, '"') + // Fix accented characters + .replace(/é/g, 'é') + .replace(/è/g, 'è') + .replace(/à /g, 'à') + .replace(/â/g, 'â') + .replace(/î/g, 'î') + .replace(/ô/g, 'ô') + .replace(/û/g, 'û') + // Fix trademark symbols + .replace(/â„¢/g, '™') + .replace(/®/g, '®') + // Clean up any remaining garbled chars + .replace(/Â/g, '') + // Clean up any double spaces + .replace(/\s+/g, ' ') + .trim(); +} + +// ==================== Control Panel ==================== + +let controlPanelInterval = null; + +async function loadControlPanel() { + const state = await api('/scrapers/state'); + if (!state) return; + + // Update monitor status + updateMonitorStatus(state); + + // Update check interval input + document.getElementById('checkIntervalInput').value = state.check_interval || 60; + + // Render scrapers grid + renderScrapersGrid(state.scrapers); + + // Setup event listeners for control panel buttons + setupControlPanelListeners(); + + // Auto-refresh control panel every 5 seconds + if (controlPanelInterval) clearInterval(controlPanelInterval); + controlPanelInterval = setInterval(async () => { + if (currentPage === 'control') { + const newState = await api('/scrapers/state'); + if (newState) { + updateMonitorStatus(newState); + renderScrapersGrid(newState.scrapers); + } + } + }, 5000); +} + +function updateMonitorStatus(state) { + const badge = document.getElementById('monitorBadge'); + const statusText = document.getElementById('monitorStatusText'); + const lastCheck = document.getElementById('controlLastCheck'); + const startBtn = document.getElementById('startMonitorBtn'); + const stopBtn = document.getElementById('stopMonitorBtn'); + + if (state.monitor_running) { + badge.textContent = 'Running'; + badge.className = 'monitor-badge running'; + statusText.textContent = `Monitor is running (PID: ${state.monitor_pid})`; + startBtn.disabled = true; + stopBtn.disabled = false; + } else { + badge.textContent = 'Stopped'; + badge.className = 'monitor-badge stopped'; + statusText.textContent = 'Monitor is not running'; + startBtn.disabled = false; + stopBtn.disabled = true; + } + + if (state.last_check) { + lastCheck.textContent = formatTime(new Date(state.last_check)); + } else { + lastCheck.textContent = 'Never'; + } +} + +function renderScrapersGrid(scrapers) { + const grid = document.getElementById('scrapersGrid'); + + grid.innerHTML = Object.entries(scrapers).map(([key, data]) => { + const info = STORE_INFO[key] || { name: key, logo: '', fallback: key.substring(0, 2).toUpperCase(), bgColor: '#333', color: '#fff' }; + const enabledClass = data.enabled ? 'enabled' : 'disabled'; + const runningClass = data.running ? 'running' : ''; + + return ` +
+ +
+
${info.name}
+
+ ${data.running ? '● Running' : + data.enabled ? '● Enabled' : + '○ Disabled'} +
+ ${data.last_run ? `
Last: ${formatTime(new Date(data.last_run))}
` : ''} + ${data.last_error ? `
${data.last_error}
` : ''} +
+ +
+ `; + }).join(''); +} + +function setupControlPanelListeners() { + // Start/Stop buttons + document.getElementById('startMonitorBtn')?.addEventListener('click', startMonitor); + document.getElementById('stopMonitorBtn')?.addEventListener('click', stopMonitor); + + // Save interval button + document.getElementById('saveIntervalBtn')?.addEventListener('click', saveCheckInterval); + + // Quick actions + document.getElementById('runCheckNowBtn')?.addEventListener('click', runCheckNow); + document.getElementById('viewLogsBtn')?.addEventListener('click', viewLogs); +} + +async function toggleScraper(scraper, enabled) { + const result = await api(`/scrapers/${scraper}/toggle`, { + method: 'POST', + body: JSON.stringify({ enabled }) + }); + + if (result && result.success) { + // Refresh the control panel + loadControlPanel(); + } else { + alert('Failed to toggle scraper: ' + (result?.error || 'Unknown error')); + } +} + +async function startMonitor() { + const btn = document.getElementById('startMonitorBtn'); + btn.disabled = true; + btn.innerHTML = '⌛ Starting...'; + + const result = await api('/scrapers/start', { method: 'POST' }); + + if (result && result.success) { + loadControlPanel(); + } else { + alert('Failed to start monitor: ' + (result?.error || 'Unknown error')); + btn.disabled = false; + btn.innerHTML = '▶ Start Monitor'; + } +} + +async function stopMonitor() { + const btn = document.getElementById('stopMonitorBtn'); + btn.disabled = true; + btn.innerHTML = '⌛ Stopping...'; + + const result = await api('/scrapers/stop', { method: 'POST' }); + + if (result && result.success) { + loadControlPanel(); + } else { + alert('Failed to stop monitor: ' + (result?.error || 'Unknown error')); + btn.disabled = false; + btn.innerHTML = '■ Stop Monitor'; + } +} + +async function saveCheckInterval() { + const input = document.getElementById('checkIntervalInput'); + const interval = parseInt(input.value); + + if (isNaN(interval) || interval < 10) { + alert('Interval must be at least 10 seconds'); + return; + } + + const result = await api('/scrapers/interval', { + method: 'POST', + body: JSON.stringify({ interval }) + }); + + if (result && result.success) { + alert('Check interval saved!'); + } else { + alert('Failed to save interval: ' + (result?.error || 'Unknown error')); + } +} + +async function runCheckNow() { + const result = await api('/check', { method: 'POST' }); + if (result) { + alert(result.message || 'Check triggered'); + } +} + +function viewLogs() { + // Open monitor.log in a new window or show in modal + alert('View monitor.log file in the project directory for detailed logs.'); +} + +// ==================== News ==================== + +let newsInitialized = false; + +async function loadNews() { + // Initialize event listeners once + if (!newsInitialized) { + initNewsEventListeners(); + newsInitialized = true; + } + + // Load stats and articles + await Promise.all([ + loadNewsStats(), + loadNewsArticles() + ]); +} + +function initNewsEventListeners() { + // Filter changes + document.getElementById('filterNewsSource')?.addEventListener('change', loadNewsArticles); + document.getElementById('filterNewsSentiment')?.addEventListener('change', loadNewsArticles); + document.getElementById('filterDropRelated')?.addEventListener('change', loadNewsArticles); + + // Refresh button + document.getElementById('refreshNewsBtn')?.addEventListener('click', refreshNews); + + // Manual news input + document.getElementById('addManualNewsBtn')?.addEventListener('click', addManualNews); +} + +async function loadNewsStats() { + const stats = await api('/news/stats'); + if (!stats) return; + + // Update sentiment bar + const total = (stats.by_sentiment?.positive || 0) + + (stats.by_sentiment?.neutral || 0) + + (stats.by_sentiment?.negative || 0); + + if (total > 0) { + const positivePercent = (stats.by_sentiment?.positive || 0) / total * 100; + const neutralPercent = (stats.by_sentiment?.neutral || 0) / total * 100; + const negativePercent = (stats.by_sentiment?.negative || 0) / total * 100; + + document.getElementById('sentimentPositiveBar').style.width = positivePercent + '%'; + document.getElementById('sentimentNeutralBar').style.width = neutralPercent + '%'; + document.getElementById('sentimentNegativeBar').style.width = negativePercent + '%'; + + document.getElementById('sentimentText').textContent = + `${Math.round(positivePercent)}% Positive, ${Math.round(neutralPercent)}% Neutral, ${Math.round(negativePercent)}% Negative`; + } else { + document.getElementById('sentimentText').textContent = 'No sentiment data yet'; + } + + // Update today count + document.getElementById('todayArticles').textContent = stats.today || 0; +} + +async function loadNewsArticles() { + const source = document.getElementById('filterNewsSource')?.value || ''; + const sentiment = document.getElementById('filterNewsSentiment')?.value || ''; + const dropRelated = document.getElementById('filterDropRelated')?.checked || false; + + let url = '/news?limit=50'; + if (source) url += `&source=${source}`; + if (sentiment) url += `&sentiment=${sentiment}`; + if (dropRelated) url += '&drop_related=true'; + + const data = await api(url); + if (!data || !data.articles) return; + + const feed = document.getElementById('newsFeed'); + + if (data.articles.length === 0) { + feed.innerHTML = '
No news articles yet. Click "Refresh News" to fetch from sources.
'; + return; + } + + feed.innerHTML = data.articles.map(article => { + const sourceIcon = getSourceIcon(article.source); + const sentimentBadge = getSentimentBadge(article.sentiment_label, article.sentiment_score); + const timeAgo = formatTime(new Date(article.published_at || article.fetched_at)); + const keywords = article.keywords || []; + const hasCorrelation = article.is_drop_related || article.is_restock_related; + + return ` +
+
+ + ${sourceIcon} + ${escapeHtml(article.source_account)} + + ${timeAgo} + ${sentimentBadge} +
+ ${article.title ? `
${escapeHtml(article.title)}
` : ''} +
${escapeHtml(article.content)}
+ ${article.url ? `View Original` : ''} + ${keywords.length > 0 ? ` +
+ ${keywords.slice(0, 5).map(kw => `${escapeHtml(kw)}`).join('')} +
+ ` : ''} + ${hasCorrelation ? ` +
+ + ${article.is_drop_related ? 'Drop Related' : ''} + ${article.is_restock_related ? 'Restock Related' : ''} + +
+ ` : ''} +
+ +
+
+ `; + }).join(''); +} + +function getSourceIcon(source) { + switch (source) { + case 'twitter': + return 'X'; + case 'pokemon_official': + return '⚡'; + case 'discord_manual': + return '💬'; + default: + return '📄'; + } +} + +function getSentimentBadge(label, score) { + if (!label) return ''; + + let badgeClass = 'neutral'; + let symbol = '●'; + + if (label === 'positive') { + badgeClass = 'positive'; + symbol = '+'; + } else if (label === 'negative') { + badgeClass = 'negative'; + symbol = '−'; + } + + const scoreText = score !== null ? ` (${score.toFixed(2)})` : ''; + return `${symbol} ${label}${scoreText}`; +} + +async function refreshNews() { + const btn = document.getElementById('refreshNewsBtn'); + btn.disabled = true; + btn.innerHTML = '⌛ Fetching...'; + + const result = await api('/news/refresh', { method: 'POST' }); + + btn.disabled = false; + btn.innerHTML = '↻ Refresh News'; + + if (result && result.success) { + const msg = `Fetched ${result.fetched} articles.`; + if (result.errors && result.errors.length > 0) { + alert(msg + '\n\nErrors:\n' + result.errors.join('\n')); + } + // Reload news + loadNews(); + } else { + alert('Failed to refresh news'); + } +} + +async function addManualNews() { + const source = document.getElementById('manualNewsSource')?.value || 'Other'; + const title = document.getElementById('manualNewsTitle')?.value.trim() || null; + const content = document.getElementById('manualNewsContent')?.value.trim(); + const url = document.getElementById('manualNewsUrl')?.value.trim() || null; + + if (!content) { + alert('Please enter the message content'); + return; + } + + const result = await api('/news/manual', { + method: 'POST', + body: JSON.stringify({ + source_account: source, + title: title, + content: content, + url: url + }) + }); + + if (result && result.success) { + // Clear form + document.getElementById('manualNewsTitle').value = ''; + document.getElementById('manualNewsContent').value = ''; + document.getElementById('manualNewsUrl').value = ''; + + // Show analysis result + const analysis = result.analysis; + alert(`News added!\n\nSentiment: ${analysis.sentiment_label} (${analysis.sentiment_score.toFixed(2)})\nDrop related: ${analysis.is_drop_related}\nRestock related: ${analysis.is_restock_related}`); + + // Reload news + loadNews(); + } else { + alert('Failed to add news: ' + (result?.error || 'Unknown error')); + } +} + +async function deleteNewsArticle(articleId) { + if (!confirm('Delete this news article?')) return; + + const result = await api(`/news/${articleId}`, { method: 'DELETE' }); + + if (result && result.success) { + loadNewsArticles(); + } else { + alert('Failed to delete article'); + } +} + +// ==================== Users Page ==================== + +let usersInitialized = false; + +async function loadUsers() { + // Initialize event listeners once + if (!usersInitialized) { + document.getElementById('addUserBtn')?.addEventListener('click', addUser); + usersInitialized = true; + } + + const data = await api('/users'); + if (!data) return; + + renderUsersList(data.users || []); +} + +function renderUsersList(users) { + const container = document.getElementById('usersList'); + if (!container) return; + + if (users.length === 0) { + container.innerHTML = '

No users yet. Add one above!

'; + return; + } + + container.innerHTML = users.map(user => ` +
+
+ ${escapeHtml(user.name)} + ${user.zip_code ? `${user.zip_code} (${user.radius_miles}mi)` : 'No location set'} +
+
+
+ + + + + +
+
+ +
+ `).join(''); +} + +async function addUser() { + const nameInput = document.getElementById('newUserName'); + const zipInput = document.getElementById('newUserZip'); + const radiusInput = document.getElementById('newUserRadius'); + + const name = nameInput.value.trim(); + const zip_code = zipInput.value.trim() || null; + const radius_miles = parseInt(radiusInput.value) || 25; + + if (!name) { + alert('Please enter a name'); + return; + } + + const result = await api('/users', { + method: 'POST', + body: JSON.stringify({ name, zip_code, radius_miles }) + }); + + if (result && result.success) { + nameInput.value = ''; + zipInput.value = ''; + radiusInput.value = '25'; + loadUsers(); + } else { + alert('Failed to add user: ' + (result?.error || 'Unknown error')); + } +} + +async function updateUserLocation(userId) { + const card = document.querySelector(`[data-user-id="${userId}"]`); + const zip_code = card.querySelector('.user-zip-input').value.trim(); + const radius_miles = parseInt(card.querySelector('.user-radius-input').value) || 25; + + if (!zip_code) { + alert('Please enter a zip code'); + return; + } + + const result = await api(`/users/${userId}/location`, { + method: 'PUT', + body: JSON.stringify({ zip_code, radius_miles }) + }); + + if (result && result.success) { + loadUsers(); + } else { + alert('Failed to update location: ' + (result?.error || 'Unknown error')); + } +} + +async function viewUserStores(userId) { + const storesDiv = document.getElementById(`userStores-${userId}`); + + // Toggle visibility + if (storesDiv.style.display !== 'none') { + storesDiv.style.display = 'none'; + return; + } + + storesDiv.innerHTML = '

Loading stores...

'; + storesDiv.style.display = 'block'; + + const result = await api(`/users/${userId}/stores`); + + if (!result || result.error) { + storesDiv.innerHTML = `

${result?.error || 'Failed to load stores'}

`; + return; + } + + let html = `
`; + for (const [retailer, stores] of Object.entries(result.stores)) { + if (stores.length > 0) { + html += `
+

${retailer} (${stores.length} stores)

+
    + ${stores.slice(0, 5).map(s => ` +
  • + ${escapeHtml(s.name)} (${s.distance_miles.toFixed(1)} mi)
    + ${escapeHtml(s.address)}, ${escapeHtml(s.city)}, ${s.state} +
    • + `).join('')} +
+
`; + } + } + html += '
'; + + storesDiv.innerHTML = html; +} + +async function deleteUser(userId) { + if (!confirm('Delete this user?')) return; + + const result = await api(`/users/${userId}`, { method: 'DELETE' }); + + if (result && result.success) { + loadUsers(); + } else { + alert('Failed to delete user: ' + (result?.error || 'Unknown error')); + } +} + +// ==================== Settings Page ==================== + +let settingsInitialized = false; + +async function loadSettings() { + // Initialize event listeners once + if (!settingsInitialized) { + document.getElementById('checkBrokenUrlsBtn')?.addEventListener('click', checkBrokenUrls); + document.getElementById('cleanupBrokenUrlsBtn')?.addEventListener('click', cleanupBrokenUrls); + settingsInitialized = true; + } +} + +async function checkBrokenUrls() { + const resultEl = document.getElementById('brokenUrlsResult'); + resultEl.textContent = 'Checking...'; + + const result = await api('/extension/broken-urls'); + + if (result) { + resultEl.textContent = `Found ${result.count} products with broken Pokemon Center URLs.`; + } else { + resultEl.textContent = 'Failed to check broken URLs'; + } +} + +async function cleanupBrokenUrls() { + if (!confirm('This will delete all Pokemon Center products with broken URLs. Continue?')) return; + + const resultEl = document.getElementById('brokenUrlsResult'); + resultEl.textContent = 'Cleaning up...'; + + const result = await api('/extension/broken-urls/cleanup', { method: 'POST' }); + + if (result && result.success) { + resultEl.textContent = `Deleted ${result.deleted} products with broken URLs.`; + } else { + resultEl.textContent = 'Failed to cleanup broken URLs'; + } +} diff --git a/dashboard/static/style.css b/dashboard/static/style.css new file mode 100644 index 0000000..c90a7f1 --- /dev/null +++ b/dashboard/static/style.css @@ -0,0 +1,1613 @@ +/* Pokemon Stock Monitor Dashboard Styles */ +/* Dark theme matching the Chrome extension */ + +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +:root { + --bg-primary: #1a1a2e; + --bg-secondary: #16213e; + --bg-card: #0f3460; + --text-primary: #eee; + --text-secondary: #aaa; + --accent-yellow: #ffcb05; + --accent-green: #00c853; + --accent-red: #c62828; + --accent-blue: #0099ff; + --border-color: #333; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: var(--bg-primary); + color: var(--text-primary); + min-height: 100vh; +} + +.app { + display: flex; + min-height: 100vh; +} + +/* Sidebar */ +.sidebar { + width: 240px; + background: var(--bg-secondary); + padding: 20px; + display: flex; + flex-direction: column; + position: fixed; + height: 100vh; + border-right: 1px solid var(--border-color); +} + +.logo { + display: flex; + align-items: center; + gap: 10px; + margin-bottom: 30px; +} + +.logo-icon { + font-size: 28px; +} + +.logo-text { + font-size: 18px; + font-weight: 600; + color: var(--accent-yellow); +} + +.nav-links { + list-style: none; + flex: 1; +} + +.nav-links li { + margin-bottom: 5px; +} + +.nav-links a { + display: block; + padding: 12px 15px; + color: var(--text-secondary); + text-decoration: none; + border-radius: 8px; + transition: all 0.2s; +} + +.nav-links a:hover { + background: rgba(255, 203, 5, 0.1); + color: var(--text-primary); +} + +.nav-links a.active { + background: var(--accent-yellow); + color: var(--bg-primary); + font-weight: 600; +} + +.sidebar-footer { + padding-top: 20px; + border-top: 1px solid var(--border-color); +} + +.status-indicator { + display: flex; + align-items: center; + gap: 8px; + font-size: 13px; +} + +.status-dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--text-secondary); +} + +.status-dot.active { + background: var(--accent-green); + box-shadow: 0 0 10px var(--accent-green); +} + +.status-dot.error { + background: var(--accent-red); +} + +/* Main Content */ +.content { + flex: 1; + margin-left: 240px; + padding: 30px; +} + +.page { + display: none; +} + +.page.active { + display: block; +} + +h1 { + margin-bottom: 25px; + font-size: 28px; +} + +h2 { + margin-bottom: 15px; + font-size: 20px; + color: var(--accent-yellow); +} + +h3 { + margin-bottom: 10px; + font-size: 16px; +} + +/* Stats Grid */ +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); + gap: 20px; + margin-bottom: 30px; +} + +.stat-card { + background: var(--bg-secondary); + border-radius: 12px; + padding: 20px; + text-align: center; + border: 1px solid var(--border-color); + transition: all 0.2s; +} + +.stat-card.clickable { + cursor: pointer; +} + +.stat-card.clickable:hover { + border-color: var(--accent-yellow); + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); +} + +.stat-card.highlight { + border-color: var(--accent-green); + background: rgba(0, 200, 83, 0.1); +} + +.stat-value { + font-size: 36px; + font-weight: 700; + color: var(--accent-yellow); + margin-bottom: 5px; +} + +.stat-label { + font-size: 13px; + color: var(--text-secondary); +} + +/* Store Cards */ +.store-cards { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 15px; + margin-bottom: 25px; +} + +.store-card { + background: var(--bg-secondary); + border-radius: 12px; + padding: 18px; + border: 1px solid var(--border-color); + display: flex; + align-items: center; + gap: 15px; + cursor: pointer; + transition: all 0.2s; +} + +.store-card:hover { + border-color: var(--accent-yellow); + transform: translateY(-2px); +} + +.store-card.no-data { + opacity: 0.6; +} + +.store-card.no-data:hover { + opacity: 0.8; +} + +.store-logo-container { + width: 60px; + height: 60px; + border-radius: 10px; + display: flex; + align-items: center; + justify-content: center; + padding: 8px; + flex-shrink: 0; +} + +.store-logo-img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.store-icon-fallback { + font-size: 20px; + font-weight: 800; + width: 100%; + height: 100%; + align-items: center; + justify-content: center; +} + +.store-arrow { + font-size: 24px; + color: var(--text-secondary); + margin-left: auto; +} + +.store-info { + flex: 1; +} + +.store-name { + font-weight: 600; + font-size: 14px; + margin-bottom: 4px; +} + +.store-stats { + display: flex; + gap: 12px; + font-size: 12px; +} + +.store-stat { + display: flex; + align-items: center; + gap: 4px; +} + +.store-stat.in-stock { + color: var(--accent-green); +} + +.store-stat.out-of-stock { + color: var(--text-secondary); +} + +.store-stat.no-data { + color: var(--text-secondary); + font-style: italic; +} + +/* Activity Feed */ +.section { + background: var(--bg-secondary); + border-radius: 12px; + padding: 20px; + margin-bottom: 20px; + border: 1px solid var(--border-color); +} + +.activity-feed { + max-height: 400px; + overflow-y: auto; +} + +.activity-item { + display: flex; + align-items: center; + gap: 15px; + padding: 12px 0; + border-bottom: 1px solid var(--border-color); +} + +.activity-item:last-child { + border-bottom: none; +} + +.activity-thumb { + width: 50px; + height: 50px; + border-radius: 8px; + object-fit: cover; + flex-shrink: 0; + background: var(--bg-card); +} + +.activity-thumb.activity-logo { + object-fit: contain; + padding: 8px; + background: #fff; +} + +.activity-store-icon { + width: 50px; + height: 50px; + border-radius: 8px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + font-weight: 800; + flex-shrink: 0; + padding: 6px; +} + +.activity-store-logo { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.activity-store-fallback { + font-size: 16px; + font-weight: 800; +} + +.activity-icon { + width: 50px; + height: 50px; + border-radius: 8px; + display: flex; + align-items: center; + justify-content: center; + font-size: 20px; + flex-shrink: 0; +} + +.activity-icon.pokemoncenter { + background: linear-gradient(135deg, #ffcb05 0%, #ff9500 100%); +} + +.activity-icon.target { + background: linear-gradient(135deg, #cc0000 0%, #990000 100%); +} + +.activity-icon.walmart { + background: linear-gradient(135deg, #0071ce 0%, #004c91 100%); +} + +.activity-icon.bestbuy { + background: linear-gradient(135deg, #0046be 0%, #003087 100%); +} + +.activity-icon.new-drop { + background: rgba(0, 153, 255, 0.2); +} + +.activity-icon.restock { + background: rgba(0, 200, 83, 0.2); +} + +.activity-name a { + color: var(--text-primary); + text-decoration: none; + transition: color 0.2s; +} + +.activity-name a:hover { + color: var(--accent-yellow); +} + +.activity-badge { + display: inline-block; + padding: 2px 6px; + border-radius: 4px; + font-size: 10px; + font-weight: 600; + margin-left: 8px; + text-transform: uppercase; +} + +.activity-badge.new-drop { + background: rgba(0, 153, 255, 0.2); + color: var(--accent-blue); +} + +.activity-badge.restock { + background: rgba(0, 200, 83, 0.2); + color: var(--accent-green); +} + +.activity-details { + flex: 1; +} + +.activity-name { + font-weight: 500; + margin-bottom: 3px; +} + +.activity-meta { + font-size: 12px; + color: var(--text-secondary); +} + +.activity-time { + font-size: 12px; + color: var(--text-secondary); +} + +/* Event Filter Badge */ +.event-filter-badge { + display: inline-flex; + align-items: center; + gap: 8px; + background: var(--accent-yellow); + color: var(--bg-primary); + padding: 8px 12px; + border-radius: 20px; + font-size: 13px; + font-weight: 600; + margin-bottom: 15px; +} + +.filter-badge-text { + display: inline-block; +} + +.filter-badge-close { + cursor: pointer; + font-size: 18px; + line-height: 1; + opacity: 0.7; + transition: opacity 0.2s; +} + +.filter-badge-close:hover { + opacity: 1; +} + +/* Filters */ +.filters { + display: flex; + gap: 15px; + margin-bottom: 20px; + flex-wrap: wrap; + align-items: center; +} + +.filters select, .filters input[type="text"] { + padding: 10px 15px; + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-secondary); + color: var(--text-primary); + font-size: 14px; +} + +.filters select:focus, .filters input:focus { + outline: none; + border-color: var(--accent-yellow); +} + +.checkbox-label { + display: flex; + align-items: center; + gap: 8px; + cursor: pointer; +} + +/* Products Grid */ +.products-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 20px; +} + +.product-card { + background: var(--bg-secondary); + border-radius: 12px; + padding: 15px; + border: 1px solid var(--border-color); + transition: transform 0.2s, border-color 0.2s; + cursor: pointer; +} + +.product-card:hover { + transform: translateY(-3px); + border-color: var(--accent-yellow); +} + +.product-card.in-stock { + border-left: 4px solid var(--accent-green); +} + +.product-card.out-of-stock { + opacity: 0.7; +} + +.product-card.favorite { + border-color: var(--accent-yellow); +} + +.product-image-container { + width: 100%; + height: 150px; + background: var(--bg-card); + border-radius: 8px; + margin-bottom: 10px; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; +} + +.product-image { + width: 100%; + height: 100%; + object-fit: contain; +} + +.product-image-placeholder { + font-size: 48px; + opacity: 0.3; +} + +.product-logo-fallback { + max-width: 80%; + max-height: 60%; + object-fit: contain; + opacity: 0.4; +} + +.product-store-icon { + width: 80px; + height: 80px; + border-radius: 12px; + display: flex; + align-items: center; + justify-content: center; + font-size: 28px; + font-weight: 800; + opacity: 0.6; + padding: 12px; +} + +.product-store-logo { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.product-store-fallback { + font-size: 24px; + font-weight: 800; + align-items: center; + justify-content: center; +} + +.product-name { + font-weight: 500; + margin-bottom: 8px; + line-height: 1.3; +} + +.product-meta { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 13px; +} + +.product-price { + color: var(--accent-yellow); + font-weight: 600; +} + +.product-stock { + padding: 4px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.product-stock.in-stock { + background: rgba(0, 200, 83, 0.2); + color: var(--accent-green); +} + +.product-stock.out-of-stock { + background: rgba(198, 40, 40, 0.2); + color: var(--accent-red); +} + +.product-actions { + display: flex; + justify-content: space-between; + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border-color); +} + +.btn-favorite { + background: none; + border: none; + cursor: pointer; + font-size: 18px; + opacity: 0.5; + transition: opacity 0.2s; +} + +.btn-favorite:hover, .btn-favorite.active { + opacity: 1; +} + +/* Charts */ +.charts-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); + gap: 20px; +} + +.chart-card { + background: var(--bg-secondary); + border-radius: 12px; + padding: 20px; + border: 1px solid var(--border-color); +} + +.chart-card canvas { + max-height: 300px; +} + +/* Favorites */ +.favorites-section { + max-width: 800px; +} + +.add-favorite-form { + background: var(--bg-secondary); + border-radius: 12px; + padding: 20px; + margin-bottom: 30px; + border: 1px solid var(--border-color); +} + +.form-row { + display: flex; + gap: 10px; + flex-wrap: wrap; +} + +.form-row input, .form-row select { + padding: 10px 15px; + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-primary); + color: var(--text-primary); + font-size: 14px; +} + +.form-row input { + flex: 1; + min-width: 200px; +} + +.suggestions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 15px; +} + +.suggestion-tag { + padding: 6px 12px; + background: var(--bg-card); + border-radius: 20px; + font-size: 12px; + cursor: pointer; + transition: background 0.2s; +} + +.suggestion-tag:hover { + background: var(--accent-yellow); + color: var(--bg-primary); +} + +.favorites-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.favorite-item { + display: flex; + align-items: center; + gap: 15px; + padding: 15px; + background: var(--bg-secondary); + border-radius: 8px; + border: 1px solid var(--border-color); +} + +.favorite-icon { + font-size: 24px; +} + +.favorite-details { + flex: 1; +} + +.favorite-name { + font-weight: 500; + margin-bottom: 3px; +} + +.favorite-type { + font-size: 12px; + color: var(--text-secondary); +} + +.favorite-priority { + padding: 4px 10px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.favorite-priority.high { + background: rgba(198, 40, 40, 0.2); + color: #ff6b6b; +} + +.favorite-priority.medium { + background: rgba(255, 203, 5, 0.2); + color: var(--accent-yellow); +} + +.favorite-priority.low { + background: rgba(0, 200, 83, 0.2); + color: var(--accent-green); +} + +.btn-delete { + background: none; + border: none; + color: var(--accent-red); + cursor: pointer; + font-size: 18px; + opacity: 0.7; + transition: opacity 0.2s; +} + +.btn-delete:hover { + opacity: 1; +} + +/* Buttons */ +.btn-primary { + padding: 10px 20px; + background: var(--accent-yellow); + color: var(--bg-primary); + border: none; + border-radius: 8px; + font-weight: 600; + cursor: pointer; + transition: background 0.2s; +} + +.btn-primary:hover { + background: #ffd633; +} + +.btn-secondary { + padding: 10px 20px; + background: var(--bg-card); + color: var(--text-primary); + border: 1px solid var(--border-color); + border-radius: 8px; + font-weight: 500; + cursor: pointer; + transition: background 0.2s; + margin-right: 10px; + margin-bottom: 10px; +} + +.btn-secondary:hover { + background: var(--border-color); +} + +/* Settings */ +.settings-section { + max-width: 600px; +} + +.settings-section h3 { + margin-top: 25px; +} + +.help-text { + color: var(--text-secondary); + font-size: 14px; + margin-bottom: 15px; +} + +/* Modal */ +.modal { + display: none; + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.7); + z-index: 1000; + align-items: center; + justify-content: center; +} + +.modal.show { + display: flex; +} + +.modal-content { + background: var(--bg-secondary); + border-radius: 12px; + padding: 30px; + max-width: 500px; + width: 90%; + position: relative; +} + +.modal-close { + position: absolute; + top: 15px; + right: 20px; + font-size: 24px; + cursor: pointer; + color: var(--text-secondary); +} + +.modal-close:hover { + color: var(--text-primary); +} + +/* Loading */ +.loading { + text-align: center; + padding: 40px; + color: var(--text-secondary); +} + +/* Scrollbar - subtle integrated style */ +::-webkit-scrollbar { + width: 6px; + height: 6px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.1); + border-radius: 3px; +} + +::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.2); +} + +::-webkit-scrollbar-corner { + background: transparent; +} + +/* Firefox scrollbar */ +* { + scrollbar-width: thin; + scrollbar-color: rgba(255, 255, 255, 0.1) transparent; +} + +/* Control Panel */ +.control-panel { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(350px, 1fr)); + gap: 20px; +} + +.control-card { + background: var(--bg-secondary); + border-radius: 12px; + border: 1px solid var(--border-color); + overflow: hidden; +} + +.control-card-header { + padding: 15px 20px; + border-bottom: 1px solid var(--border-color); + display: flex; + justify-content: space-between; + align-items: center; +} + +.control-card-header h3 { + margin: 0; + font-size: 16px; +} + +.control-card-body { + padding: 20px; +} + +.control-card-actions { + padding: 15px 20px; + border-top: 1px solid var(--border-color); + display: flex; + gap: 10px; +} + +.monitor-status-card { + grid-column: span 2; +} + +.monitor-badge { + padding: 4px 12px; + border-radius: 20px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; +} + +.monitor-badge.running { + background: rgba(0, 200, 83, 0.2); + color: var(--accent-green); +} + +.monitor-badge.stopped { + background: rgba(198, 40, 40, 0.2); + color: var(--accent-red); +} + +.last-check-text { + font-size: 13px; + color: var(--text-secondary); + margin-top: 10px; +} + +.btn-large { + padding: 12px 24px; + font-size: 15px; + flex: 1; +} + +.btn-danger { + padding: 10px 20px; + background: var(--accent-red); + color: white; + border: none; + border-radius: 8px; + font-weight: 600; + cursor: pointer; + transition: background 0.2s; +} + +.btn-danger:hover:not(:disabled) { + background: #d32f2f; +} + +.btn-danger:disabled, .btn-primary:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-icon { + margin-right: 6px; +} + +.interval-input { + display: flex; + align-items: center; + gap: 10px; +} + +.interval-input input { + width: 80px; + padding: 10px; + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-primary); + color: var(--text-primary); + font-size: 16px; + text-align: center; +} + +.scrapers-card { + grid-column: span 2; +} + +.scrapers-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 15px; +} + +.scraper-card { + background: var(--bg-primary); + border-radius: 10px; + padding: 15px; + display: flex; + align-items: center; + gap: 15px; + border: 1px solid var(--border-color); + transition: border-color 0.2s; +} + +.scraper-card.enabled { + border-color: var(--accent-green); +} + +.scraper-card.running { + border-color: var(--accent-yellow); + box-shadow: 0 0 10px rgba(255, 203, 5, 0.2); +} + +.scraper-logo { + width: 50px; + height: 50px; + border-radius: 10px; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + padding: 8px; +} + +.scraper-logo img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.scraper-logo-fallback { + font-size: 16px; + font-weight: 800; + align-items: center; + justify-content: center; +} + +.scraper-info { + flex: 1; +} + +.scraper-name { + font-weight: 600; + margin-bottom: 4px; +} + +.scraper-status { + font-size: 12px; +} + +.status-running { + color: var(--accent-yellow); +} + +.status-enabled { + color: var(--accent-green); +} + +.status-disabled { + color: var(--text-secondary); +} + +.scraper-last-run { + font-size: 11px; + color: var(--text-secondary); + margin-top: 4px; +} + +.scraper-error { + font-size: 11px; + color: var(--accent-red); + margin-top: 4px; +} + +/* Toggle Switch */ +.toggle-switch { + position: relative; + display: inline-block; + width: 50px; + height: 26px; +} + +.toggle-switch input { + opacity: 0; + width: 0; + height: 0; +} + +.toggle-slider { + position: absolute; + cursor: pointer; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: var(--border-color); + transition: 0.3s; + border-radius: 26px; +} + +.toggle-slider:before { + position: absolute; + content: ""; + height: 20px; + width: 20px; + left: 3px; + bottom: 3px; + background-color: white; + transition: 0.3s; + border-radius: 50%; +} + +.toggle-switch input:checked + .toggle-slider { + background-color: var(--accent-green); +} + +.toggle-switch input:checked + .toggle-slider:before { + transform: translateX(24px); +} + +/* News Page */ +.news-stats-bar { + display: flex; + justify-content: space-between; + align-items: center; + background: var(--bg-secondary); + padding: 15px 20px; + border-radius: 12px; + margin-bottom: 20px; + border: 1px solid var(--border-color); +} + +.sentiment-overview { + display: flex; + align-items: center; + gap: 15px; +} + +.sentiment-label { + color: var(--text-secondary); + font-size: 14px; +} + +.sentiment-bar { + display: flex; + width: 200px; + height: 8px; + border-radius: 4px; + overflow: hidden; + background: var(--border-color); +} + +.sentiment-positive { + background: var(--accent-green); +} + +.sentiment-neutral { + background: var(--text-secondary); +} + +.sentiment-negative { + background: var(--accent-red); +} + +.sentiment-text { + font-size: 12px; + color: var(--text-secondary); +} + +.news-count { + font-size: 14px; + color: var(--text-secondary); +} + +.news-filters { + margin-bottom: 15px; +} + +.manual-news-section { + margin-bottom: 20px; +} + +.manual-news-section details { + background: var(--bg-secondary); + border-radius: 12px; + border: 1px solid var(--border-color); +} + +.manual-news-section summary { + padding: 15px 20px; + cursor: pointer; + font-weight: 500; + color: var(--text-primary); +} + +.manual-news-section summary:hover { + color: var(--accent-yellow); +} + +.manual-news-form { + padding: 0 20px 20px 20px; +} + +.manual-news-form textarea { + width: 100%; + padding: 12px 15px; + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-primary); + color: var(--text-primary); + font-size: 14px; + font-family: inherit; + resize: vertical; + margin: 10px 0; +} + +.manual-news-form textarea:focus { + outline: none; + border-color: var(--accent-yellow); +} + +.news-feed { + display: flex; + flex-direction: column; + gap: 15px; +} + +.news-card { + background: var(--bg-secondary); + border-radius: 12px; + padding: 20px; + border: 1px solid var(--border-color); + transition: border-color 0.2s; +} + +.news-card:hover { + border-color: var(--accent-yellow); +} + +.news-header { + display: flex; + align-items: center; + gap: 15px; + margin-bottom: 12px; + flex-wrap: wrap; +} + +.news-source { + display: flex; + align-items: center; + gap: 8px; + font-weight: 500; +} + +.source-icon { + font-size: 16px; +} + +.news-time { + color: var(--text-secondary); + font-size: 12px; + margin-left: auto; +} + +.sentiment-badge { + padding: 4px 10px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.sentiment-badge.positive { + background: rgba(0, 200, 83, 0.2); + color: var(--accent-green); +} + +.sentiment-badge.neutral { + background: rgba(170, 170, 170, 0.2); + color: var(--text-secondary); +} + +.sentiment-badge.negative { + background: rgba(198, 40, 40, 0.2); + color: var(--accent-red); +} + +.news-title { + font-weight: 600; + font-size: 16px; + margin-bottom: 8px; + color: var(--accent-yellow); +} + +.news-content { + line-height: 1.5; + margin-bottom: 12px; + color: var(--text-primary); +} + +.news-link { + color: var(--accent-blue); + text-decoration: none; + font-size: 13px; +} + +.news-link:hover { + text-decoration: underline; +} + +.news-keywords { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 12px; +} + +.keyword-tag { + padding: 4px 10px; + background: var(--bg-card); + border-radius: 12px; + font-size: 11px; + color: var(--text-secondary); +} + +.news-correlation { + margin-top: 10px; +} + +.correlation-badge { + display: inline-block; + padding: 4px 10px; + background: rgba(255, 203, 5, 0.2); + color: var(--accent-yellow); + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.news-actions { + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border-color); + display: flex; + justify-content: flex-end; +} + +.btn-delete-small { + padding: 6px 12px; + background: none; + border: 1px solid var(--accent-red); + color: var(--accent-red); + border-radius: 6px; + font-size: 12px; + cursor: pointer; + transition: all 0.2s; +} + +.btn-delete-small:hover { + background: var(--accent-red); + color: white; +} + +/* Users Page */ +.users-section { + max-width: 800px; +} + +.add-user-form { + background: var(--bg-card); + padding: 20px; + border-radius: 8px; + margin-bottom: 30px; +} + +.add-user-form h3 { + margin-bottom: 15px; + color: var(--accent-yellow); +} + +.form-group { + margin-bottom: 15px; +} + +.form-group label { + display: block; + margin-bottom: 5px; + color: var(--text-secondary); + font-size: 0.9rem; +} + +.form-group input { + width: 100%; + padding: 10px; + border: 1px solid var(--border-color); + border-radius: 4px; + background: var(--bg-secondary); + color: var(--text-primary); + font-size: 1rem; +} + +.users-list { + display: flex; + flex-direction: column; + gap: 15px; +} + +.user-card { + background: var(--bg-card); + padding: 20px; + border-radius: 8px; + border-left: 4px solid var(--accent-blue); +} + +.user-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 15px; +} + +.user-name { + font-size: 1.2rem; + font-weight: bold; + color: var(--accent-yellow); +} + +.user-location { + color: var(--text-secondary); + font-size: 0.9rem; +} + +.user-details { + margin-bottom: 10px; +} + +.form-inline { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.form-inline input { + padding: 8px 12px; + border: 1px solid var(--border-color); + border-radius: 4px; + background: var(--bg-secondary); + color: var(--text-primary); +} + +.user-zip-input { + width: 120px; +} + +.user-radius-input { + width: 80px; +} + +.btn-small { + padding: 6px 12px !important; + font-size: 0.85rem !important; +} + +.btn-danger { + background: var(--accent-red); + color: white; + border: none; + padding: 8px 16px; + border-radius: 4px; + cursor: pointer; + transition: opacity 0.2s; +} + +.btn-danger:hover { + opacity: 0.8; +} + +.user-stores { + margin-top: 15px; + padding-top: 15px; + border-top: 1px solid var(--border-color); +} + +.stores-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 20px; +} + +.retailer-section h4 { + color: var(--accent-blue); + margin-bottom: 10px; +} + +.store-list { + list-style: none; + padding: 0; +} + +.store-list li { + padding: 10px; + background: var(--bg-secondary); + border-radius: 4px; + margin-bottom: 8px; +} + +.store-address { + color: var(--text-secondary); + font-size: 0.85rem; +} + +.empty-state { + text-align: center; + padding: 40px; + color: var(--text-secondary); +} + +.loading { + text-align: center; + padding: 20px; + color: var(--text-secondary); +} + +.error { + color: var(--accent-red); + padding: 10px; +} + +/* Responsive */ +@media (max-width: 768px) { + .sidebar { + width: 60px; + padding: 15px 10px; + } + + .logo-text, .nav-links a span { + display: none; + } + + .content { + margin-left: 60px; + } + + .stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .charts-grid { + grid-template-columns: 1fr; + } + + .control-panel { + grid-template-columns: 1fr; + } + + .monitor-status-card, + .scrapers-card { + grid-column: span 1; + } + + .control-card-actions { + flex-direction: column; + } + + .scrapers-grid { + grid-template-columns: 1fr; + } +} diff --git a/dashboard/templates/index.html b/dashboard/templates/index.html new file mode 100644 index 0000000..d1bb20a --- /dev/null +++ b/dashboard/templates/index.html @@ -0,0 +1,371 @@ + + + + + + Pokemon Stock Monitor + + + + +
+ + + + +
+ +
+

Dashboard

+ + +
+ +
+ +
+
+
-
+
New Drops Today
+
+
+
-
+
Restocks Today
+
+
+
-
+
New Drops (7 days)
+
+
+
-
+
Restocks (7 days)
+
+
+ +
+

Recent Activity

+
+
Loading...
+
+
+
+ + +
+

Products

+ + + +
+ + + + +
+ +
+
Loading products...
+
+
+ + +
+

Analytics

+ +
+
+

Drop Timing (by hour)

+ +
+
+

Stock Duration by Category

+ +
+
+

Products by Site

+ +
+
+
+ + +
+

News

+ + +
+
+ Sentiment: + + + + + + Loading... +
+
+ 0 articles today +
+
+ + +
+ + + + +
+ + +
+
+ Add Manual News (Discord messages, etc.) +
+
+ + +
+ +
+ + +
+
+
+
+ + +
+
Loading news...
+
+
+ + +
+

Favorites

+ +
+
+

Add New Favorite

+
+ + + + +
+
+
+ +

Your Favorites

+
+
Loading favorites...
+
+
+
+ + +
+

Control Panel

+ +
+ +
+
+

Monitor Status

+ Stopped +
+
+

Monitor is not running

+

Last check: Never

+
+
+ + +
+
+ + +
+
+

Check Interval

+
+
+
+ + seconds + +
+

How often to check for new products (minimum 10 seconds)

+
+
+ + +
+
+

Scrapers

+
+
+
+ +
+
+
+ + +
+
+

Quick Actions

+
+
+ + +
+
+
+
+ + +
+

Users

+

Manage users and their store locations for local stock checking

+ +
+
+

Add New User

+
+ + +
+
+ + +
+
+ + +
+ +
+ +

Current Users

+
+ +

Loading users...

+
+
+
+ + +
+

Settings

+ +
+

Discord Webhook

+

Configure your Discord webhook URL in config.py

+ +

Monitor Status

+
+

Last check: -

+

Next check: -

+
+ +

Data Management

+ + + +

Pokemon Center Cleanup

+

Remove Pokemon Center products with broken URLs (SKUs without product slugs)

+ + +

+
+
+
+
+ + + + + + + diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000..60779c2 --- /dev/null +++ b/data/__init__.py @@ -0,0 +1 @@ +# Data directory - contains runtime data files diff --git a/debug_bestbuy.html b/debug_bestbuy.html new file mode 100644 index 0000000..cc4c922 --- /dev/null +++ b/debug_bestbuy.html @@ -0,0 +1,688 @@ +pokemon tcg - Best Buy
+ + + + + + + + + + +
Skip to contentGo to Product Search
Gift Ideas
Cart
+ + +
+ +
+
Main Content

pokemon tcg

in Trading Cards (267)
Sort by:
1-18 of 267 items
123...15
Next
Tell us about your search experience

Get it fast

Select Store
55423

Brand

Price

to

Franchise

Customer Rating

Color

Recommended Age

Sponsored
Sponsored
Sponsored

Similar products from outside of Best Buy

sponsored
Sponsored
+ + + + + + + + + + + + +

\ No newline at end of file diff --git a/debug_gamestop.html b/debug_gamestop.html new file mode 100644 index 0000000..7eaae7b --- /dev/null +++ b/debug_gamestop.html @@ -0,0 +1 @@ +Just a moment...
Icon for www.gamestop.com

www.gamestop.com

Performing security verification

This website uses a security service to protect against malicious bots. This page is displayed while the website verifies you are not a bot.

\ No newline at end of file diff --git a/discord_notifier.py b/discord_notifier.py deleted file mode 100644 index 8f30c19..0000000 --- a/discord_notifier.py +++ /dev/null @@ -1,183 +0,0 @@ -""" -Discord webhook notifications for stock alerts -Sends rich embeds with product info and direct links -""" - -import logging -import requests -from datetime import datetime -from typing import Optional -from config import DISCORD_WEBHOOK_URL - -logger = logging.getLogger(__name__) - -# Colors for different notification types -COLOR_RESTOCK = 0x00FF00 # Green - item back in stock -COLOR_NEW_DROP = 0x0099FF # Blue - new product listing -COLOR_PREORDER = 0xFFAA00 # Orange - pre-order available -COLOR_ERROR = 0xFF0000 # Red - error notification - -# Site icons/emojis -SITE_EMOJIS = { - "pokemoncenter": "\U0001F7E1", # Yellow circle - "target": "\U0001F534", # Red circle - "walmart": "\U0001F535", # Blue circle - "bestbuy": "\U0001F7E1", # Yellow circle -} - - -def send_stock_alert( - product_name: str, - product_url: str, - price: str, - site: str, - alert_type: str = "restock", - image_url: Optional[str] = None, -): - """ - Send a Discord notification for a stock alert - - Args: - product_name: Name of the product - product_url: Direct link to the product - price: Price string (e.g., "$49.99") - site: Site name (pokemoncenter, target, etc.) - alert_type: "restock", "new_drop", or "preorder" - image_url: Optional product image URL - """ - if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": - logger.error("Discord webhook URL not configured! Update config.py") - return False - - # Choose color based on alert type - if alert_type == "restock": - color = COLOR_RESTOCK - title = f"\U0001F6A8 RESTOCK ALERT" - elif alert_type == "new_drop": - color = COLOR_NEW_DROP - title = f"\U0001F195 NEW DROP" - elif alert_type == "preorder": - color = COLOR_PREORDER - title = f"\u23F0 PRE-ORDER AVAILABLE" - else: - color = COLOR_RESTOCK - title = f"\U0001F514 STOCK ALERT" - - site_emoji = SITE_EMOJIS.get(site.lower(), "\U0001F6D2") - site_display = site.replace("pokemoncenter", "Pokemon Center").title() - - # Build the embed - embed = { - "title": title, - "description": f"**{product_name}**", - "url": product_url, - "color": color, - "fields": [ - {"name": "Price", "value": price or "See link", "inline": True}, - {"name": "Store", "value": f"{site_emoji} {site_display}", "inline": True}, - {"name": "Link", "value": f"[\U0001F6D2 BUY NOW]({product_url})", "inline": False}, - ], - "footer": {"text": "Pokemon Stock Monitor"}, - "timestamp": datetime.utcnow().isoformat(), - } - - if image_url: - embed["thumbnail"] = {"url": image_url} - - payload = { - "content": "@everyone", # Ping everyone - "embeds": [embed], - } - - try: - response = requests.post( - DISCORD_WEBHOOK_URL, - json=payload, - timeout=10, - ) - response.raise_for_status() - logger.info(f"Discord notification sent for: {product_name}") - return True - except requests.exceptions.RequestException as e: - logger.error(f"Failed to send Discord notification: {e}") - return False - - -def send_error_notification(error_message: str, site: str = "Unknown"): - """Send an error notification to Discord""" - if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": - return False - - embed = { - "title": "\u26A0\uFE0F Monitor Error", - "description": error_message, - "color": COLOR_ERROR, - "fields": [{"name": "Site", "value": site, "inline": True}], - "footer": {"text": "Pokemon Stock Monitor"}, - "timestamp": datetime.utcnow().isoformat(), - } - - payload = {"embeds": [embed]} - - try: - response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10) - response.raise_for_status() - return True - except requests.exceptions.RequestException as e: - logger.error(f"Failed to send error notification: {e}") - return False - - -def send_startup_notification(): - """Send a notification that the monitor has started""" - if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": - return False - - embed = { - "title": "\u2705 Monitor Started", - "description": "Pokemon Stock Monitor is now running and watching for restocks!", - "color": 0x00FF00, - "footer": {"text": "Pokemon Stock Monitor"}, - "timestamp": datetime.utcnow().isoformat(), - } - - payload = {"embeds": [embed]} - - try: - response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10) - response.raise_for_status() - return True - except requests.exceptions.RequestException as e: - logger.error(f"Failed to send startup notification: {e}") - return False - - -def test_webhook(): - """Test the Discord webhook connection""" - print("Testing Discord webhook...") - - if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": - print("ERROR: Webhook URL not configured!") - print("Edit config.py and set DISCORD_WEBHOOK_URL") - return False - - # Send a test notification - success = send_stock_alert( - product_name="Test Product - Pokemon TCG Booster", - product_url="https://www.pokemoncenter.com/test", - price="$4.99", - site="pokemoncenter", - alert_type="restock", - ) - - if success: - print("SUCCESS! Check your Discord channel for the test message.") - else: - print("FAILED! Check the webhook URL and try again.") - - return success - - -if __name__ == "__main__": - # Run webhook test - test_webhook() diff --git a/docs/BOT_DETECTION_STRATEGIES.md b/docs/BOT_DETECTION_STRATEGIES.md new file mode 100644 index 0000000..bb32791 --- /dev/null +++ b/docs/BOT_DETECTION_STRATEGIES.md @@ -0,0 +1,189 @@ +# Pokemon Center Bot Detection - Strategies & Solutions + +Pokemon Center uses **Imperva (Incapsula)** bot protection, which is one of the most aggressive anti-bot systems. Here are practical approaches to work around it. + +## Current Problem +- Direct API calls: **403 Forbidden** +- Sitemap.xml: **Blocked by Imperva** +- Browser automation: **CAPTCHA after a few requests** + +--- + +## Strategy 1: Third-Party Stock Trackers (Easiest) + +Instead of scraping Pokemon Center directly, integrate with existing stock tracking services: + +### Discord Bots/Webhooks +Several Discord servers track Pokemon Center in real-time: +- **Pokemon TCG Drops** - Dedicated Pokemon TCG stock alerts +- **Stock Informer** - General retail stock tracking +- Search Discord for "Pokemon Center stock alerts" + +You can join these servers and set up webhook forwarding to your own Discord. + +### Stock Alert Services +- **NowInStock.net** - Has Pokemon Center tracking +- **Distill.io** - Browser extension for change detection +- **Visualping** - Monitors page changes + +### Implementation +```python +# Forward alerts from a public tracker to your system +# Set up a Discord bot to listen to stock alert channels +# Then forward to your own notification system +``` + +--- + +## Strategy 2: Undetected Chrome Driver + +Use `undetected-chromedriver` which patches Chrome to avoid detection: + +```bash +pip install undetected-chromedriver +``` + +```python +import undetected_chromedriver as uc + +driver = uc.Chrome(headless=False) # Headless often gets detected +driver.get("https://www.pokemoncenter.com/category/tcg-cards") + +# Add human-like delays +import time +import random +time.sleep(random.uniform(3, 7)) + +# Scroll like a human +driver.execute_script("window.scrollBy(0, 500)") +time.sleep(random.uniform(1, 3)) +``` + +**Pros:** Often bypasses Imperva +**Cons:** Slower, still may trigger CAPTCHA eventually + +--- + +## Strategy 3: Session Persistence + +Keep a browser session alive and logged in: + +1. **Manual login once** - Complete any CAPTCHA manually +2. **Save cookies** - Export session cookies +3. **Reuse session** - Load cookies on each check +4. **Longer intervals** - Check every 3-5 minutes instead of 60-90 seconds + +```python +# Save cookies after manual login +import pickle +pickle.dump(driver.get_cookies(), open("cookies.pkl", "wb")) + +# Load cookies on subsequent runs +cookies = pickle.load(open("cookies.pkl", "rb")) +for cookie in cookies: + driver.add_cookie(cookie) +``` + +--- + +## Strategy 4: Residential Proxy Rotation + +Use residential proxies that look like real home internet connections: + +### Providers +- **Bright Data** (formerly Luminati) - Best but expensive +- **Oxylabs** - Good quality +- **Smartproxy** - Budget option +- **IPRoyal** - Pay-per-GB + +### Cost +- ~$10-15/GB for residential proxies +- ~50-100 requests per MB depending on page size + +```python +proxies = { + "http": "http://user:pass@proxy.provider.com:port", + "https": "http://user:pass@proxy.provider.com:port" +} +``` + +--- + +## Strategy 5: Human-Like Behavior Pattern + +If continuing with browser automation: + +```python +import random +import time + +def human_like_check(): + # Random delay between checks (2-5 minutes) + delay = random.uniform(120, 300) + + # Random user agents + user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...", + ] + + # Vary timing throughout the day + # Check more during business hours, less at night + hour = datetime.now().hour + if 2 <= hour <= 6: # 2am-6am + delay *= 2 # Slower at night + + # Add random mouse movements + # Add random scrolling + # Occasionally visit other pages (home, about) + + return delay +``` + +--- + +## Strategy 6: Webhook from Manual Monitoring + +The most reliable approach for Pokemon Center: + +1. **Open Pokemon Center in your actual browser** +2. **Use Distill.io browser extension** to monitor changes +3. **Set up webhook** to forward alerts to your system + +This way: +- You're using a real browser with real cookies +- Imperva sees normal human behavior +- Change detection triggers your notification system + +--- + +## Recommended Approach + +Given Pokemon Center's aggressive protection, I recommend a **hybrid approach**: + +1. **Primary: Join existing stock alert Discord servers** + - Let others deal with the bot detection + - Forward their alerts to your system + +2. **Secondary: Use undetected-chromedriver with long intervals** + - Check every 3-5 minutes + - Use session persistence + - Add human-like behavior + +3. **Backup: Manual Distill.io monitoring** + - For specific products you really care about + - Most reliable but requires keeping browser open + +--- + +## Code Changes Needed + +To implement these strategies, we would need to: + +1. Add `undetected-chromedriver` as an option +2. Implement cookie/session persistence +3. Add configurable random delays +4. Add proxy support +5. Consider adding Discord bot listener for third-party alerts + +Would you like me to implement any of these strategies? diff --git a/docs/NOTES.md b/docs/NOTES.md new file mode 100644 index 0000000..0a98e7a --- /dev/null +++ b/docs/NOTES.md @@ -0,0 +1,13 @@ +# Pokemon Stock Monitor - Notes + +## Bot Detection / CAPTCHA Log + +| Date | Time | Site | Notes | +|------|------|------|-------| +| 2026-03-25 | ~10:30 AM | PokemonCenter | CAPTCHAs appeared, bot protection triggered | + +## Tips +- Pokemon Center uses Imperva bot protection +- CAPTCHAs more likely during high traffic (product drops) +- Using Chrome extension with real browser avoids most detection +- If blocked, clear cookies and wait before retrying diff --git a/docs/POKEMON_CENTER_DROPS_NOTES.md b/docs/POKEMON_CENTER_DROPS_NOTES.md new file mode 100644 index 0000000..77f8948 --- /dev/null +++ b/docs/POKEMON_CENTER_DROPS_NOTES.md @@ -0,0 +1,46 @@ +# Pokemon Center Drop Strategy Notes + +Source: https://www.pokepullz.ca/blog/pokemon-center-drops-guide.html + +## Queue System +- Pokemon Center uses a **randomized virtual queue** that activates before major drops +- Position is NOT based on when you arrived - it's assigned randomly once queue activates +- This prevents site crashes and distributes access fairly + +## Timing Signals +- Security changes on the Pokemon Center site serve as early warning signals that drops are imminent +- Products sometimes load in waves rather than all at once + +## Multi-Device Strategy +1. Join queue on primary device (desktop on WiFi) first +2. Wait a few minutes, then join on second device (phone on mobile data) +3. This gives you two separate queue positions and accounts for staggered product loading + +## Technical Tips + +### The Cart Trick +When Add to Cart button isn't displaying during glitchy drops: +1. Click the cart icon to go to your cart page +2. Hit browser back button to return to product page +3. This forces a page reload that may show the button + +## Order Flagging Risks (Avoid These!) +- Typos in billing/shipping information +- Mismatched address data +- VPNs or browser extensions +- Multiple rapid orders +- Overloaded carts +- Queue-checking tools or auto-refreshers + +## Pre-Drop Prep Checklist +- [ ] Log into account days ahead +- [ ] Have devices ready +- [ ] Monitor Discord alerts +- [ ] Disable browser extensions +- [ ] Turn off VPN + +## Implications for Our Monitor +1. **Don't use VPN/proxies during actual purchases** - only for monitoring +2. **Queue position is random** - early detection still helps you GET in queue early +3. **Watch for security changes** - could indicate imminent drop +4. **Multi-wave loading** - may need multiple checks to catch all products diff --git a/docs/SMART_MONITOR_DESIGN.md b/docs/SMART_MONITOR_DESIGN.md new file mode 100644 index 0000000..a85b44c --- /dev/null +++ b/docs/SMART_MONITOR_DESIGN.md @@ -0,0 +1,199 @@ +# Smart Monitor Design Document + +## Overview +Design for an adaptive monitoring system that mimics human behavior and automatically switches modes based on detected activity. + +--- + +## Monitoring Modes + +### 1. STEALTH MODE (Default) +- **Check interval**: 45-90 seconds (randomized around 1 minute) +- **Behavior**: Human-like patterns, decoy requests +- **Purpose**: Avoid detection during normal monitoring + +### 2. ALERT MODE (Triggered by new SKU) +- **Check interval**: 15-30 seconds (faster, still randomized) +- **Duration**: 30-60 minutes after detection +- **Purpose**: Catch the drop going live quickly + +### 3. COOLDOWN MODE (After alert period) +- **Gradually slow back down** to stealth mode +- **Prevents sudden behavior change** which could trigger detection + +### 4. SLEEP MODE (Optional - night hours) +- **Check interval**: 5-10 minutes +- **Active hours**: e.g., 2am-6am local time +- **Purpose**: Real humans sleep, bots don't + +--- + +## Human-Like Obfuscation Techniques + +### Timing Randomization +```python +# Instead of exact intervals: +time.sleep(60) # BAD - robotic + +# Use gaussian distribution around target: +import random +base_interval = 60 +jitter = random.gauss(0, 10) # +/- 10 seconds standard deviation +time.sleep(max(30, base_interval + jitter)) # GOOD - human-like +``` + +### Decoy Requests +Mix in non-API requests to look like a real browser: +- Occasionally fetch an image from the site +- Load the homepage or a random category page +- Request favicon, robots.txt, etc. + +```python +DECOY_URLS = [ + "/favicon.ico", + "/", + "/category/plush", + "/category/accessories", +] + +def make_decoy_request(): + """Occasionally make a non-API request""" + if random.random() < 0.2: # 20% of the time + url = random.choice(DECOY_URLS) + session.get(API_BASE + url) +``` + +### Referrer Rotation +Change the Referer header to look like you're browsing: +```python +REFERRERS = [ + "https://www.pokemoncenter.com/", + "https://www.pokemoncenter.com/category/tcg-cards", + "https://www.pokemoncenter.com/category/new-releases", + "https://www.google.com/", +] + +headers["Referer"] = random.choice(REFERRERS) +``` + +### User-Agent Variation +Slightly vary the User-Agent (within reason): +```python +# Base UA with minor variations +chrome_versions = ["146.0.0.0", "145.0.0.0", "146.0.7680.154"] +version = random.choice(chrome_versions) +headers["User-Agent"] = f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{version} Safari/537.36" +``` + +### Session Breaks +Occasionally "take a break" like a human would: +```python +def maybe_take_break(): + """Occasionally pause for a longer period""" + if random.random() < 0.05: # 5% chance + break_duration = random.randint(120, 300) # 2-5 minutes + logger.info(f"Taking a break for {break_duration}s...") + time.sleep(break_duration) +``` + +### Time-of-Day Awareness +Adjust behavior based on time: +```python +from datetime import datetime + +def get_interval_for_time(): + hour = datetime.now().hour + + if 2 <= hour < 6: + # Late night - slower checks + return random.randint(300, 600) # 5-10 min + elif 9 <= hour < 17: + # Business hours - prime drop time + return random.randint(45, 90) # ~1 min + else: + # Evening/early morning + return random.randint(60, 120) # 1-2 min +``` + +--- + +## Mode Transition Logic + +``` + ┌─────────────────┐ + │ STEALTH MODE │ + │ (45-90 sec) │ + └────────┬────────┘ + │ + New SKU detected? + │ + YES ───────────┴─────────── NO + │ │ + ▼ │ + ┌─────────────────┐ │ + │ ALERT MODE │ │ + │ (15-30 sec) │◄──────────────────┘ + └────────┬────────┘ + │ + 30-60 min elapsed? + │ + YES │ + ▼ + ┌─────────────────┐ + │ COOLDOWN MODE │ + │ (gradually │ + │ slow down) │ + └────────┬────────┘ + │ + ▼ + Back to STEALTH MODE +``` + +--- + +## Detection Tracking + +When a new product is detected: + +1. **Log detection** with timestamp +2. **Fetch full product details** +3. **Send Discord alert** +4. **Enter ALERT MODE** +5. **Track if product becomes "announced"**: + - Visible on homepage + - Linked from category pages + - Social media announcement + +When product is announced, return to STEALTH MODE. + +--- + +## Implementation Priority + +1. [x] Basic backend monitor (done) +2. [ ] Add timing randomization +3. [ ] Implement mode switching +4. [ ] Add decoy requests +5. [ ] Add time-of-day awareness +6. [ ] Add session breaks +7. [ ] Integrate with Discord notifier + +--- + +## Testing Plan + +1. **Test with cookies**: Run warmup, verify API access works +2. **Test SKU detection**: Manually add a fake SKU to known list, verify detection +3. **Test mode switching**: Simulate detection, verify faster checks activate +4. **Monitor for blocks**: Run for extended period, note when cookies expire + +--- + +## Risk Mitigation + +| Risk | Mitigation | +|------|------------| +| Cookies expire | Auto-detect 403, prompt for re-warmup | +| IP blocked | Support proxy rotation (already built) | +| Pattern detected | Randomization + decoys + breaks | +| Rate limited | Back off on 429 responses | diff --git a/docs/backend-api-monitor.md b/docs/backend-api-monitor.md new file mode 100644 index 0000000..373708a --- /dev/null +++ b/docs/backend-api-monitor.md @@ -0,0 +1,285 @@ +# PokemonCenter Backend API Monitor + +## Overview + +This document outlines how to build a backend API monitor that detects new products before they appear on the website - similar to what accounts like @pokepullzhq do. + +## Why Backend Monitoring? + +- **Faster detection**: Products are loaded into the backend/API before the frontend displays them +- **Less aggressive**: API calls are lighter than full page loads +- **Avoids bot detection**: Direct API calls look different than browser automation +- **More reliable**: JSON responses are easier to parse than HTML + +--- + +## Step 1: Find the API Endpoints + +### How to Investigate + +1. Open https://www.pokemoncenter.com/category/tcg-cards in Chrome +2. Open DevTools (F12) +3. Go to **Network** tab +4. Check **Preserve log** +5. Filter by **Fetch/XHR** +6. Refresh the page +7. Look for API calls that return product data + +### What to Look For + +**Common patterns:** +- GraphQL endpoints: `/graphql` or `/api/graphql` +- REST APIs: `/api/products`, `/api/catalog`, `/api/search` +- Third-party services: + - **Algolia** (search): `*.algolia.net` or `*.algolianet.com` + - **Contentful** (CMS): `cdn.contentful.com` + - **Commercetools** (e-commerce): `*.commercetools.com` + - **Salesforce Commerce**: `*.demandware.net` + +**Signs you found the right endpoint:** +- Response contains product names, prices, SKUs +- Response has `inStock`, `availability`, or similar fields +- Response includes product URLs or IDs + +### Example Findings to Document + +For each endpoint found, note: +``` +URL: https://api.pokemoncenter.com/products?category=tcg +Method: GET +Headers: + - Authorization: Bearer xxx (if any) + - x-api-key: xxx (if any) +Response format: JSON +Contains: productId, name, price, availability, url +``` + +--- + +## Step 2: API Response Analysis + +Once you find the product API, analyze the response structure: + +### Key Fields to Track + +```json +{ + "products": [ + { + "id": "12345", // Unique product ID + "sku": "PKM-CR-ETB", // SKU code + "name": "Chaos Rising ETB", + "url": "/product/chaos-rising-etb", + "price": 54.99, + "availability": { + "inStock": false, // Current stock status + "preorder": true, // Pre-order available? + "releaseDate": "2026-04-15" + }, + "status": "ACTIVE", // May be "HIDDEN", "DRAFT" before launch + "publishedAt": null // null = not yet visible on site + } + ] +} +``` + +### Detection Strategies + +1. **New Product Detection** + - Compare product IDs against known list + - New ID = new product added to backend + +2. **Pre-Launch Detection** + - Product exists but `publishedAt` is null + - Product has `status: "DRAFT"` or similar + - Product `availability.inStock` changes from false to true + +3. **Stock Change Detection** + - Track `inStock` or `availability` field changes + +--- + +## Step 3: Implementation Plan + +### Option A: Add to Chrome Extension + +```javascript +// In background.js - add API monitoring alongside page monitoring + +async function checkBackendAPI() { + const API_URL = "https://api.pokemoncenter.com/products?category=tcg"; + + try { + const response = await fetch(API_URL, { + headers: { + // Add any required headers discovered during investigation + 'Accept': 'application/json', + } + }); + + const data = await response.json(); + const products = data.products || []; + + // Compare against known products + for (const product of products) { + if (!knownBackendProducts[product.id]) { + // NEW PRODUCT DETECTED! + await sendDiscordNotification({ + name: product.name, + url: `https://www.pokemoncenter.com${product.url}`, + price: `$${product.price}`, + inStock: product.availability?.inStock + }, "backend_detect"); + + knownBackendProducts[product.id] = product; + } + } + } catch (error) { + console.error("Backend API check failed:", error); + } +} +``` + +### Option B: Standalone Python Script + +```python +# backend_monitor.py +import requests +import time +import json +from discord_webhook import DiscordWebhook, DiscordEmbed + +API_URL = "https://api.pokemoncenter.com/products" +DISCORD_WEBHOOK = "your-webhook-url" +KNOWN_PRODUCTS_FILE = "known_backend_products.json" +CHECK_INTERVAL = 30 # seconds - can be faster for API + +def load_known_products(): + try: + with open(KNOWN_PRODUCTS_FILE, 'r') as f: + return json.load(f) + except FileNotFoundError: + return {} + +def save_known_products(products): + with open(KNOWN_PRODUCTS_FILE, 'w') as f: + json.dump(products, f) + +def check_api(): + headers = { + 'User-Agent': 'Mozilla/5.0...', + # Add discovered headers + } + + response = requests.get(API_URL, headers=headers) + return response.json() + +def send_alert(product, alert_type): + webhook = DiscordWebhook(url=DISCORD_WEBHOOK, content="@everyone") + + embed = DiscordEmbed( + title=f"🚨 {alert_type.upper()}", + description=f"**{product['name']}**", + color=0xFF0000 if alert_type == "BACKEND DETECT" else 0x00FF00 + ) + embed.add_embed_field(name="Price", value=f"${product.get('price', 'TBD')}") + embed.add_embed_field(name="Status", value=product.get('status', 'Unknown')) + embed.add_embed_field(name="Link", value=f"[VIEW]({product['url']})", inline=False) + + webhook.add_embed(embed) + webhook.execute() + +def main(): + known = load_known_products() + print(f"Loaded {len(known)} known products") + + while True: + try: + data = check_api() + products = data.get('products', []) + + for product in products: + pid = product['id'] + + if pid not in known: + print(f"NEW BACKEND PRODUCT: {product['name']}") + send_alert(product, "BACKEND DETECT") + known[pid] = product + save_known_products(known) + + # Check for status changes + elif known[pid].get('status') != product.get('status'): + print(f"STATUS CHANGE: {product['name']}") + send_alert(product, "STATUS CHANGE") + known[pid] = product + save_known_products(known) + + print(f"Checked {len(products)} products") + + except Exception as e: + print(f"Error: {e}") + + time.sleep(CHECK_INTERVAL) + +if __name__ == "__main__": + main() +``` + +--- + +## Step 4: Rate Limiting Considerations + +### API vs Page Scraping + +| Approach | Safe Interval | Notes | +|----------|---------------|-------| +| Full page load | 60-90 sec | Heavy, triggers bot detection | +| API call | 15-30 sec | Lighter, more tolerant | +| GraphQL query | 15-30 sec | Depends on complexity | + +### Best Practices + +1. **Use proper headers**: Include realistic User-Agent, Accept, etc. +2. **Don't hammer**: Even APIs have rate limits +3. **Cache responses**: Don't re-process unchanged data +4. **Handle 429s gracefully**: Back off when rate limited + +--- + +## Step 5: Advanced - Multiple Detection Layers + +For maximum coverage, run both: + +1. **Backend API Monitor** (every 30 sec) + - Fast detection of new products in system + - Lighter on resources + +2. **Frontend Page Monitor** (every 90 sec) + - Confirms products are live on website + - Catches anything API might miss + +### Alert Priority + +``` +BACKEND DETECT (API) = "Product loaded, drop imminent!" +FRONTEND DETECT (Page) = "Product is LIVE, buy now!" +RESTOCK = "Back in stock!" +``` + +--- + +## Next Steps + +1. **Wait for rate limit to clear** (~30 min) +2. **Investigate the API** using DevTools Network tab +3. **Document the endpoints** you find +4. **Share the findings** so we can build the monitor + +### Questions to Answer During Investigation + +- [ ] What URL serves product data? +- [ ] What headers are required? +- [ ] Is authentication needed? +- [ ] What does the response structure look like? +- [ ] Are there pagination parameters? +- [ ] Is there a "hidden" or "draft" status visible in API? diff --git a/main.py b/main.py index f61e738..d9e0c6c 100644 --- a/main.py +++ b/main.py @@ -16,14 +16,25 @@ from config import ( SITES_ENABLED, POKEMON_CENTER_URLS, TARGET_URLS, + GAMESTOP_URLS, + BESTBUY_URLS, + WALMART_URLS, KEYWORD_FILTER_ENABLED, KEYWORDS, LOG_LEVEL, + NEW_DROP_NOTIFICATIONS, + MAX_PAGES, +) +from src.browser import get_browser, shutdown_browser +from src.product_tracker import ProductTracker +from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification +from scrapers import ( + PokemonCenterScraper, + TargetScraper, + GameStopScraper, + BestBuyScraper, + WalmartScraper, ) -from browser import get_browser, shutdown_browser -from product_tracker import ProductTracker -from discord_notifier import send_stock_alert, send_startup_notification, send_error_notification -from scrapers import PokemonCenterScraper, TargetScraper # Setup logging logging.basicConfig( @@ -43,6 +54,9 @@ tracker = ProductTracker() scrapers = { "pokemoncenter": PokemonCenterScraper(), "target": TargetScraper(), + "gamestop": GameStopScraper(), + "bestbuy": BestBuyScraper(), + "walmart": WalmartScraper(), } @@ -54,7 +68,7 @@ def check_pokemoncenter(): try: for url in POKEMON_CENTER_URLS: logger.info(f"Scraping: {url}") - products = scraper.scrape_category_page(url) + products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1)) # Apply keyword filter if enabled if KEYWORD_FILTER_ENABLED and KEYWORDS: @@ -64,18 +78,19 @@ def check_pokemoncenter(): # Process and detect changes new_products, restocked_products = tracker.process_products(products) - # Send notifications for new products - for product in new_products: - if product.in_stock: - send_stock_alert( - product_name=product.name, - product_url=product.url, - price=product.price or "See link", - site="pokemoncenter", - alert_type="new_drop", - image_url=product.image_url, - ) - logger.info(f"Sent notification for new product: {product.name}") + # Send notifications for new products (if enabled) + if NEW_DROP_NOTIFICATIONS: + for product in new_products: + if product.in_stock: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="pokemoncenter", + alert_type="new_drop", + image_url=product.image_url, + ) + logger.info(f"Sent notification for new product: {product.name}") # Send notifications for restocks for product in restocked_products: @@ -110,7 +125,10 @@ def check_target(): try: for url in TARGET_URLS: logger.info(f"Scraping: {url}") - products = scraper.scrape_category_page(url) + products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("target", 1)) + + # Always filter to Pokemon products only + products = scraper.filter_pokemon_products(products) # Apply keyword filter if enabled if KEYWORD_FILTER_ENABLED and KEYWORDS: @@ -120,18 +138,19 @@ def check_target(): # Process and detect changes new_products, restocked_products = tracker.process_products(products) - # Send notifications for new products - for product in new_products: - if product.in_stock: - send_stock_alert( - product_name=product.name, - product_url=product.url, - price=product.price or "See link", - site="target", - alert_type="new_drop", - image_url=product.image_url, - ) - logger.info(f"Sent notification for new product: {product.name}") + # Send notifications for new products (if enabled) + if NEW_DROP_NOTIFICATIONS: + for product in new_products: + if product.in_stock: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="target", + alert_type="new_drop", + image_url=product.image_url, + ) + logger.info(f"Sent notification for new product: {product.name}") # Send notifications for restocks for product in restocked_products: @@ -158,6 +177,186 @@ def check_target(): send_error_notification(f"Error checking Target: {str(e)}", "Target") +def check_gamestop(): + """Check GameStop for restocks and new drops""" + logger.info("Checking GameStop...") + scraper = scrapers["gamestop"] + + try: + for url in GAMESTOP_URLS: + logger.info(f"Scraping: {url}") + products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("gamestop", 1)) + + # Always filter to Pokemon products only + products = scraper.filter_pokemon_products(products) + + # Apply keyword filter if enabled + if KEYWORD_FILTER_ENABLED and KEYWORDS: + products = scraper.filter_by_keywords(products, KEYWORDS) + logger.info(f"After keyword filter: {len(products)} products") + + # Process and detect changes + new_products, restocked_products = tracker.process_products(products) + + # Send notifications for new products (if enabled) + if NEW_DROP_NOTIFICATIONS: + for product in new_products: + if product.in_stock: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="gamestop", + alert_type="new_drop", + image_url=product.image_url, + ) + logger.info(f"Sent notification for new product: {product.name}") + + # Send notifications for restocks + for product in restocked_products: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="gamestop", + alert_type="restock", + image_url=product.image_url, + ) + logger.info(f"Sent notification for restock: {product.name}") + + # Log stats + stats = tracker.get_stats() + logger.info( + f"Stats: {stats['total_products']} total, " + f"{stats['in_stock']} in stock, " + f"{stats['out_of_stock']} out of stock" + ) + + except Exception as e: + logger.error(f"Error checking GameStop: {e}") + send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop") + + +def check_bestbuy(): + """Check Best Buy for restocks and new drops""" + logger.info("Checking Best Buy...") + scraper = scrapers["bestbuy"] + + try: + for url in BESTBUY_URLS: + logger.info(f"Scraping: {url}") + products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("bestbuy", 1)) + + # Always filter to Pokemon products only + products = scraper.filter_pokemon_products(products) + + # Apply keyword filter if enabled + if KEYWORD_FILTER_ENABLED and KEYWORDS: + products = scraper.filter_by_keywords(products, KEYWORDS) + logger.info(f"After keyword filter: {len(products)} products") + + # Process and detect changes + new_products, restocked_products = tracker.process_products(products) + + # Send notifications for new products (if enabled) + if NEW_DROP_NOTIFICATIONS: + for product in new_products: + if product.in_stock: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="bestbuy", + alert_type="new_drop", + image_url=product.image_url, + ) + logger.info(f"Sent notification for new product: {product.name}") + + # Send notifications for restocks + for product in restocked_products: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="bestbuy", + alert_type="restock", + image_url=product.image_url, + ) + logger.info(f"Sent notification for restock: {product.name}") + + # Log stats + stats = tracker.get_stats() + logger.info( + f"Stats: {stats['total_products']} total, " + f"{stats['in_stock']} in stock, " + f"{stats['out_of_stock']} out of stock" + ) + + except Exception as e: + logger.error(f"Error checking Best Buy: {e}") + send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy") + + +def check_walmart(): + """Check Walmart for restocks and new drops""" + logger.info("Checking Walmart...") + scraper = scrapers["walmart"] + + try: + for url in WALMART_URLS: + logger.info(f"Scraping: {url}") + products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("walmart", 1)) + + # Always filter to Pokemon products only + products = scraper.filter_pokemon_products(products) + + # Apply keyword filter if enabled + if KEYWORD_FILTER_ENABLED and KEYWORDS: + products = scraper.filter_by_keywords(products, KEYWORDS) + logger.info(f"After keyword filter: {len(products)} products") + + # Process and detect changes + new_products, restocked_products = tracker.process_products(products) + + # Send notifications for new products (if enabled) + if NEW_DROP_NOTIFICATIONS: + for product in new_products: + if product.in_stock: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="walmart", + alert_type="new_drop", + image_url=product.image_url, + ) + logger.info(f"Sent notification for new product: {product.name}") + + # Send notifications for restocks + for product in restocked_products: + send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price or "See link", + site="walmart", + alert_type="restock", + image_url=product.image_url, + ) + logger.info(f"Sent notification for restock: {product.name}") + + # Log stats + stats = tracker.get_stats() + logger.info( + f"Stats: {stats['total_products']} total, " + f"{stats['in_stock']} in stock, " + f"{stats['out_of_stock']} out of stock" + ) + + except Exception as e: + logger.error(f"Error checking Walmart: {e}") + send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart") + + def run_checks(): """Run all enabled site checks""" logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}") @@ -168,6 +367,15 @@ def run_checks(): if "target" in SITES_ENABLED: check_target() + if "gamestop" in SITES_ENABLED: + check_gamestop() + + if "bestbuy" in SITES_ENABLED: + check_bestbuy() + + if "walmart" in SITES_ENABLED: + check_walmart() + logger.info("Check cycle complete") diff --git a/product_tracker.py b/product_tracker.py deleted file mode 100644 index 18c7f79..0000000 --- a/product_tracker.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Product tracker - keeps track of known products to detect new drops and restocks -""" - -import json -import logging -from pathlib import Path -from typing import Dict, List, Set, Optional -from dataclasses import asdict -from datetime import datetime - -from scrapers.base import Product - -logger = logging.getLogger(__name__) - -# File to store known products -PRODUCTS_FILE = Path(__file__).parent / "products.json" - - -class ProductTracker: - """Tracks known products to detect new listings and stock changes""" - - def __init__(self, products_file: Path = PRODUCTS_FILE): - self.products_file = products_file - self.products: Dict[str, dict] = {} # URL -> product data - self.load() - - def load(self): - """Load products from file""" - if self.products_file.exists(): - try: - with open(self.products_file, "r", encoding="utf-8") as f: - self.products = json.load(f) - logger.info(f"Loaded {len(self.products)} tracked products") - except (json.JSONDecodeError, IOError) as e: - logger.error(f"Error loading products file: {e}") - self.products = {} - else: - self.products = {} - logger.info("No existing products file, starting fresh") - - def save(self): - """Save products to file""" - try: - with open(self.products_file, "w", encoding="utf-8") as f: - json.dump(self.products, f, indent=2, ensure_ascii=False) - logger.debug(f"Saved {len(self.products)} products") - except IOError as e: - logger.error(f"Error saving products file: {e}") - - def process_products(self, products: List[Product]) -> tuple[List[Product], List[Product]]: - """ - Process a list of scraped products and detect changes - - Args: - products: List of products from scraper - - Returns: - Tuple of (new_products, restocked_products) - """ - new_products = [] - restocked_products = [] - - for product in products: - url = product.url - - if url not in self.products: - # New product! - new_products.append(product) - self.products[url] = { - "name": product.name, - "url": url, - "price": product.price, - "in_stock": product.in_stock, - "image_url": product.image_url, - "site": product.site, - "product_id": product.product_id, - "first_seen": datetime.now().isoformat(), - "last_seen": datetime.now().isoformat(), - "last_in_stock": datetime.now().isoformat() if product.in_stock else None, - } - logger.info(f"NEW PRODUCT: {product.name}") - - else: - # Existing product - check for restock - existing = self.products[url] - was_in_stock = existing.get("in_stock", False) - - # Update last seen - existing["last_seen"] = datetime.now().isoformat() - existing["price"] = product.price or existing.get("price") - existing["image_url"] = product.image_url or existing.get("image_url") - - if product.in_stock and not was_in_stock: - # RESTOCK! - restocked_products.append(product) - existing["last_in_stock"] = datetime.now().isoformat() - logger.info(f"RESTOCK: {product.name}") - - existing["in_stock"] = product.in_stock - self.products[url] = existing - - self.save() - return new_products, restocked_products - - def get_known_urls(self) -> Set[str]: - """Get all known product URLs""" - return set(self.products.keys()) - - def get_product(self, url: str) -> Optional[dict]: - """Get a specific product by URL""" - return self.products.get(url) - - def mark_out_of_stock(self, url: str): - """Mark a product as out of stock""" - if url in self.products: - self.products[url]["in_stock"] = False - self.save() - - def clear(self): - """Clear all tracked products""" - self.products = {} - self.save() - logger.info("Cleared all tracked products") - - def get_stats(self) -> dict: - """Get tracking statistics""" - total = len(self.products) - in_stock = sum(1 for p in self.products.values() if p.get("in_stock", False)) - out_of_stock = total - in_stock - - return { - "total_products": total, - "in_stock": in_stock, - "out_of_stock": out_of_stock, - } diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..8cff5f3 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning diff --git a/requirements.txt b/requirements.txt index c88db0d..d6aba90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,13 @@ playwright-stealth>=1.0.6 requests>=2.31.0 schedule>=1.2.1 beautifulsoup4>=4.12.0 +flask>=3.0.0 +flask-cors>=4.0.0 +psutil>=5.9.0 +undetected-chromedriver>=3.5.0 +selenium>=4.15.0 +setuptools>=70.0.0 # Required for Python 3.13+ (distutils compatibility) +discord.py>=2.3.0 # Discord bot for location-based store search +nltk>=3.8.0 # Sentiment analysis using VADER +pytest>=8.0.0 # Testing framework +pytest-cov>=4.0.0 # Coverage reporting diff --git a/scrapers/__init__.py b/scrapers/__init__.py index dd14901..4414ce3 100644 --- a/scrapers/__init__.py +++ b/scrapers/__init__.py @@ -1,5 +1,15 @@ # Scrapers package from .pokemoncenter import PokemonCenterScraper from .target import TargetScraper +from .gamestop import GameStopScraper, warmup_gamestop +from .bestbuy import BestBuyScraper +from .walmart import WalmartScraper -__all__ = ["PokemonCenterScraper", "TargetScraper"] +__all__ = [ + "PokemonCenterScraper", + "TargetScraper", + "GameStopScraper", + "BestBuyScraper", + "WalmartScraper", + "warmup_gamestop", +] diff --git a/scrapers/base.py b/scrapers/base.py index d70c017..abde8a2 100644 --- a/scrapers/base.py +++ b/scrapers/base.py @@ -35,6 +35,50 @@ class BaseScraper(ABC): site_name: str = "unknown" + # Terms that indicate a Pokemon product + POKEMON_TERMS = [ + "pokemon", "pokémon", "poke", "tcg", + "pikachu", "charizard", "mewtwo", "eevee", "snorlax", + "booster", "elite trainer", "etb", + "scarlet", "violet", "prismatic", "evolutions", + ] + + # Terms that indicate NOT a Pokemon product (false positives from search) + EXCLUDE_TERMS = [ + "ice cube", "oven", "barbie", "hot wheels", "lego", + "furniture", "appliance", "kitchen", "bedding", + "glitter girls", "masters of the universe", "transformers", + "room essentials", "threshold", + ] + + def is_pokemon_product(self, product: Product) -> bool: + """ + Check if a product is actually a Pokemon product. + Filters out false positives from search results. + """ + name_lower = product.name.lower() + + # Check for exclusion terms first + for term in self.EXCLUDE_TERMS: + if term in name_lower: + return False + + # Check for Pokemon terms + for term in self.POKEMON_TERMS: + if term in name_lower: + return True + + # If no Pokemon terms found, reject it + return False + + def filter_pokemon_products(self, products: List[Product]) -> List[Product]: + """Filter to only include valid Pokemon products""" + filtered = [p for p in products if self.is_pokemon_product(p)] + rejected = len(products) - len(filtered) + if rejected > 0: + logger.info(f"Filtered out {rejected} non-Pokemon products") + return filtered + @abstractmethod def scrape_category_page(self, url: str) -> List[Product]: """ diff --git a/scrapers/bestbuy.py b/scrapers/bestbuy.py new file mode 100644 index 0000000..8dcb5cb --- /dev/null +++ b/scrapers/bestbuy.py @@ -0,0 +1,505 @@ +""" +Best Buy scraper - Uses undetected-chromedriver to bypass bot protection +""" + +import re +import json +import logging +import time +import sys +import os +from typing import List, Optional +from bs4 import BeautifulSoup + +from .base import BaseScraper, Product + +# Add parent directory to path for tools import +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +logger = logging.getLogger(__name__) + + +class BestBuyScraper(BaseScraper): + """Scraper for BestBuy.com - Uses undetected-chromedriver""" + + site_name = "bestbuy" + base_url = "https://www.bestbuy.com" + + def __init__(self): + self._stealth_browser = None + + def _get_stealth_browser(self): + """Get or create stealth browser for Best Buy""" + if self._stealth_browser is None: + from tools.stealth_browser import StealthBrowser + self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy") + self._stealth_browser.start() + return self._stealth_browser + + def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]: + """ + Scrape a Best Buy search/category page for all products + Uses undetected-chromedriver to bypass bot protection + + Args: + url: Search/category page URL + max_pages: Maximum number of pages to scrape (default 1) + + Returns: + List of Product objects + """ + all_products = [] + + # Add sort by newest if not already in URL + if "sort=" not in url: + separator = "&" if "?" in url else "?" + url = f"{url}{separator}sp=-releasedate" + + # Use stealth browser + try: + browser = self._get_stealth_browser() + except Exception as e: + logger.error(f"Failed to start stealth browser: {e}") + return all_products + + for page_num in range(1, max_pages + 1): + products = [] + page_url = url if page_num == 1 else f"{url}&cp={page_num}" + + try: + logger.info(f"Navigating to: {page_url}") + browser.driver.get(page_url) + time.sleep(8) # Wait longer for initial page load + + # Scroll for lazy loading - wait longer between scrolls + browser.driver.execute_script("window.scrollTo(0, 500)") + time.sleep(2) + browser.driver.execute_script("window.scrollTo(0, 1500)") + time.sleep(2) + browser.driver.execute_script("window.scrollTo(0, 3000)") + time.sleep(2) + browser.driver.execute_script("window.scrollTo(0, 0)") # Scroll back to top + time.sleep(3) + + # Wait for products to load (check for content) + for _ in range(10): + html = browser.driver.page_source + if "sku-item" in html or "sku-title" in html or "priceView" in html: + logger.info("Product content detected") + break + time.sleep(1) + + # Get HTML after waiting + html = browser.driver.page_source + + # Save debug screenshot + try: + browser.driver.save_screenshot("debug_bestbuy.png") + logger.info("Saved debug screenshot to debug_bestbuy.png") + except: + pass + + soup = BeautifulSoup(html, "html.parser") + + # Try to find product data in JSON scripts + scripts = soup.find_all("script", type="application/json") + for script in scripts: + try: + data = json.loads(script.string) + products.extend(self._extract_products_from_json(data)) + except (json.JSONDecodeError, TypeError): + continue + + # Try multiple selectors for product cards - Best Buy updates these frequently + product_cards = ( + soup.select("li.sku-item") + or soup.select("[data-sku-id]") + or soup.select(".sku-item") + or soup.select("[class*='sku-item']") + or soup.select("div.shop-sku-list-item") + or soup.select("[class*='productCard']") + or soup.select("[class*='product-card']") + or soup.select(".list-item") + ) + + logger.info(f"Found {len(product_cards)} product cards on page {page_num}") + + for card in product_cards: + product = self._parse_product_card(card) + if product: + products.append(product) + + # Fallback: parse product links + if not product_cards: + product_links = soup.select("a[href*='/site/'][href*='.p']") + logger.info(f"Fallback: Found {len(product_links)} product links") + + href_to_links = {} + for link in product_links: + href = link.get("href", "") + if not href or ".p" not in href: + continue + if href not in href_to_links: + href_to_links[href] = [] + href_to_links[href].append(link) + + for href, links in href_to_links.items(): + best_link = links[0] + for link in links: + if link.get("aria-label") or len(link.get_text(strip=True)) > 10: + best_link = link + break + + product = self._parse_product_link(best_link, soup) + if product: + products.append(product) + + # Also try parsing from data attributes and script tags + if not products: + products.extend(self._extract_from_page_data(soup, browser)) + + # Don't close - reuse browser for next page + + except Exception as e: + logger.error(f"Error scraping Best Buy page {page_num}: {e}") + break # Don't raise - just return what we have + + all_products.extend(products) + + # Stop if no products found on this page (no more pages) + if not products: + break + + # Remove duplicates + seen_urls = set() + unique_products = [] + for p in all_products: + if p.url not in seen_urls: + seen_urls.add(p.url) + unique_products.append(p) + + logger.info(f"Scraped {len(unique_products)} unique products from Best Buy") + return unique_products + + def _extract_from_page_data(self, soup, browser) -> List[Product]: + """Extract products from page data attributes and evaluate JS if needed""" + products = [] + + # Try to get product data from data attributes + items_with_data = soup.select("[data-testid][data-sku-id]") + for item in items_with_data: + sku_id = item.get("data-sku-id", "") + if sku_id: + # Find name and price within this element + name_elem = item.select_one("h4") or item.select_one("[class*='title']") or item.select_one("a") + name = name_elem.get_text(strip=True) if name_elem else "" + + if name and len(name) > 5: + price_text = item.get_text() + price_match = re.search(r"\$[\d,]+\.?\d*", price_text) + price = price_match.group() if price_match else None + + products.append(Product( + name=name, + url=f"{self.base_url}/site/{sku_id}.p", + price=price, + in_stock=True, + image_url=None, + site=self.site_name, + product_id=sku_id, + )) + + # Try to extract from window.__INITIAL_STATE__ or similar JS objects + try: + initial_state = browser.driver.execute_script(""" + if (window.__INITIAL_STATE__) return JSON.stringify(window.__INITIAL_STATE__); + if (window.__NEXT_DATA__) return JSON.stringify(window.__NEXT_DATA__); + return null; + """) + if initial_state: + data = json.loads(initial_state) + products.extend(self._extract_products_from_json(data)) + except Exception as e: + logger.debug(f"Could not extract from JS state: {e}") + + return products + + def _extract_products_from_json(self, data, depth=0) -> List[Product]: + """Recursively search JSON for product data""" + products = [] + if depth > 10: + return products + + if isinstance(data, dict): + # Check if this looks like a Best Buy product + if "skuId" in data or ("name" in data and "regularPrice" in data): + product = self._parse_product_json(data) + if product: + products.append(product) + + for value in data.values(): + products.extend(self._extract_products_from_json(value, depth + 1)) + + elif isinstance(data, list): + for item in data: + products.extend(self._extract_products_from_json(item, depth + 1)) + + return products + + def _parse_product_json(self, data: dict) -> Optional[Product]: + """Parse a product from Best Buy's JSON data""" + try: + name = data.get("name") or data.get("displayName", "") + if not name: + return None + + sku_id = data.get("skuId") or data.get("sku", "") + url_slug = data.get("url") or "" + + if url_slug: + url = url_slug if url_slug.startswith("http") else f"{self.base_url}{url_slug}" + elif sku_id: + url = f"{self.base_url}/site/{sku_id}.p" + else: + return None + + # Get price + price = None + if "regularPrice" in data: + price = f"${data['regularPrice']:.2f}" + elif "salePrice" in data: + price = f"${data['salePrice']:.2f}" + + # Check availability + in_stock = True + availability = data.get("availability", {}) + if isinstance(availability, dict): + in_stock = availability.get("isAvailable", True) + elif data.get("orderable") is False: + in_stock = False + + # Get image + image_url = data.get("image") or data.get("thumbnailImage") + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=str(sku_id), + ) + + except Exception as e: + logger.debug(f"Error parsing Best Buy product JSON: {e}") + return None + + def _parse_product_card(self, card) -> Optional[Product]: + """Parse a product card element""" + try: + # Find link + link = card.select_one("a[href*='/site/'][href*='.p']") or card.select_one("a.image-link") + if not link: + link = card.select_one("a") + if not link: + return None + + href = link.get("href", "") + if not href: + return None + url = href if href.startswith("http") else f"{self.base_url}{href}" + + # Get name + name_elem = ( + card.select_one(".sku-title a") + or card.select_one("h4.sku-header a") + or card.select_one("[data-testid='product-title']") + or card.select_one(".sku-title") + or link + ) + name = name_elem.get_text(strip=True) if name_elem else "Unknown" + + # Skip if name too short + if len(name) < 5: + return None + + # Get price + price_elem = ( + card.select_one(".priceView-customer-price span") + or card.select_one("[data-testid='customer-price']") + or card.select_one(".pricing-price__regular-price") + or card.select_one("[class*='price']") + ) + price = None + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + # Check stock + in_stock = self._check_card_stock_status(card) + + # Get image + img = card.select_one("img") + image_url = None + if img: + image_url = img.get("src") or img.get("data-src") + if image_url and not image_url.startswith("http"): + image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}" + + # Extract SKU ID from URL + product_id = "" + match = re.search(r"/(\d+)\.p", url) + if match: + product_id = match.group(1) + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing Best Buy product card: {e}") + return None + + def _parse_product_link(self, link, soup) -> Optional[Product]: + """Parse a product from a product link element""" + try: + href = link.get("href", "") + if not href: + return None + + url = href if href.startswith("http") else f"{self.base_url}{href}" + + name = link.get("aria-label", "") or link.get_text(strip=True) + + if not name or len(name) < 5: + return None + + # Extract SKU ID + product_id = "" + match = re.search(r"/(\d+)\.p", url) + if match: + product_id = match.group(1) + + # Find price near link + parent = link.find_parent() + price = None + for _ in range(5): + if parent: + price_text = parent.get_text() + price_match = re.search(r"\$[\d,]+\.?\d*", price_text) + if price_match: + price = price_match.group() + break + parent = parent.find_parent() + + # Find image + image_url = None + parent = link.find_parent() + for _ in range(5): + if parent: + img = parent.select_one("img") + if img: + image_url = img.get("src") or img.get("data-src") + break + parent = parent.find_parent() + + # Check stock + in_stock = True + parent = link.find_parent() + for _ in range(5): + if parent: + text = parent.get_text().lower() + if any(phrase in text for phrase in ["sold out", "out of stock", "unavailable"]): + in_stock = False + break + parent = parent.find_parent() + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing Best Buy product link: {e}") + return None + + def _check_card_stock_status(self, card) -> bool: + """Check if a product card indicates in-stock status""" + card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower() + + # Check for disabled add to cart button + add_btn = card.select_one(".add-to-cart-button") + if add_btn and "btn-disabled" in add_btn.get("class", []): + return False + + out_of_stock_phrases = [ + "sold out", + "out of stock", + "unavailable", + "coming soon", + "not available", + ] + + for phrase in out_of_stock_phrases: + if phrase in card_text: + return False + + in_stock_phrases = [ + "add to cart", + "add to bag", + "available", + "in stock", + ] + + for phrase in in_stock_phrases: + if phrase in card_text: + return True + + return True + + def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]: + """Check if a specific product is in stock""" + try: + browser = self._get_stealth_browser() + browser.driver.get(product_url) + time.sleep(3) + html = browser.driver.page_source + soup = BeautifulSoup(html, "html.parser") + page_text = soup.get_text().lower() + + out_of_stock = any( + phrase in page_text + for phrase in ["sold out", "out of stock", "unavailable", "coming soon"] + ) + + in_stock = not out_of_stock + + # Get price + price = None + price_elem = ( + soup.select_one(".priceView-customer-price span") + or soup.select_one("[data-testid='customer-price']") + ) + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + return in_stock, price + + except Exception as e: + logger.error(f"Error checking Best Buy product stock: {e}") + return False, None diff --git a/scrapers/gamestop.py b/scrapers/gamestop.py new file mode 100644 index 0000000..9937530 --- /dev/null +++ b/scrapers/gamestop.py @@ -0,0 +1,478 @@ +""" +GameStop.com scraper +""" + +import re +import logging +import time +import sys +import os +from typing import List, Optional +from bs4 import BeautifulSoup + +from .base import BaseScraper, Product +from src.browser import get_browser +from config import CAPTCHA_WAIT_TIMEOUT + +# Add parent directory to path for tools import +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +logger = logging.getLogger(__name__) + + +def warmup_gamestop(): + """ + Warmup function to solve Cloudflare CAPTCHA manually. + Uses undetected-chromedriver to bypass bot detection. + """ + from tools.stealth_browser import StealthBrowser + + print("\n" + "=" * 60) + print("GAMESTOP WARMUP - Using Undetected Chrome") + print("=" * 60) + print("A browser window should open.") + print("If you see a Cloudflare challenge, solve it manually.") + print(f"Waiting up to {CAPTCHA_WAIT_TIMEOUT} seconds...") + print("=" * 60 + "\n") + + logger.info("Starting GameStop warmup with undetected-chromedriver...") + + # Use stealth browser instead of Playwright + browser = StealthBrowser(headless=False, session_name="gamestop") + + try: + browser.start() + logger.info("Stealth browser started") + + # Navigate to GameStop + browser.driver.get("https://www.gamestop.com") + time.sleep(3) + + # Check for Cloudflare challenge + start_time = time.time() + challenge_detected = False + + while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT: + try: + html = browser.driver.page_source + html_lower = html.lower() + title = browser.driver.title.lower() + + # Cloudflare challenge indicators + is_cloudflare_challenge = ( + "just a moment" in title or + "checking your browser" in html_lower or + "cf-challenge" in html_lower or + "turnstile" in html_lower or + (len(html) < 5000 and "challenge" in html_lower) + ) + + if is_cloudflare_challenge: + if not challenge_detected: + challenge_detected = True + print(">>> Cloudflare challenge detected! Please solve it in the browser window.") + logger.info("Cloudflare challenge detected - waiting for manual solve...") + time.sleep(5) + else: + # Check if we're on actual GameStop content + if len(html) > 10000 and "gamestop" in html_lower: + print(">>> Challenge solved! GameStop page loaded successfully.") + logger.info("GameStop warmup complete!") + browser.stop() + return True + time.sleep(2) + except Exception as e: + logger.debug(f"Error checking page: {e}") + time.sleep(2) + + logger.warning("GameStop warmup timed out - CAPTCHA may not be solved") + print(">>> Warmup timed out. You may need to try again.") + browser.stop() + return False + + except Exception as e: + logger.error(f"Error during warmup: {e}") + try: + browser.stop() + except: + pass + return False + + +class GameStopScraper(BaseScraper): + """Scraper for GameStop.com - Uses undetected-chromedriver to bypass Cloudflare""" + + site_name = "gamestop" + base_url = "https://www.gamestop.com" + + def __init__(self): + self._stealth_browser = None + + def _get_stealth_browser(self): + """Get or create stealth browser for GameStop""" + if self._stealth_browser is None: + from tools.stealth_browser import StealthBrowser + self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop") + self._stealth_browser.start() + return self._stealth_browser + + def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]: + """ + Scrape a GameStop search/category page for all products + Uses undetected-chromedriver to bypass Cloudflare + + Args: + url: Search/category page URL + max_pages: Maximum number of pages to scrape (default 1) + + Returns: + List of Product objects + """ + all_products = [] + + # Add sort by newest if not already in URL + if "sort=" not in url.lower(): + separator = "&" if "?" in url else "?" + url = f"{url}{separator}sort=date-desc" + + # Use stealth browser for GameStop + try: + browser = self._get_stealth_browser() + except Exception as e: + logger.error(f"Failed to start stealth browser: {e}") + return all_products + + for page_num in range(1, max_pages + 1): + products = [] + page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}" + + try: + logger.info(f"Navigating to: {page_url}") + browser.driver.get(page_url) + time.sleep(5) # Wait for page load + + html = browser.driver.page_source + + # Check for Cloudflare challenge + html_lower = html.lower() + title = browser.driver.title.lower() + + is_cloudflare = ( + "just a moment" in title or + "checking your browser" in html_lower or + "cf-challenge" in html_lower + ) + + if is_cloudflare: + logger.warning("Cloudflare challenge detected! Waiting for manual solve...") + start_time = time.time() + while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT: + time.sleep(5) + html = browser.driver.page_source + html_lower = html.lower() + title = browser.driver.title.lower() + if "just a moment" not in title and "cf-challenge" not in html_lower: + logger.info("Cloudflare challenge solved!") + break + else: + logger.error("Cloudflare challenge not solved in time") + return all_products + + # Save debug screenshot + try: + browser.driver.save_screenshot("debug_gamestop.png") + logger.info("Saved debug screenshot to debug_gamestop.png") + except: + pass + + soup = BeautifulSoup(html, "html.parser") + + # Try multiple selectors for product cards + product_cards = ( + soup.select(".product-tile") + or soup.select("[data-testid='product-tile']") + or soup.select(".grid-tile") + or soup.select(".product-grid-tile") + or soup.select("[class*='ProductTile']") + ) + + logger.info(f"Found {len(product_cards)} product cards") + + # If no cards found, try link-based extraction + if not product_cards: + product_links = soup.select("a[href*='/products/']") + logger.info(f"Fallback: Found {len(product_links)} product links") + + # Group by href + href_to_links = {} + for link in product_links: + href = link.get("href", "") + if not href or "/products/" not in href: + continue + if href not in href_to_links: + href_to_links[href] = [] + href_to_links[href].append(link) + + for href, links in href_to_links.items(): + # Pick best link + best_link = links[0] + for link in links: + if link.get("aria-label") or len(link.get_text(strip=True)) > 10: + best_link = link + break + + product = self._parse_product_link(best_link, soup) + if product: + products.append(product) + else: + for card in product_cards: + product = self._parse_product_card(card) + if product: + products.append(product) + + # Don't close - reuse browser for next page + + except Exception as e: + logger.error(f"Error scraping GameStop page {page_num}: {e}") + break # Don't raise - just return what we have + + all_products.extend(products) + + # Stop if no products found on this page + if not products: + break + + # Remove duplicates + seen_urls = set() + unique_products = [] + for p in all_products: + if p.url not in seen_urls: + seen_urls.add(p.url) + unique_products.append(p) + + logger.info(f"Scraped {len(unique_products)} unique products from GameStop") + return unique_products + + def _parse_product_card(self, card) -> Optional[Product]: + """Parse a product card element""" + try: + # Find link - try multiple patterns + link = ( + card.select_one("a[href*='/products/']") or + card.select_one("a[href*='/video-games/']") or + card.select_one("a[href*='/collectibles/']") or + card.select_one("a.product-tile-link") or + card.select_one("a") + ) + if not link: + logger.debug("No link found in card") + return None + + href = link.get("href", "") + if not href: + logger.debug("Empty href") + return None + url = href if href.startswith("http") else f"{self.base_url}{href}" + + # Get name - try multiple patterns + name_elem = ( + card.select_one(".product-tile-title") + or card.select_one(".product-name a") + or card.select_one(".product-name") + or card.select_one(".product-title") + or card.select_one("[data-testid='product-name']") + or card.select_one("a[aria-label]") + or link + ) + + # Try to get name from aria-label first + name = "" + if name_elem: + name = name_elem.get("aria-label", "") or name_elem.get_text(strip=True) + + # Skip if name too short + if len(name) < 5: + logger.debug(f"Name too short: '{name}' from {url[:50]}") + return None + + # Get price + price_elem = ( + card.select_one(".price-sales") + or card.select_one(".product-price") + or card.select_one("[data-testid='price']") + or card.select_one("[class*='price']") + ) + price = None + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + # Check stock + in_stock = self._check_card_stock_status(card) + + # Get image + img = card.select_one("img") + image_url = None + if img: + image_url = img.get("src") or img.get("data-src") or img.get("data-lazy") + if image_url and not image_url.startswith("http"): + image_url = f"{self.base_url}{image_url}" + + # Extract product ID from URL + product_id = "" + match = re.search(r"/products/[^/]+/(\d+)", url) + if match: + product_id = match.group(1) + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing GameStop product card: {e}") + return None + + def _parse_product_link(self, link, soup) -> Optional[Product]: + """Parse a product from a product link element""" + try: + href = link.get("href", "") + if not href: + return None + + url = href if href.startswith("http") else f"{self.base_url}{href}" + + # Get name + name = link.get("aria-label", "") or link.get_text(strip=True) + + # Skip if name too short + if not name or len(name) < 5: + return None + + # Extract product ID from URL + product_id = "" + match = re.search(r"/products/[^/]+/(\d+)", url) + if match: + product_id = match.group(1) + + # Try to find price near this link + parent = link.find_parent() + price = None + for _ in range(5): + if parent: + price_text = parent.get_text() + price_match = re.search(r"\$[\d,]+\.?\d*", price_text) + if price_match: + price = price_match.group() + break + parent = parent.find_parent() + + # Try to find image + image_url = None + parent = link.find_parent() + for _ in range(5): + if parent: + img = parent.select_one("img") + if img: + image_url = img.get("src") or img.get("data-src") + if image_url and not image_url.startswith("http"): + image_url = f"{self.base_url}{image_url}" + break + parent = parent.find_parent() + + # Check stock + in_stock = True + parent = link.find_parent() + for _ in range(5): + if parent: + text = parent.get_text().lower() + if any(phrase in text for phrase in ["sold out", "out of stock", "not available"]): + in_stock = False + break + parent = parent.find_parent() + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing GameStop product link: {e}") + return None + + def _check_card_stock_status(self, card) -> bool: + """Check if a product card indicates in-stock status""" + card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower() + + out_of_stock_phrases = [ + "sold out", + "out of stock", + "not available", + "unavailable", + "currently unavailable", + ] + + for phrase in out_of_stock_phrases: + if phrase in card_text: + return False + + in_stock_phrases = [ + "add to cart", + "add to bag", + "available", + "buy now", + "in stock", + ] + + for phrase in in_stock_phrases: + if phrase in card_text: + return True + + # Default: assume in stock if listed + return True + + def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]: + """Check if a specific product is in stock""" + try: + browser = self._get_stealth_browser() + browser.driver.get(product_url) + time.sleep(3) + html = browser.driver.page_source + soup = BeautifulSoup(html, "html.parser") + page_text = soup.get_text().lower() + + out_of_stock = any( + phrase in page_text + for phrase in ["sold out", "out of stock", "not available", "unavailable"] + ) + + in_stock = not out_of_stock + + # Get price + price = None + price_elem = ( + soup.select_one(".product-price") + or soup.select_one(".price-sales") + or soup.select_one("[data-testid='price']") + ) + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + return in_stock, price + + except Exception as e: + logger.error(f"Error checking GameStop product stock: {e}") + return False, None diff --git a/scrapers/pokemoncenter.py b/scrapers/pokemoncenter.py index 87bccd4..5dc0316 100644 --- a/scrapers/pokemoncenter.py +++ b/scrapers/pokemoncenter.py @@ -1,6 +1,6 @@ """ PokemonCenter.com scraper -Handles bot protection with Playwright stealth +Handles bot protection with undetected-chromedriver stealth browser """ import re @@ -9,44 +9,83 @@ from typing import List, Optional from bs4 import BeautifulSoup from .base import BaseScraper, Product -from browser import get_browser +import config logger = logging.getLogger(__name__) +def get_stealth_or_regular_browser(): + """Get appropriate browser based on config""" + if config.USE_STEALTH_BROWSER: + from tools.stealth_browser import get_stealth_browser + return get_stealth_browser( + headless=config.STEALTH_HEADLESS, + session_name=config.STEALTH_SESSION_NAME + ), True + else: + from src.browser import get_browser + return get_browser(), False + + class PokemonCenterScraper(BaseScraper): """Scraper for PokemonCenter.com""" site_name = "pokemoncenter" base_url = "https://www.pokemoncenter.com" - def scrape_category_page(self, url: str) -> List[Product]: + def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]: """ Scrape a PokemonCenter category page for all products Args: url: Category page URL + max_pages: Maximum pages to scrape (not yet implemented for PokemonCenter) Returns: List of Product objects """ - browser = get_browser() + # TODO: Implement pagination for PokemonCenter when needed + browser, is_stealth = get_stealth_or_regular_browser() products = [] try: - # Navigate and wait for page to load (don't wait for specific selector) - page, html = browser.get_page_content( - url, - wait_for_selector=None, # Let it use networkidle instead - timeout=60000, - ) + if is_stealth: + # Use stealth browser (undetected-chromedriver) + html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX) - # Save screenshot for debugging if needed - try: - page.screenshot(path="debug_screenshot.png") - logger.info("Saved debug screenshot to debug_screenshot.png") - except: - pass + # Check for CAPTCHA + if browser.check_for_captcha(): + logger.warning("CAPTCHA detected on PokemonCenter!") + if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS: + browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT) + # Re-fetch page after CAPTCHA solve + html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX) + else: + logger.error("Cannot solve CAPTCHA in headless mode") + return [] + + # Save screenshot for debugging + try: + browser.screenshot("debug_screenshot.png") + logger.info("Saved debug screenshot to debug_screenshot.png") + except: + pass + + page = None # Stealth browser doesn't return page object + else: + # Use regular Playwright browser + page, html = browser.get_page_content( + url, + wait_for_selector=None, # Let it use networkidle instead + timeout=60000, + ) + + # Save screenshot for debugging if needed + try: + page.screenshot(path="debug_screenshot.png") + logger.info("Saved debug screenshot to debug_screenshot.png") + except: + pass soup = BeautifulSoup(html, "html.parser") @@ -96,7 +135,9 @@ class PokemonCenterScraper(BaseScraper): if product: products.append(product) - page.close() + # Close page only for Playwright browser + if page is not None: + page.close() except Exception as e: logger.error(f"Error scraping PokemonCenter category page: {e}") @@ -224,14 +265,30 @@ class PokemonCenterScraper(BaseScraper): Returns: Tuple of (is_in_stock, price) """ - browser = get_browser() + browser, is_stealth = get_stealth_or_regular_browser() try: - page, html = browser.get_page_content( - product_url, - wait_for_selector="button, [data-testid]", - timeout=30000, - ) + if is_stealth: + # Use stealth browser + html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX) + + # Check for CAPTCHA + if browser.check_for_captcha(): + logger.warning("CAPTCHA detected on product page!") + if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS: + browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT) + html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX) + else: + return False, None + + page = None + else: + # Use regular Playwright browser + page, html = browser.get_page_content( + product_url, + wait_for_selector="button, [data-testid]", + timeout=30000, + ) soup = BeautifulSoup(html, "html.parser") page_text = soup.get_text().lower() @@ -257,7 +314,8 @@ class PokemonCenterScraper(BaseScraper): if price_match: price = price_match.group() - page.close() + if page is not None: + page.close() return in_stock, price except Exception as e: diff --git a/scrapers/target.py b/scrapers/target.py index 026ebdd..0ab3a58 100644 --- a/scrapers/target.py +++ b/scrapers/target.py @@ -9,7 +9,7 @@ from typing import List, Optional from bs4 import BeautifulSoup from .base import BaseScraper, Product -from browser import get_browser +from src.browser import get_browser logger = logging.getLogger(__name__) @@ -20,87 +20,105 @@ class TargetScraper(BaseScraper): site_name = "target" base_url = "https://www.target.com" - def scrape_category_page(self, url: str) -> List[Product]: + def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]: """ Scrape a Target search/category page for all products Args: url: Search/category page URL + max_pages: Maximum number of pages to scrape (default 1) Returns: List of Product objects """ browser = get_browser() - products = [] + all_products = [] - try: - page, html = browser.get_page_content( - url, - wait_for_selector=None, - timeout=60000, - ) + # Add sort by newest if not already in URL + if "sortBy=" not in url: + separator = "&" if "?" in url else "?" + url = f"{url}{separator}sortBy=newest" + + for page_num in range(1, max_pages + 1): + products = [] + page_url = url if page_num == 1 else f"{url}&Nao={24 * (page_num - 1)}" - # Save debug screenshot try: - page.screenshot(path="debug_target.png") - logger.info("Saved debug screenshot to debug_target.png") - except: - pass + page, html = browser.get_page_content( + page_url, + wait_for_selector=None, + timeout=60000, + ) - soup = BeautifulSoup(html, "html.parser") - - # Try to find product data in page scripts (Target uses React/hydration) - scripts = soup.find_all("script", type="application/json") - for script in scripts: + # Save debug screenshot try: - data = json.loads(script.string) - # Look for product data in the JSON - products.extend(self._extract_products_from_json(data)) - except (json.JSONDecodeError, TypeError): - continue + page.screenshot(path="debug_target.png") + logger.info("Saved debug screenshot to debug_target.png") + except: + pass - # Parse product links from the page - product_links = soup.select("a[href*='/p/']") - logger.info(f"Found {len(product_links)} product links") + soup = BeautifulSoup(html, "html.parser") - # Group links by href and pick the best one (with aria-label or text) - href_to_links = {} - for link in product_links: - href = link.get("href", "") - if not href: - continue - if href not in href_to_links: - href_to_links[href] = [] - href_to_links[href].append(link) + # Try to find product data in page scripts (Target uses React/hydration) + scripts = soup.find_all("script", type="application/json") + for script in scripts: + try: + data = json.loads(script.string) + # Look for product data in the JSON + products.extend(self._extract_products_from_json(data)) + except (json.JSONDecodeError, TypeError): + continue - logger.info(f"Found {len(href_to_links)} unique hrefs") + # Parse product links from the page + product_links = soup.select("a[href*='/p/']") + logger.info(f"Found {len(product_links)} product links") - for href, links in href_to_links.items(): - # Find the best link (one with aria-label or text content) - best_link = None - for link in links: - aria = link.get("aria-label", "") - text = link.get_text(strip=True) - if aria or (text and len(text) > 10): - best_link = link - break - if not best_link: - best_link = links[0] # Fallback to first link + # Group links by href and pick the best one (with aria-label or text) + href_to_links = {} + for link in product_links: + href = link.get("href", "") + if not href: + continue + if href not in href_to_links: + href_to_links[href] = [] + href_to_links[href].append(link) - product = self._parse_product_link(best_link, soup) - if product: - products.append(product) + logger.info(f"Found {len(href_to_links)} unique hrefs") - page.close() + for href, links in href_to_links.items(): + # Find the best link (one with aria-label or text content) + best_link = None + for link in links: + aria = link.get("aria-label", "") + text = link.get_text(strip=True) + if aria or (text and len(text) > 10): + best_link = link + break + if not best_link: + best_link = links[0] # Fallback to first link - except Exception as e: - logger.error(f"Error scraping Target category page: {e}") - raise + product = self._parse_product_link(best_link, soup) + if product: + products.append(product) + + page.close() + + except Exception as e: + logger.error(f"Error scraping Target page {page_num}: {e}") + if page_num == 1: + raise + break + + all_products.extend(products) + + # Stop if no products found on this page + if not products: + break # Remove duplicates seen_urls = set() unique_products = [] - for p in products: + for p in all_products: if p.url not in seen_urls: seen_urls.add(p.url) unique_products.append(p) diff --git a/scrapers/walmart.py b/scrapers/walmart.py new file mode 100644 index 0000000..cb1a227 --- /dev/null +++ b/scrapers/walmart.py @@ -0,0 +1,444 @@ +""" +Walmart.com scraper + +Note: Walmart has aggressive bot protection (PerimeterX). +May need stealth browser mode for sustained use. +""" + +import re +import json +import logging +from typing import List, Optional +from bs4 import BeautifulSoup + +from .base import BaseScraper, Product +from src.browser import get_browser + +logger = logging.getLogger(__name__) + + +class WalmartScraper(BaseScraper): + """Scraper for Walmart.com""" + + site_name = "walmart" + base_url = "https://www.walmart.com" + + def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]: + """ + Scrape a Walmart search/category page for all products + + Args: + url: Search/category page URL + max_pages: Maximum pages to scrape (not yet implemented for Walmart) + + Returns: + List of Product objects + """ + # TODO: Implement pagination for Walmart when needed + browser = get_browser() + products = [] + + try: + page, html = browser.get_page_content( + url, + wait_for_selector=None, + timeout=60000, + ) + + # Save debug screenshot + try: + page.screenshot(path="debug_walmart.png") + logger.info("Saved debug screenshot to debug_walmart.png") + except: + pass + + soup = BeautifulSoup(html, "html.parser") + + # Try to extract from __NEXT_DATA__ JSON (Walmart uses Next.js) + next_data = soup.select_one("script#__NEXT_DATA__") + if next_data: + try: + data = json.loads(next_data.string) + products.extend(self._extract_products_from_next_data(data)) + except (json.JSONDecodeError, TypeError) as e: + logger.debug(f"Error parsing __NEXT_DATA__: {e}") + + # Also try application/json scripts + scripts = soup.find_all("script", type="application/json") + for script in scripts: + try: + data = json.loads(script.string) + products.extend(self._extract_products_from_json(data)) + except (json.JSONDecodeError, TypeError): + continue + + # Try multiple selectors for product cards + product_cards = ( + soup.select("[data-item-id]") + or soup.select(".search-result-gridview-item") + or soup.select("[data-testid='list-view']") + or soup.select("[class*='product-item']") + or soup.select("[class*='ProductCard']") + ) + + logger.info(f"Found {len(product_cards)} product cards") + + for card in product_cards: + product = self._parse_product_card(card) + if product: + products.append(product) + + # Fallback: parse product links + if not product_cards and not products: + product_links = soup.select("a[href*='/ip/']") + logger.info(f"Fallback: Found {len(product_links)} product links") + + href_to_links = {} + for link in product_links: + href = link.get("href", "") + if not href or "/ip/" not in href: + continue + if href not in href_to_links: + href_to_links[href] = [] + href_to_links[href].append(link) + + for href, links in href_to_links.items(): + best_link = links[0] + for link in links: + if link.get("aria-label") or len(link.get_text(strip=True)) > 10: + best_link = link + break + + product = self._parse_product_link(best_link, soup) + if product: + products.append(product) + + page.close() + + except Exception as e: + logger.error(f"Error scraping Walmart category page: {e}") + raise + + # Remove duplicates + seen_urls = set() + unique_products = [] + for p in products: + if p.url not in seen_urls: + seen_urls.add(p.url) + unique_products.append(p) + + logger.info(f"Scraped {len(unique_products)} unique products from Walmart") + return unique_products + + def _extract_products_from_next_data(self, data: dict) -> List[Product]: + """Extract products from Next.js __NEXT_DATA__ JSON""" + products = [] + + try: + # Navigate to search results in Next.js data structure + props = data.get("props", {}) + page_props = props.get("pageProps", {}) + initial_data = page_props.get("initialData", {}) + search_result = initial_data.get("searchResult", {}) + item_stacks = search_result.get("itemStacks", []) + + for stack in item_stacks: + items = stack.get("items", []) + for item in items: + product = self._parse_walmart_item(item) + if product: + products.append(product) + + except Exception as e: + logger.debug(f"Error extracting from __NEXT_DATA__: {e}") + + return products + + def _extract_products_from_json(self, data, depth=0) -> List[Product]: + """Recursively search JSON for product data""" + products = [] + if depth > 10: + return products + + if isinstance(data, dict): + # Check if this looks like a Walmart product + if "usItemId" in data or ("name" in data and "priceInfo" in data): + product = self._parse_walmart_item(data) + if product: + products.append(product) + + for value in data.values(): + products.extend(self._extract_products_from_json(value, depth + 1)) + + elif isinstance(data, list): + for item in data: + products.extend(self._extract_products_from_json(item, depth + 1)) + + return products + + def _parse_walmart_item(self, data: dict) -> Optional[Product]: + """Parse a product from Walmart's JSON data""" + try: + name = data.get("name") or data.get("title", "") + if not name: + return None + + item_id = data.get("usItemId") or data.get("id", "") + canonical_url = data.get("canonicalUrl") or data.get("productPageUrl", "") + + if canonical_url: + url = canonical_url if canonical_url.startswith("http") else f"{self.base_url}{canonical_url}" + elif item_id: + url = f"{self.base_url}/ip/{item_id}" + else: + return None + + # Get price + price = None + price_info = data.get("priceInfo", {}) + if isinstance(price_info, dict): + current_price = price_info.get("currentPrice", {}) + if isinstance(current_price, dict): + price = current_price.get("priceString") + elif price_info.get("priceString"): + price = price_info.get("priceString") + + if not price and "price" in data: + price_val = data.get("price") + if isinstance(price_val, (int, float)): + price = f"${price_val:.2f}" + + # Check availability + in_stock = True + availability = data.get("availabilityStatusV2", {}) + if isinstance(availability, dict): + status = availability.get("value", "").upper() + in_stock = status not in ["OUT_OF_STOCK", "NOT_AVAILABLE"] + elif data.get("availabilityStatus"): + in_stock = data.get("availabilityStatus") != "OUT_OF_STOCK" + + # Get image + image_url = None + image_info = data.get("imageInfo", {}) + if isinstance(image_info, dict): + image_url = image_info.get("thumbnailUrl") or image_info.get("url") + elif data.get("image"): + image_url = data.get("image") + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=str(item_id), + ) + + except Exception as e: + logger.debug(f"Error parsing Walmart product JSON: {e}") + return None + + def _parse_product_card(self, card) -> Optional[Product]: + """Parse a product card element""" + try: + # Find link + link = card.select_one("a[href*='/ip/']") or card.select_one("a") + if not link: + return None + + href = link.get("href", "") + if not href: + return None + url = href if href.startswith("http") else f"{self.base_url}{href}" + + # Get name + name_elem = ( + card.select_one("[data-automation-id='product-title']") + or card.select_one(".product-title-link span") + or card.select_one("[class*='ProductTitle']") + or card.select_one("[class*='product-title']") + or link + ) + name = name_elem.get_text(strip=True) if name_elem else "Unknown" + + # Skip if name too short + if len(name) < 5: + return None + + # Get price + price_elem = ( + card.select_one("[data-automation-id='product-price']") + or card.select_one(".price-current") + or card.select_one("[class*='ProductPrice']") + or card.select_one("[class*='price']") + ) + price = None + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + # Check stock + in_stock = self._check_card_stock_status(card) + + # Get image + img = card.select_one("img") + image_url = None + if img: + image_url = img.get("src") or img.get("data-src") + if image_url and not image_url.startswith("http"): + image_url = f"https:{image_url}" if image_url.startswith("//") else f"{self.base_url}{image_url}" + + # Extract item ID from URL + product_id = "" + match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url) + if match: + product_id = match.group(1) + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing Walmart product card: {e}") + return None + + def _parse_product_link(self, link, soup) -> Optional[Product]: + """Parse a product from a product link element""" + try: + href = link.get("href", "") + if not href: + return None + + url = href if href.startswith("http") else f"{self.base_url}{href}" + + name = link.get("aria-label", "") or link.get_text(strip=True) + + if not name or len(name) < 5: + return None + + # Extract item ID + product_id = "" + match = re.search(r"/ip/[^/]+/(\d+)", url) or re.search(r"/ip/(\d+)", url) + if match: + product_id = match.group(1) + + # Find price near link + parent = link.find_parent() + price = None + for _ in range(5): + if parent: + price_text = parent.get_text() + price_match = re.search(r"\$[\d,]+\.?\d*", price_text) + if price_match: + price = price_match.group() + break + parent = parent.find_parent() + + # Find image + image_url = None + parent = link.find_parent() + for _ in range(5): + if parent: + img = parent.select_one("img") + if img: + image_url = img.get("src") or img.get("data-src") + break + parent = parent.find_parent() + + # Check stock + in_stock = True + parent = link.find_parent() + for _ in range(5): + if parent: + text = parent.get_text().lower() + if any(phrase in text for phrase in ["out of stock", "unavailable", "not available"]): + in_stock = False + break + parent = parent.find_parent() + + return Product( + name=name, + url=url, + price=price, + in_stock=in_stock, + image_url=image_url, + site=self.site_name, + product_id=product_id, + ) + + except Exception as e: + logger.debug(f"Error parsing Walmart product link: {e}") + return None + + def _check_card_stock_status(self, card) -> bool: + """Check if a product card indicates in-stock status""" + card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower() + + out_of_stock_phrases = [ + "out of stock", + "unavailable", + "not available", + "sold out", + "pickup not available", + ] + + for phrase in out_of_stock_phrases: + if phrase in card_text: + return False + + in_stock_phrases = [ + "add to cart", + "available", + "in stock", + "pickup available", + "delivery available", + ] + + for phrase in in_stock_phrases: + if phrase in card_text: + return True + + return True + + def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]: + """Check if a specific product is in stock""" + browser = get_browser() + + try: + page, html = browser.get_page_content(product_url, timeout=30000) + soup = BeautifulSoup(html, "html.parser") + page_text = soup.get_text().lower() + + out_of_stock = any( + phrase in page_text + for phrase in ["out of stock", "unavailable", "not available", "sold out"] + ) + + in_stock = not out_of_stock + + # Get price + price = None + price_elem = ( + soup.select_one("[data-automation-id='product-price']") + or soup.select_one("[itemprop='price']") + or soup.select_one("[class*='price']") + ) + if price_elem: + price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text()) + if price_match: + price = price_match.group() + + page.close() + return in_stock, price + + except Exception as e: + logger.error(f"Error checking Walmart product stock: {e}") + return False, None diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..8eaed50 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +# Core application modules diff --git a/browser.py b/src/browser.py similarity index 94% rename from browser.py rename to src/browser.py index 3c8af36..1f6fc57 100644 --- a/browser.py +++ b/src/browser.py @@ -71,6 +71,8 @@ class StealthBrowser: """Start using an existing Chrome profile (better for bot detection)""" self.using_persistent = True + logger.info(f"Using profile path: {CHROME_USER_DATA_DIR}") + # Launch persistent context with user's REAL Chrome (not Playwright's Chromium) # This is needed because Chrome encrypts credentials with its own binary self.context = self.playwright.chromium.launch_persistent_context( @@ -84,9 +86,19 @@ class StealthBrowser: args=[ "--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage", + "--no-first-run", + "--no-default-browser-check", ] ) + # Close any default blank pages that were opened + for page in self.context.pages: + if page.url == "about:blank": + try: + page.close() + except: + pass + # Apply stealth stealth.apply_stealth_sync(self.context) diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000..465a668 --- /dev/null +++ b/src/database.py @@ -0,0 +1,929 @@ +""" +SQLite database for Pokemon Stock Monitor stats tracking. +Stores product history, price changes, stock events, and favorites. +""" + +import sqlite3 +import json +import logging +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional, List, Dict, Any +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +# Database file path +DB_PATH = Path(__file__).parent.parent / "data" / "stats.db" + + +class Database: + """SQLite database wrapper for stats tracking""" + + def __init__(self, db_path: str = None): + self.db_path = db_path or str(DB_PATH) + self._init_schema() + + @contextmanager + def get_connection(self): + """Context manager for database connections""" + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row # Return rows as dictionaries + try: + yield conn + conn.commit() + except Exception as e: + conn.rollback() + raise e + finally: + conn.close() + + def _init_schema(self): + """Initialize database schema""" + with self.get_connection() as conn: + cursor = conn.cursor() + + # Products table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS products ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + site TEXT NOT NULL, + product_id TEXT, + image_url TEXT, + current_price TEXT, + in_stock BOOLEAN DEFAULT 0, + first_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + last_seen DATETIME, + category TEXT + ) + """) + + # Price history table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS price_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product_id INTEGER NOT NULL, + price TEXT, + recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + """) + + # Stock events table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS stock_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product_id INTEGER NOT NULL, + event_type TEXT NOT NULL, + recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (product_id) REFERENCES products(id) + ) + """) + + # Favorites table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + value TEXT NOT NULL, + display_name TEXT, + priority TEXT DEFAULT 'high', + notify_discord BOOLEAN DEFAULT 1, + notify_sound BOOLEAN DEFAULT 0, + custom_webhook TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE(type, value) + ) + """) + + # Check logs table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS check_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site TEXT, + products_found INTEGER DEFAULT 0, + new_products INTEGER DEFAULT 0, + restocks INTEGER DEFAULT 0, + duration_ms INTEGER, + success BOOLEAN DEFAULT 1, + error_message TEXT, + checked_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + + # News articles table + cursor.execute(""" + CREATE TABLE IF NOT EXISTS news_articles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + source_account TEXT, + external_id TEXT UNIQUE, + title TEXT, + content TEXT NOT NULL, + url TEXT, + author TEXT, + image_url TEXT, + published_at DATETIME, + fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, + sentiment_score REAL, + sentiment_label TEXT, + keywords TEXT, + related_product_ids TEXT, + is_drop_related BOOLEAN DEFAULT 0, + is_restock_related BOOLEAN DEFAULT 0 + ) + """) + + # Users table for multi-user support + cursor.execute(""" + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + zip_code TEXT, + radius_miles INTEGER DEFAULT 25, + discord_webhook TEXT, + notify_enabled BOOLEAN DEFAULT 1, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_active DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + + # Create indexes for common queries + cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_site ON products(site)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_product ON stock_events(product_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_type ON stock_events(event_type)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_check_logs_site ON check_logs(site)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_source ON news_articles(source)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_sentiment ON news_articles(sentiment_label)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_users_name ON users(name)") + + logger.info("Database schema initialized") + + # ==================== Product Methods ==================== + + def get_or_create_product(self, url: str, name: str, site: str, + product_id: str = None, image_url: str = None, + price: str = None, in_stock: bool = False) -> int: + """Get existing product or create new one. Returns product ID.""" + category = self._detect_category(name) + + with self.get_connection() as conn: + cursor = conn.cursor() + + # Try to get existing product + cursor.execute("SELECT id FROM products WHERE url = ?", (url,)) + row = cursor.fetchone() + + if row: + # Update existing product + cursor.execute(""" + UPDATE products + SET name = ?, current_price = ?, in_stock = ?, + last_seen = CURRENT_TIMESTAMP, image_url = COALESCE(?, image_url), + category = COALESCE(?, category) + WHERE id = ? + """, (name, price, in_stock, image_url, category, row['id'])) + return row['id'] + else: + # Create new product + cursor.execute(""" + INSERT INTO products (url, name, site, product_id, image_url, + current_price, in_stock, last_seen, category) + VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?) + """, (url, name, site, product_id, image_url, price, in_stock, category)) + return cursor.lastrowid + + def get_product(self, product_id: int) -> Optional[Dict]: + """Get product by ID""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM products WHERE id = ?", (product_id,)) + row = cursor.fetchone() + return dict(row) if row else None + + def get_product_by_url(self, url: str) -> Optional[Dict]: + """Get product by URL""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM products WHERE url = ?", (url,)) + row = cursor.fetchone() + return dict(row) if row else None + + def update_product_price(self, product_id: int, price: str): + """Update product price and record in history""" + if not price: + return + + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE products SET current_price = ?, last_seen = CURRENT_TIMESTAMP + WHERE id = ? + """, (price, product_id)) + + # Record in price history (only if changed) + self.record_price(product_id, price) + + def update_product_stock(self, product_id: int, in_stock: bool): + """Update product stock status""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE products SET in_stock = ?, last_seen = CURRENT_TIMESTAMP + WHERE id = ? + """, (in_stock, product_id)) + + def get_products(self, site: str = None, category: str = None, + in_stock: bool = None, favorites_only: bool = False, + event_type: str = None, period: str = None, + limit: int = 100, offset: int = 0) -> List[Dict]: + """Get products with optional filters + + Args: + event_type: Filter by recent event type ('new_drop' or 'restock') + period: Time period for event filter ('today' or 'week') + """ + with self.get_connection() as conn: + cursor = conn.cursor() + + # If filtering by event type, use a join with stock_events + if event_type and period: + period_sql = "-1 day" if period == "today" else "-7 days" + query = """ + SELECT DISTINCT p.* FROM products p + INNER JOIN stock_events e ON p.id = e.product_id + WHERE e.event_type = ? + AND e.recorded_at >= datetime('now', ?) + """ + params = [event_type, period_sql] + else: + query = "SELECT * FROM products WHERE 1=1" + params = [] + + if site: + query += " AND p.site = ?" if event_type else " AND site = ?" + params.append(site) + if category: + query += " AND p.category = ?" if event_type else " AND category = ?" + params.append(category) + if in_stock is not None: + query += " AND p.in_stock = ?" if event_type else " AND in_stock = ?" + params.append(in_stock) + if favorites_only: + prefix = "p." if event_type else "" + query += f""" AND ( + {prefix}url IN (SELECT value FROM favorites WHERE type = 'product') + OR {prefix}category IN (SELECT value FROM favorites WHERE type = 'category') + )""" + + order_col = "p.last_seen" if event_type else "last_seen" + query += f" ORDER BY {order_col} DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + cursor.execute(query, params) + return [dict(row) for row in cursor.fetchall()] + + def get_product_count(self, site: str = None, in_stock: bool = None) -> int: + """Get count of products matching filters""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = "SELECT COUNT(*) FROM products WHERE 1=1" + params = [] + + if site: + query += " AND site = ?" + params.append(site) + if in_stock is not None: + query += " AND in_stock = ?" + params.append(in_stock) + + cursor.execute(query, params) + return cursor.fetchone()[0] + + def _detect_category(self, name: str) -> str: + """Auto-detect product category from name""" + name_lower = name.lower() + + if "elite trainer" in name_lower or "etb" in name_lower: + return "ETB" + elif "booster bundle" in name_lower: + return "Booster Bundle" + elif "booster box" in name_lower: + return "Booster Box" + elif "booster pack" in name_lower or "sleeved booster" in name_lower: + return "Booster Pack" + elif "collection" in name_lower: + return "Collection Box" + elif "tin" in name_lower: + return "Tin" + elif "blister" in name_lower: + return "Blister" + elif "binder" in name_lower or "album" in name_lower: + return "Accessories" + else: + return "Other" + + # ==================== Price History Methods ==================== + + def record_price(self, product_id: int, price: str): + """Record a price point for a product""" + if not price: + return + + with self.get_connection() as conn: + cursor = conn.cursor() + + # Check if price changed from last record + cursor.execute(""" + SELECT price FROM price_history + WHERE product_id = ? + ORDER BY recorded_at DESC LIMIT 1 + """, (product_id,)) + row = cursor.fetchone() + + # Only record if price changed or no history exists + if not row or row['price'] != price: + cursor.execute(""" + INSERT INTO price_history (product_id, price) + VALUES (?, ?) + """, (product_id, price)) + logger.debug(f"Recorded price change for product {product_id}: {price}") + + def get_price_history(self, product_id: int, days: int = 30) -> List[Dict]: + """Get price history for a product""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT price, recorded_at FROM price_history + WHERE product_id = ? AND recorded_at >= datetime('now', ?) + ORDER BY recorded_at ASC + """, (product_id, f"-{days} days")) + return [dict(row) for row in cursor.fetchall()] + + # ==================== Stock Event Methods ==================== + + def record_stock_event(self, product_id: int, event_type: str): + """Record a stock event (new_drop, restock, in_stock, out_of_stock)""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + INSERT INTO stock_events (product_id, event_type) + VALUES (?, ?) + """, (product_id, event_type)) + logger.debug(f"Recorded {event_type} event for product {product_id}") + + def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]: + """Get recent stock events with product info""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = """ + SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price + FROM stock_events e + JOIN products p ON e.product_id = p.id + """ + params = [] + + if event_types: + placeholders = ",".join("?" * len(event_types)) + query += f" WHERE e.event_type IN ({placeholders})" + params.extend(event_types) + + query += " ORDER BY e.recorded_at DESC LIMIT ?" + params.append(limit) + + cursor.execute(query, params) + return [dict(row) for row in cursor.fetchall()] + + def get_events_today(self, event_type: str = None) -> int: + """Count events from today""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = """ + SELECT COUNT(*) FROM stock_events + WHERE DATE(recorded_at) = DATE('now') + """ + params = [] + + if event_type: + query += " AND event_type = ?" + params.append(event_type) + + cursor.execute(query, params) + return cursor.fetchone()[0] + + def get_events_this_week(self, event_type: str = None) -> int: + """Count events from this week""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = """ + SELECT COUNT(*) FROM stock_events + WHERE recorded_at >= datetime('now', '-7 days') + """ + params = [] + + if event_type: + query += " AND event_type = ?" + params.append(event_type) + + cursor.execute(query, params) + return cursor.fetchone()[0] + + # ==================== Analytics Methods ==================== + + def get_drop_timing_stats(self, days: int = 30) -> List[Dict]: + """Get drop timing by hour of day""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT + strftime('%H', recorded_at) as hour, + COUNT(*) as count + FROM stock_events + WHERE event_type IN ('new_drop', 'restock') + AND recorded_at >= datetime('now', ?) + GROUP BY hour + ORDER BY hour + """, (f"-{days} days",)) + return [dict(row) for row in cursor.fetchall()] + + def get_stock_duration_stats(self) -> List[Dict]: + """Get average time items stay in stock""" + with self.get_connection() as conn: + cursor = conn.cursor() + # Calculate duration between in_stock and out_of_stock events + cursor.execute(""" + SELECT + p.category, + AVG( + CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER) + ) as avg_minutes_in_stock + FROM stock_events in_event + JOIN stock_events out_event ON in_event.product_id = out_event.product_id + AND out_event.event_type = 'out_of_stock' + AND out_event.recorded_at > in_event.recorded_at + JOIN products p ON in_event.product_id = p.id + WHERE in_event.event_type IN ('restock', 'new_drop') + GROUP BY p.category + """) + return [dict(row) for row in cursor.fetchall()] + + def get_site_stats(self) -> List[Dict]: + """Get stats per site""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT + site, + COUNT(*) as total_products, + SUM(CASE WHEN in_stock = 1 THEN 1 ELSE 0 END) as in_stock_count + FROM products + GROUP BY site + """) + return [dict(row) for row in cursor.fetchall()] + + # ==================== Check Log Methods ==================== + + def log_check(self, site: str, products_found: int, new_products: int, + restocks: int, duration_ms: int, success: bool = True, + error_message: str = None): + """Log a monitoring check""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + INSERT INTO check_logs (site, products_found, new_products, + restocks, duration_ms, success, error_message) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, (site, products_found, new_products, restocks, duration_ms, + success, error_message)) + + def get_last_check(self, site: str = None) -> Optional[Dict]: + """Get the most recent check log""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = "SELECT * FROM check_logs" + params = [] + + if site: + query += " WHERE site = ?" + params.append(site) + + query += " ORDER BY checked_at DESC LIMIT 1" + cursor.execute(query, params) + row = cursor.fetchone() + return dict(row) if row else None + + def get_check_history(self, site: str = None, limit: int = 100) -> List[Dict]: + """Get check history""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = "SELECT * FROM check_logs" + params = [] + + if site: + query += " WHERE site = ?" + params.append(site) + + query += " ORDER BY checked_at DESC LIMIT ?" + params.append(limit) + + cursor.execute(query, params) + return [dict(row) for row in cursor.fetchall()] + + # ==================== Favorites Methods ==================== + + def add_favorite(self, fav_type: str, value: str, display_name: str = None, + priority: str = "high", notify_discord: bool = True, + notify_sound: bool = False, custom_webhook: str = None) -> int: + """Add a favorite. Returns favorite ID.""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + INSERT OR REPLACE INTO favorites + (type, value, display_name, priority, notify_discord, notify_sound, custom_webhook) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, (fav_type, value, display_name or value, priority, + notify_discord, notify_sound, custom_webhook)) + return cursor.lastrowid + + def remove_favorite(self, favorite_id: int): + """Remove a favorite by ID""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM favorites WHERE id = ?", (favorite_id,)) + + def get_favorites(self, fav_type: str = None) -> List[Dict]: + """Get all favorites, optionally filtered by type""" + with self.get_connection() as conn: + cursor = conn.cursor() + + if fav_type: + cursor.execute("SELECT * FROM favorites WHERE type = ? ORDER BY created_at DESC", + (fav_type,)) + else: + cursor.execute("SELECT * FROM favorites ORDER BY created_at DESC") + + return [dict(row) for row in cursor.fetchall()] + + def get_favorite(self, favorite_id: int) -> Optional[Dict]: + """Get a favorite by ID""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM favorites WHERE id = ?", (favorite_id,)) + row = cursor.fetchone() + return dict(row) if row else None + + def update_favorite(self, favorite_id: int, **kwargs): + """Update favorite settings""" + allowed_fields = ['display_name', 'priority', 'notify_discord', + 'notify_sound', 'custom_webhook'] + updates = {k: v for k, v in kwargs.items() if k in allowed_fields} + + if not updates: + return + + with self.get_connection() as conn: + cursor = conn.cursor() + set_clause = ", ".join(f"{k} = ?" for k in updates.keys()) + cursor.execute( + f"UPDATE favorites SET {set_clause} WHERE id = ?", + list(updates.values()) + [favorite_id] + ) + + def check_is_favorite(self, url: str = None, category: str = None) -> Optional[Dict]: + """Check if a product URL or category is favorited""" + with self.get_connection() as conn: + cursor = conn.cursor() + + if url: + cursor.execute(""" + SELECT * FROM favorites + WHERE type = 'product' AND value = ? + """, (url,)) + row = cursor.fetchone() + if row: + return dict(row) + + if category: + cursor.execute(""" + SELECT * FROM favorites + WHERE type = 'category' AND value = ? + """, (category,)) + row = cursor.fetchone() + if row: + return dict(row) + + return None + + # ==================== News Article Methods ==================== + + def add_news_article(self, source: str, content: str, source_account: str = None, + external_id: str = None, title: str = None, url: str = None, + author: str = None, image_url: str = None, published_at: str = None, + sentiment_score: float = None, sentiment_label: str = None, + keywords: List[str] = None, related_product_ids: List[int] = None, + is_drop_related: bool = False, is_restock_related: bool = False) -> int: + """Add a news article. Returns article ID.""" + with self.get_connection() as conn: + cursor = conn.cursor() + + # Check if article already exists by external_id + if external_id: + cursor.execute("SELECT id FROM news_articles WHERE external_id = ?", (external_id,)) + existing = cursor.fetchone() + if existing: + return existing['id'] + + cursor.execute(""" + INSERT INTO news_articles (source, source_account, external_id, title, content, + url, author, image_url, published_at, sentiment_score, + sentiment_label, keywords, related_product_ids, + is_drop_related, is_restock_related) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + source, source_account, external_id, title, content, + url, author, image_url, published_at, sentiment_score, + sentiment_label, + json.dumps(keywords) if keywords else None, + json.dumps(related_product_ids) if related_product_ids else None, + is_drop_related, is_restock_related + )) + return cursor.lastrowid + + def get_news_articles(self, source: str = None, sentiment: str = None, + drop_related: bool = None, limit: int = 50, + offset: int = 0) -> List[Dict]: + """Get news articles with optional filters""" + with self.get_connection() as conn: + cursor = conn.cursor() + + query = "SELECT * FROM news_articles WHERE 1=1" + params = [] + + if source: + query += " AND source = ?" + params.append(source) + if sentiment: + query += " AND sentiment_label = ?" + params.append(sentiment) + if drop_related is not None: + query += " AND (is_drop_related = ? OR is_restock_related = ?)" + params.extend([drop_related, drop_related]) + + query += " ORDER BY published_at DESC, fetched_at DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + cursor.execute(query, params) + articles = [] + for row in cursor.fetchall(): + article = dict(row) + # Parse JSON fields + if article.get('keywords'): + article['keywords'] = json.loads(article['keywords']) + if article.get('related_product_ids'): + article['related_product_ids'] = json.loads(article['related_product_ids']) + articles.append(article) + return articles + + def get_news_article(self, article_id: int) -> Optional[Dict]: + """Get a news article by ID""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM news_articles WHERE id = ?", (article_id,)) + row = cursor.fetchone() + if row: + article = dict(row) + if article.get('keywords'): + article['keywords'] = json.loads(article['keywords']) + if article.get('related_product_ids'): + article['related_product_ids'] = json.loads(article['related_product_ids']) + return article + return None + + def get_news_stats(self) -> Dict: + """Get news statistics""" + with self.get_connection() as conn: + cursor = conn.cursor() + + # Total articles + cursor.execute("SELECT COUNT(*) FROM news_articles") + total = cursor.fetchone()[0] + + # By source + cursor.execute(""" + SELECT source, COUNT(*) as count + FROM news_articles GROUP BY source + """) + by_source = {row['source']: row['count'] for row in cursor.fetchall()} + + # By sentiment + cursor.execute(""" + SELECT sentiment_label, COUNT(*) as count + FROM news_articles WHERE sentiment_label IS NOT NULL + GROUP BY sentiment_label + """) + by_sentiment = {row['sentiment_label']: row['count'] for row in cursor.fetchall()} + + # Today's articles + cursor.execute(""" + SELECT COUNT(*) FROM news_articles + WHERE DATE(fetched_at) = DATE('now') + """) + today = cursor.fetchone()[0] + + # Drop related + cursor.execute(""" + SELECT COUNT(*) FROM news_articles + WHERE is_drop_related = 1 OR is_restock_related = 1 + """) + drop_related = cursor.fetchone()[0] + + return { + 'total': total, + 'today': today, + 'by_source': by_source, + 'by_sentiment': by_sentiment, + 'drop_related': drop_related + } + + def update_news_correlation(self, article_id: int, related_product_ids: List[int], + is_drop_related: bool, is_restock_related: bool): + """Update news article with correlation data""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + UPDATE news_articles + SET related_product_ids = ?, is_drop_related = ?, is_restock_related = ? + WHERE id = ? + """, (json.dumps(related_product_ids), is_drop_related, is_restock_related, article_id)) + + def get_correlated_news(self, hours: int = 24) -> List[Dict]: + """Get news articles correlated with recent drops""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM news_articles + WHERE (is_drop_related = 1 OR is_restock_related = 1) + AND fetched_at >= datetime('now', ?) + ORDER BY published_at DESC + """, (f"-{hours} hours",)) + articles = [] + for row in cursor.fetchall(): + article = dict(row) + if article.get('keywords'): + article['keywords'] = json.loads(article['keywords']) + if article.get('related_product_ids'): + article['related_product_ids'] = json.loads(article['related_product_ids']) + articles.append(article) + return articles + + def delete_old_news(self, days: int = 30): + """Delete news articles older than specified days""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + DELETE FROM news_articles + WHERE fetched_at < datetime('now', ?) + """, (f"-{days} days",)) + deleted = cursor.rowcount + logger.info(f"Deleted {deleted} old news articles") + return deleted + + # ==================== Migration Methods ==================== + + def migrate_from_json(self, json_path: str): + """Migrate existing products.json to database""" + try: + with open(json_path, 'r') as f: + products = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + logger.warning(f"Could not load {json_path} for migration") + return + + migrated = 0 + for url, data in products.items(): + product_id = self.get_or_create_product( + url=url, + name=data.get('name', 'Unknown'), + site=data.get('site', 'unknown'), + product_id=data.get('product_id'), + image_url=data.get('image_url'), + price=data.get('price'), + in_stock=data.get('in_stock', False) + ) + + # Record initial price if available + if data.get('price'): + self.record_price(product_id, data['price']) + + # Record as existing product (not new_drop since it's historical) + if data.get('in_stock'): + self.record_stock_event(product_id, 'in_stock') + + migrated += 1 + + logger.info(f"Migrated {migrated} products from {json_path}") + + # ==================== User Methods ==================== + + def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25, + discord_webhook: str = None, notify_enabled: bool = True) -> int: + """Create a new user. Returns user ID.""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + INSERT INTO users (name, zip_code, radius_miles, discord_webhook, notify_enabled) + VALUES (?, ?, ?, ?, ?) + """, (name, zip_code, radius_miles, discord_webhook, notify_enabled)) + logger.info(f"Created user: {name}") + return cursor.lastrowid + + def get_user(self, user_id: int) -> Optional[Dict]: + """Get user by ID""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = cursor.fetchone() + return dict(row) if row else None + + def get_user_by_name(self, name: str) -> Optional[Dict]: + """Get user by name""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM users WHERE name = ?", (name,)) + row = cursor.fetchone() + return dict(row) if row else None + + def get_all_users(self) -> List[Dict]: + """Get all users""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM users ORDER BY name") + return [dict(row) for row in cursor.fetchall()] + + def update_user(self, user_id: int, **kwargs) -> bool: + """Update user settings""" + allowed_fields = ['name', 'zip_code', 'radius_miles', 'discord_webhook', 'notify_enabled'] + updates = {k: v for k, v in kwargs.items() if k in allowed_fields} + + if not updates: + return False + + with self.get_connection() as conn: + cursor = conn.cursor() + set_clause = ", ".join(f"{k} = ?" for k in updates.keys()) + cursor.execute( + f"UPDATE users SET {set_clause}, last_active = CURRENT_TIMESTAMP WHERE id = ?", + list(updates.values()) + [user_id] + ) + logger.info(f"Updated user {user_id}: {updates}") + return cursor.rowcount > 0 + + def delete_user(self, user_id: int) -> bool: + """Delete a user""" + with self.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM users WHERE id = ?", (user_id,)) + return cursor.rowcount > 0 + + def update_user_location(self, user_id: int, zip_code: str, radius_miles: int = 25) -> bool: + """Update user's location settings""" + return self.update_user(user_id, zip_code=zip_code, radius_miles=radius_miles) + + # ==================== Stats Summary ==================== + + def get_dashboard_stats(self) -> Dict: + """Get summary stats for dashboard""" + return { + 'total_products': self.get_product_count(), + 'in_stock_count': self.get_product_count(in_stock=True), + 'new_drops_today': self.get_events_today('new_drop'), + 'new_drops_week': self.get_events_this_week('new_drop'), + 'restocks_today': self.get_events_today('restock'), + 'restocks_week': self.get_events_this_week('restock'), + 'last_check': self.get_last_check(), + 'sites': self.get_site_stats(), + 'favorites_count': len(self.get_favorites()) + } + + +# Global database instance +_db: Database = None + + +def get_database() -> Database: + """Get the global database instance""" + global _db + if _db is None: + _db = Database() + return _db diff --git a/src/discord_bot.py b/src/discord_bot.py new file mode 100644 index 0000000..4a73de8 --- /dev/null +++ b/src/discord_bot.py @@ -0,0 +1,353 @@ +""" +Pokemon Stock Monitor Discord Bot + +Interactive bot for setting location and checking local store stock. + +Commands: + !setlocation - Set your location + !stores - Find nearby stores + !stock - Check Pokemon TCG stock at local stores + !help - Show commands + +Setup: + 1. Create a bot at https://discord.com/developers/applications + 2. Get your bot token + 3. Add bot to your server with Message Content Intent enabled + 4. Set DISCORD_BOT_TOKEN in config.py or environment variable +""" + +import os +import asyncio +import logging +from datetime import datetime +from typing import Optional + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Try to import discord.py +try: + import discord + from discord.ext import commands + DISCORD_AVAILABLE = True +except ImportError: + DISCORD_AVAILABLE = False + logger.warning("discord.py not installed. Run: pip install discord.py") + +from .store_locator import ( + find_all_nearby_stores, + TargetLocator, + BestBuyLocator, + GameStopLocator, + WalmartLocator, + load_location_config, + save_location_config, + format_stores_for_discord +) + +# User locations stored by Discord user ID +user_locations = {} + + +def load_user_locations(): + """Load user locations from file""" + import json + from pathlib import Path + + loc_file = Path(__file__).parent.parent / "data" / "user_locations.json" + if loc_file.exists(): + with open(loc_file, 'r') as f: + return json.load(f) + return {} + + +def save_user_locations(): + """Save user locations to file""" + import json + from pathlib import Path + + loc_file = Path(__file__).parent.parent / "data" / "user_locations.json" + with open(loc_file, 'w') as f: + json.dump(user_locations, f, indent=2) + + +if DISCORD_AVAILABLE: + + # Bot setup with intents + intents = discord.Intents.default() + intents.message_content = True + + bot = commands.Bot(command_prefix='!', intents=intents, help_command=None) + + @bot.event + async def on_ready(): + global user_locations + user_locations = load_user_locations() + logger.info(f"Bot is ready! Logged in as {bot.user}") + logger.info(f"Loaded {len(user_locations)} user locations") + + @bot.command(name='help') + async def help_command(ctx): + """Show help message""" + embed = discord.Embed( + title="Pokemon Stock Monitor Bot", + description="Find Pokemon TCG products at local stores!", + color=0xFFCC00 + ) + + embed.add_field( + name="Commands", + value=""" +`!setlocation ` - Set your location (e.g., `!setlocation 90210`) +`!stores` - Find nearby Target, Best Buy, GameStop, Walmart +`!stock` - Check Pokemon TCG stock at your local stores +`!stock target` - Check Target specifically +`!mylocation` - Show your saved location + """, + inline=False + ) + + embed.add_field( + name="Supported Retailers", + value="Target, Best Buy, GameStop, Walmart", + inline=False + ) + + embed.set_footer(text="Pokemon Stock Monitor") + await ctx.send(embed=embed) + + @bot.command(name='setlocation') + async def set_location(ctx, zip_code: str, radius: int = 25): + """Set user's location""" + # Validate zip code (basic check) + if not zip_code.isdigit() or len(zip_code) != 5: + await ctx.send("Please provide a valid 5-digit US zip code. Example: `!setlocation 90210`") + return + + user_id = str(ctx.author.id) + user_locations[user_id] = { + "zip_code": zip_code, + "radius_miles": radius, + "set_at": datetime.now().isoformat() + } + save_user_locations() + + embed = discord.Embed( + title="Location Saved!", + description=f"Your location has been set to **{zip_code}** with a **{radius} mile** search radius.", + color=0x00FF00 + ) + embed.add_field(name="Next Steps", value="Use `!stores` to find nearby stores or `!stock` to check inventory.") + await ctx.send(embed=embed) + + @bot.command(name='mylocation') + async def my_location(ctx): + """Show user's saved location""" + user_id = str(ctx.author.id) + + if user_id not in user_locations: + await ctx.send("You haven't set a location yet. Use `!setlocation ` to set one.") + return + + loc = user_locations[user_id] + embed = discord.Embed( + title="Your Location", + color=0x0099FF + ) + embed.add_field(name="Zip Code", value=loc["zip_code"], inline=True) + embed.add_field(name="Search Radius", value=f"{loc['radius_miles']} miles", inline=True) + await ctx.send(embed=embed) + + @bot.command(name='stores') + async def find_stores(ctx): + """Find nearby stores""" + user_id = str(ctx.author.id) + + if user_id not in user_locations: + await ctx.send("Please set your location first with `!setlocation `") + return + + loc = user_locations[user_id] + zip_code = loc["zip_code"] + radius = loc["radius_miles"] + + # Send "searching" message + searching_msg = await ctx.send(f"Searching for stores near {zip_code}...") + + # Find stores (run in thread to avoid blocking) + loop = asyncio.get_event_loop() + stores = await loop.run_in_executor( + None, + find_all_nearby_stores, + zip_code, + radius + ) + + # Build embed + embed = discord.Embed( + title=f"Stores Near {zip_code}", + description=f"Within {radius} miles", + color=0xFFCC00 + ) + + total_stores = 0 + for retailer, store_list in stores.items(): + if store_list: + total_stores += len(store_list) + stores_text = "" + for store in store_list[:5]: # Show top 5 per retailer + stores_text += f"**{store.name}** ({store.distance_miles:.1f} mi)\n" + stores_text += f"{store.city}, {store.state}\n\n" + + embed.add_field( + name=f"{retailer} ({len(store_list)} stores)", + value=stores_text[:1024] if stores_text else "No stores found", + inline=False + ) + else: + embed.add_field( + name=retailer, + value="No stores found nearby", + inline=False + ) + + embed.set_footer(text=f"Total: {total_stores} stores found") + + await searching_msg.edit(content=None, embed=embed) + + @bot.command(name='stock') + async def check_stock(ctx, retailer: Optional[str] = None): + """Check Pokemon TCG stock at local stores""" + user_id = str(ctx.author.id) + + if user_id not in user_locations: + await ctx.send("Please set your location first with `!setlocation `") + return + + loc = user_locations[user_id] + zip_code = loc["zip_code"] + radius = loc["radius_miles"] + + # Determine which retailers to check + retailers_to_check = [] + if retailer: + retailer_lower = retailer.lower() + if "target" in retailer_lower: + retailers_to_check = [("Target", TargetLocator)] + elif "best" in retailer_lower or "buy" in retailer_lower: + retailers_to_check = [("Best Buy", BestBuyLocator)] + elif "game" in retailer_lower or "stop" in retailer_lower: + retailers_to_check = [("GameStop", GameStopLocator)] + elif "walmart" in retailer_lower: + retailers_to_check = [("Walmart", WalmartLocator)] + else: + await ctx.send(f"Unknown retailer: {retailer}. Try: target, bestbuy, gamestop, walmart") + return + else: + # Check all + retailers_to_check = [ + ("Target", TargetLocator), + ("Best Buy", BestBuyLocator), + ("GameStop", GameStopLocator), + ("Walmart", WalmartLocator), + ] + + searching_msg = await ctx.send(f"Checking Pokemon TCG stock near {zip_code}...") + + embed = discord.Embed( + title="Pokemon TCG Local Stock", + description=f"Checking stores near {zip_code}", + color=0xFF6600 + ) + + for name, LocatorClass in retailers_to_check: + try: + locator = LocatorClass(zip_code, radius) + stores = locator.find_stores()[:3] # Check closest 3 + + if stores: + store_info = "" + for store in stores: + # Note: Full stock check requires product IDs + # For now, just show store info + store_info += f"**{store.name}**\n" + store_info += f"{store.address}\n" + store_info += f"{store.city}, {store.state} ({store.distance_miles:.1f} mi)\n" + if store.phone: + store_info += f"Phone: {store.phone}\n" + store_info += "\n" + + embed.add_field( + name=f"{name}", + value=store_info[:1024] or "No stores found", + inline=False + ) + else: + embed.add_field(name=name, value="No stores found nearby", inline=False) + + except Exception as e: + logger.error(f"Error checking {name}: {e}") + embed.add_field(name=name, value="Error checking stores", inline=False) + + embed.add_field( + name="Tip", + value="Call ahead to confirm Pokemon TCG availability. Stock changes frequently!", + inline=False + ) + + embed.set_footer(text="Pokemon Stock Monitor") + await searching_msg.edit(content=None, embed=embed) + + @bot.command(name='ping') + async def ping(ctx): + """Check if bot is alive""" + await ctx.send(f"Pong! Latency: {round(bot.latency * 1000)}ms") + + +def run_bot(token: str): + """Run the Discord bot""" + if not DISCORD_AVAILABLE: + print("discord.py is not installed!") + print("Run: pip install discord.py") + return + + bot.run(token) + + +if __name__ == "__main__": + # Try to get token from config or environment + token = os.environ.get("DISCORD_BOT_TOKEN") + + if not token: + try: + from config import DISCORD_BOT_TOKEN + token = DISCORD_BOT_TOKEN + except ImportError: + pass + + if not token: + print("=" * 60) + print("Discord Bot Setup") + print("=" * 60) + print() + print("To run the bot, you need a Discord Bot Token.") + print() + print("1. Go to https://discord.com/developers/applications") + print("2. Create a New Application") + print("3. Go to 'Bot' section, create a bot") + print("4. Enable 'Message Content Intent' under Privileged Intents") + print("5. Copy the token") + print() + print("Then either:") + print(" - Set DISCORD_BOT_TOKEN in config.py") + print(" - Or set DISCORD_BOT_TOKEN environment variable") + print() + print("To invite bot to your server:") + print(" - Go to OAuth2 > URL Generator") + print(" - Select 'bot' scope") + print(" - Select permissions: Send Messages, Embed Links, Read Message History") + print(" - Use the generated URL to invite") + print() + else: + print("Starting Discord bot...") + run_bot(token) diff --git a/src/discord_notifier.py b/src/discord_notifier.py new file mode 100644 index 0000000..d34db72 --- /dev/null +++ b/src/discord_notifier.py @@ -0,0 +1,340 @@ +""" +Discord webhook notifications for stock alerts +Sends rich embeds with product info and direct links +Now supports favorites with priority notifications and custom webhooks +""" + +import json +import logging +import requests +from datetime import datetime +from pathlib import Path +from typing import Optional, Dict, Set +from config import DISCORD_WEBHOOK_URL, NOTIFICATIONS_ENABLED, SKIP_DUPLICATE_SKUS + +logger = logging.getLogger(__name__) + +# Track notified products to avoid duplicates +NOTIFIED_FILE = Path(__file__).parent.parent / "data" / "notified_products.json" +_notified_urls: Set[str] = set() + + +def _load_notified(): + """Load previously notified product URLs""" + global _notified_urls + try: + if NOTIFIED_FILE.exists(): + with open(NOTIFIED_FILE, 'r') as f: + _notified_urls = set(json.load(f)) + except Exception: + _notified_urls = set() + + +def _save_notified(): + """Save notified product URLs""" + try: + NOTIFIED_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(NOTIFIED_FILE, 'w') as f: + json.dump(list(_notified_urls), f) + except Exception as e: + logger.error(f"Failed to save notified products: {e}") + + +def was_already_notified(product_url: str) -> bool: + """Check if we already sent a notification for this product""" + if not _notified_urls: + _load_notified() + return product_url in _notified_urls + + +def mark_as_notified(product_url: str): + """Mark a product as notified""" + _notified_urls.add(product_url) + _save_notified() + +# Colors for different notification types +COLOR_RESTOCK = 0x00FF00 # Green - item back in stock +COLOR_NEW_DROP = 0x0099FF # Blue - new product listing +COLOR_PREORDER = 0xFFAA00 # Orange - pre-order available +COLOR_ERROR = 0xFF0000 # Red - error notification +COLOR_FAVORITE_HIGH = 0xFF0000 # Red - high priority favorite +COLOR_FAVORITE_MEDIUM = 0xFFAA00 # Orange - medium priority favorite + +# Site icons/emojis +SITE_EMOJIS = { + "pokemoncenter": "\U0001F7E1", # Yellow circle + "target": "\U0001F534", # Red circle + "walmart": "\U0001F535", # Blue circle + "bestbuy": "\U0001F7E1", # Yellow circle +} + +# Priority indicators +PRIORITY_EMOJI = { + "high": "\u2B50\u2B50\u2B50", # Three stars + "medium": "\u2B50\u2B50", # Two stars + "low": "\u2B50", # One star + "normal": "", # No stars +} + + +def send_stock_alert( + product_name: str, + product_url: str, + price: str, + site: str, + alert_type: str = "restock", + image_url: Optional[str] = None, + priority_settings: Optional[Dict] = None, +): + """ + Send a Discord notification for a stock alert + + Args: + product_name: Name of the product + product_url: Direct link to the product + price: Price string (e.g., "$49.99") + site: Site name (pokemoncenter, target, etc.) + alert_type: "restock", "new_drop", or "preorder" + image_url: Optional product image URL + priority_settings: Optional dict with favorite/priority info + """ + # Check if notifications are enabled + if not NOTIFICATIONS_ENABLED: + logger.debug(f"Notifications disabled - skipping alert for: {product_name}") + return True # Return True so caller thinks it succeeded + + # Skip duplicate notifications for new drops (restocks always go through) + if SKIP_DUPLICATE_SKUS and alert_type == "new_drop": + if was_already_notified(product_url): + logger.debug(f"Already notified about this product - skipping: {product_name}") + return True + + # Determine which webhook to use + webhook_url = DISCORD_WEBHOOK_URL + if priority_settings and priority_settings.get('custom_webhook'): + webhook_url = priority_settings['custom_webhook'] + + if webhook_url == "YOUR_WEBHOOK_URL_HERE": + logger.error("Discord webhook URL not configured! Update config.py") + return False + + # Check if this is a favorite + is_favorite = priority_settings.get('is_favorite', False) if priority_settings else False + priority = priority_settings.get('priority', 'normal') if priority_settings else 'normal' + + # Choose color based on alert type and priority + if is_favorite and priority == 'high': + color = COLOR_FAVORITE_HIGH + elif is_favorite and priority == 'medium': + color = COLOR_FAVORITE_MEDIUM + elif alert_type == "restock": + color = COLOR_RESTOCK + elif alert_type == "new_drop": + color = COLOR_NEW_DROP + elif alert_type == "preorder": + color = COLOR_PREORDER + else: + color = COLOR_RESTOCK + + # Build title with priority indicator + priority_indicator = PRIORITY_EMOJI.get(priority, "") + favorite_badge = "\u2B50 FAVORITE " if is_favorite else "" + + if alert_type == "restock": + title = f"\U0001F6A8 {favorite_badge}RESTOCK ALERT {priority_indicator}" + elif alert_type == "new_drop": + title = f"\U0001F195 {favorite_badge}NEW DROP {priority_indicator}" + elif alert_type == "preorder": + title = f"\u23F0 {favorite_badge}PRE-ORDER AVAILABLE {priority_indicator}" + else: + title = f"\U0001F514 {favorite_badge}STOCK ALERT {priority_indicator}" + + site_emoji = SITE_EMOJIS.get(site.lower(), "\U0001F6D2") + site_display = site.replace("pokemoncenter", "Pokemon Center").title() + + # Build the embed + embed = { + "title": title.strip(), + "description": f"**{product_name}**", + "url": product_url, + "color": color, + "fields": [ + {"name": "Price", "value": price or "See link", "inline": True}, + {"name": "Store", "value": f"{site_emoji} {site_display}", "inline": True}, + {"name": "Link", "value": f"[\U0001F6D2 BUY NOW]({product_url})", "inline": False}, + ], + "footer": {"text": "Pokemon Stock Monitor"}, + "timestamp": datetime.utcnow().isoformat(), + } + + # Add priority field for favorites + if is_favorite: + embed["fields"].insert(0, { + "name": "Priority", + "value": f"{priority_indicator} {priority.upper()}", + "inline": True + }) + + if image_url: + embed["thumbnail"] = {"url": image_url} + + # Ping @everyone for high priority favorites, or just regular ping otherwise + if is_favorite and priority == 'high': + content = "@everyone \U0001F6A8 HIGH PRIORITY ALERT!" + else: + content = "@everyone" + + payload = { + "content": content, + "embeds": [embed], + } + + try: + response = requests.post( + webhook_url, + json=payload, + timeout=10, + ) + response.raise_for_status() + logger.info(f"Discord notification sent for: {product_name} (priority: {priority})") + # Track that we notified about this product + if alert_type == "new_drop": + mark_as_notified(product_url) + return True + except requests.exceptions.RequestException as e: + logger.error(f"Failed to send Discord notification: {e}") + return False + + +def send_stock_alert_with_priority(product, alert_type: str = "restock", + priority_settings: Optional[Dict] = None): + """ + Convenience method to send alert from Product object with priority settings + + Args: + product: Product object from scraper + alert_type: "restock", "new_drop", or "preorder" + priority_settings: Optional dict with favorite/priority info + """ + return send_stock_alert( + product_name=product.name, + product_url=product.url, + price=product.price, + site=product.site, + alert_type=alert_type, + image_url=product.image_url, + priority_settings=priority_settings + ) + + +def send_error_notification(error_message: str, site: str = "Unknown"): + """Send an error notification to Discord""" + if not NOTIFICATIONS_ENABLED: + return True + if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": + return False + + embed = { + "title": "\u26A0\uFE0F Monitor Error", + "description": error_message, + "color": COLOR_ERROR, + "fields": [{"name": "Site", "value": site, "inline": True}], + "footer": {"text": "Pokemon Stock Monitor"}, + "timestamp": datetime.utcnow().isoformat(), + } + + payload = {"embeds": [embed]} + + try: + response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10) + response.raise_for_status() + return True + except requests.exceptions.RequestException as e: + logger.error(f"Failed to send error notification: {e}") + return False + + +def send_startup_notification(): + """Send a notification that the monitor has started""" + if not NOTIFICATIONS_ENABLED: + return True + if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": + return False + + embed = { + "title": "\u2705 Monitor Started", + "description": "Pokemon Stock Monitor is now running and watching for restocks!", + "color": 0x00FF00, + "footer": {"text": "Pokemon Stock Monitor"}, + "timestamp": datetime.utcnow().isoformat(), + } + + payload = {"embeds": [embed]} + + try: + response = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10) + response.raise_for_status() + return True + except requests.exceptions.RequestException as e: + logger.error(f"Failed to send startup notification: {e}") + return False + + +def test_webhook(): + """Test the Discord webhook connection""" + print("Testing Discord webhook...") + + if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": + print("ERROR: Webhook URL not configured!") + print("Edit config.py and set DISCORD_WEBHOOK_URL") + return False + + # Send a test notification + success = send_stock_alert( + product_name="Test Product - Pokemon TCG Booster", + product_url="https://www.pokemoncenter.com/test", + price="$4.99", + site="pokemoncenter", + alert_type="restock", + ) + + if success: + print("SUCCESS! Check your Discord channel for the test message.") + else: + print("FAILED! Check the webhook URL and try again.") + + return success + + +def test_priority_webhook(): + """Test a high priority favorite notification""" + print("Testing high priority webhook...") + + if DISCORD_WEBHOOK_URL == "YOUR_WEBHOOK_URL_HERE": + print("ERROR: Webhook URL not configured!") + return False + + success = send_stock_alert( + product_name="Chaos Rising Elite Trainer Box", + product_url="https://www.pokemoncenter.com/test", + price="$54.99", + site="pokemoncenter", + alert_type="new_drop", + priority_settings={ + 'is_favorite': True, + 'priority': 'high', + 'custom_webhook': None + } + ) + + if success: + print("SUCCESS! Check your Discord channel for the priority test message.") + else: + print("FAILED! Check the webhook URL and try again.") + + return success + + +if __name__ == "__main__": + # Run webhook test + test_webhook() diff --git a/src/favorites.py b/src/favorites.py new file mode 100644 index 0000000..652e8df --- /dev/null +++ b/src/favorites.py @@ -0,0 +1,192 @@ +""" +Favorites system for Pokemon Stock Monitor. +Handles priority products and categories with custom notification settings. +""" + +import logging +from typing import Optional, List, Dict, Any +from .database import get_database + +logger = logging.getLogger(__name__) + + +# Common category keywords for auto-detection +CATEGORY_KEYWORDS = { + "ETB": ["elite trainer", "etb"], + "Booster Bundle": ["booster bundle"], + "Booster Box": ["booster box"], + "Booster Pack": ["booster pack", "sleeved booster"], + "Collection Box": ["collection", "premium collection", "special collection"], + "Tin": ["tin", "mini tin"], + "Blister": ["blister", "3-pack", "check lane"], + "Accessories": ["binder", "album", "sleeves", "deck box", "playmat"], +} + +# Known Pokemon TCG sets for category matching +KNOWN_SETS = [ + "chaos rising", + "prismatic evolutions", + "surging sparks", + "twilight masquerade", + "temporal forces", + "paldean fates", + "obsidian flames", + "paldea evolved", + "scarlet & violet", + "crown zenith", + "silver tempest", + "lost origin", + "pokemon go", + "astral radiance", + "brilliant stars", + "fusion strike", + "celebrations", +] + + +class FavoritesManager: + """Manages favorites and priority notifications""" + + def __init__(self): + self.db = get_database() + + def add_product_favorite(self, url: str, display_name: str = None, + priority: str = "high", notify_discord: bool = True, + notify_sound: bool = False, custom_webhook: str = None) -> int: + """Add a product URL as favorite""" + return self.db.add_favorite( + fav_type="product", + value=url, + display_name=display_name, + priority=priority, + notify_discord=notify_discord, + notify_sound=notify_sound, + custom_webhook=custom_webhook + ) + + def add_category_favorite(self, category: str, display_name: str = None, + priority: str = "high", notify_discord: bool = True, + notify_sound: bool = False, custom_webhook: str = None) -> int: + """Add a category (e.g., 'ETB', 'Chaos Rising') as favorite""" + return self.db.add_favorite( + fav_type="category", + value=category.lower(), + display_name=display_name or category, + priority=priority, + notify_discord=notify_discord, + notify_sound=notify_sound, + custom_webhook=custom_webhook + ) + + def remove_favorite(self, favorite_id: int): + """Remove a favorite""" + self.db.remove_favorite(favorite_id) + + def get_all_favorites(self) -> List[Dict]: + """Get all favorites""" + return self.db.get_favorites() + + def get_product_favorites(self) -> List[Dict]: + """Get product favorites only""" + return self.db.get_favorites(fav_type="product") + + def get_category_favorites(self) -> List[Dict]: + """Get category favorites only""" + return self.db.get_favorites(fav_type="category") + + def update_favorite(self, favorite_id: int, **kwargs): + """Update favorite settings""" + self.db.update_favorite(favorite_id, **kwargs) + + def check_product_priority(self, url: str, name: str, category: str = None) -> Optional[Dict]: + """ + Check if a product matches any favorites. + Returns favorite info if matched, None otherwise. + """ + # Check direct URL match + fav = self.db.check_is_favorite(url=url) + if fav: + logger.debug(f"Product {name} matched favorite URL") + return fav + + # Check category match + if category: + fav = self.db.check_is_favorite(category=category.lower()) + if fav: + logger.debug(f"Product {name} matched category favorite: {category}") + return fav + + # Check if product name contains any category favorite keywords + name_lower = name.lower() + category_favorites = self.get_category_favorites() + + for fav in category_favorites: + keyword = fav['value'].lower() + if keyword in name_lower: + logger.debug(f"Product {name} matched keyword favorite: {keyword}") + return fav + + return None + + def get_notification_settings(self, url: str, name: str, category: str = None) -> Dict: + """ + Get notification settings for a product based on favorites. + Returns default settings if not a favorite. + """ + favorite = self.check_product_priority(url, name, category) + + if favorite: + return { + 'is_favorite': True, + 'priority': favorite['priority'], + 'notify_discord': bool(favorite['notify_discord']), + 'notify_sound': bool(favorite['notify_sound']), + 'custom_webhook': favorite['custom_webhook'], + 'display_name': favorite['display_name'] + } + else: + return { + 'is_favorite': False, + 'priority': 'normal', + 'notify_discord': True, + 'notify_sound': False, + 'custom_webhook': None, + 'display_name': None + } + + def get_suggested_categories(self) -> List[str]: + """Get list of suggested categories for favorites""" + categories = list(CATEGORY_KEYWORDS.keys()) + categories.extend([s.title() for s in KNOWN_SETS]) + return sorted(set(categories)) + + def detect_matching_categories(self, name: str) -> List[str]: + """Detect which categories a product name matches""" + name_lower = name.lower() + matches = [] + + # Check product type categories + for category, keywords in CATEGORY_KEYWORDS.items(): + for keyword in keywords: + if keyword in name_lower: + matches.append(category) + break + + # Check set names + for set_name in KNOWN_SETS: + if set_name in name_lower: + matches.append(set_name.title()) + + return matches + + +# Global favorites manager instance +_favorites: FavoritesManager = None + + +def get_favorites_manager() -> FavoritesManager: + """Get the global favorites manager instance""" + global _favorites + if _favorites is None: + _favorites = FavoritesManager() + return _favorites diff --git a/src/news/__init__.py b/src/news/__init__.py new file mode 100644 index 0000000..31f0f5c --- /dev/null +++ b/src/news/__init__.py @@ -0,0 +1,9 @@ +""" +News aggregation module for Pokemon Stock Monitor. +Fetches news from Twitter, Pokemon.com, and manual Discord input. +""" + +from .base import NewsArticle +from .sentiment import analyze_sentiment, extract_keywords + +__all__ = ['NewsArticle', 'analyze_sentiment', 'extract_keywords'] diff --git a/src/news/base.py b/src/news/base.py new file mode 100644 index 0000000..de25168 --- /dev/null +++ b/src/news/base.py @@ -0,0 +1,96 @@ +""" +Base classes and data structures for news aggregation. +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, List + + +@dataclass +class NewsArticle: + """Represents a news article from any source""" + source: str # 'twitter', 'pokemon_official', 'discord_manual' + source_account: str # '@PokemonRestocks', 'Pokemon.com', 'PokePings Discord' + content: str # Tweet text, article summary, or message content + author: str # Username or account name + published_at: datetime + + # Optional fields + external_id: Optional[str] = None # Tweet ID, article URL slug + title: Optional[str] = None # For official news articles + url: Optional[str] = None # Link to original source + image_url: Optional[str] = None # Attached image if any + + # Sentiment analysis results (populated after analysis) + sentiment_score: Optional[float] = None # -1.0 to 1.0 + sentiment_label: Optional[str] = None # 'positive', 'negative', 'neutral' + keywords: List[str] = field(default_factory=list) # Extracted keywords + + # Correlation with stock events (populated after correlation) + related_product_ids: List[int] = field(default_factory=list) + is_drop_related: bool = False + is_restock_related: bool = False + + def __hash__(self): + """Hash by external_id or content for deduplication""" + if self.external_id: + return hash(self.external_id) + return hash((self.source, self.content[:100], self.published_at)) + + def __eq__(self, other): + if not isinstance(other, NewsArticle): + return False + if self.external_id and other.external_id: + return self.external_id == other.external_id + return (self.source == other.source and + self.content[:100] == other.content[:100] and + self.published_at == other.published_at) + + def to_dict(self) -> dict: + """Convert to dictionary for database storage""" + return { + 'source': self.source, + 'source_account': self.source_account, + 'external_id': self.external_id, + 'title': self.title, + 'content': self.content, + 'url': self.url, + 'author': self.author, + 'image_url': self.image_url, + 'published_at': self.published_at.isoformat() if self.published_at else None, + 'sentiment_score': self.sentiment_score, + 'sentiment_label': self.sentiment_label, + 'keywords': self.keywords, + 'related_product_ids': self.related_product_ids, + 'is_drop_related': self.is_drop_related, + 'is_restock_related': self.is_restock_related + } + + @classmethod + def from_dict(cls, data: dict) -> 'NewsArticle': + """Create from dictionary (database row)""" + published_at = data.get('published_at') + if published_at and isinstance(published_at, str): + try: + published_at = datetime.fromisoformat(published_at) + except ValueError: + published_at = datetime.now() + + return cls( + source=data['source'], + source_account=data.get('source_account', ''), + content=data['content'], + author=data.get('author', ''), + published_at=published_at or datetime.now(), + external_id=data.get('external_id'), + title=data.get('title'), + url=data.get('url'), + image_url=data.get('image_url'), + sentiment_score=data.get('sentiment_score'), + sentiment_label=data.get('sentiment_label'), + keywords=data.get('keywords', []), + related_product_ids=data.get('related_product_ids', []), + is_drop_related=data.get('is_drop_related', False), + is_restock_related=data.get('is_restock_related', False) + ) diff --git a/src/news/pokemon_fetcher.py b/src/news/pokemon_fetcher.py new file mode 100644 index 0000000..858f2fb --- /dev/null +++ b/src/news/pokemon_fetcher.py @@ -0,0 +1,222 @@ +""" +Fetches official Pokemon TCG news from Pokemon.com +""" + +import logging +import re +import requests +from datetime import datetime +from typing import List, Optional +from bs4 import BeautifulSoup + +from .base import NewsArticle +from .sentiment import analyze_article + +logger = logging.getLogger(__name__) + +# Pokemon.com TCG news URL +POKEMON_NEWS_URL = "https://www.pokemon.com/us/pokemon-tcg-news/" + +# Request headers to mimic browser +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', +} + + +class PokemonNewsFetcher: + """Fetches news from Pokemon.com TCG news page""" + + def __init__(self): + self.session = requests.Session() + self.session.headers.update(HEADERS) + + def fetch_latest(self, limit: int = 20) -> List[NewsArticle]: + """ + Fetch latest Pokemon TCG news articles. + + Args: + limit: Maximum number of articles to fetch + + Returns: + List of NewsArticle objects + """ + try: + response = self.session.get(POKEMON_NEWS_URL, timeout=30) + response.raise_for_status() + return self._parse_news_page(response.text, limit) + except requests.RequestException as e: + logger.error(f"Failed to fetch Pokemon news: {e}") + return [] + + def _parse_news_page(self, html: str, limit: int) -> List[NewsArticle]: + """Parse the news page HTML and extract articles""" + soup = BeautifulSoup(html, 'html.parser') + articles = [] + + # Find article containers - Pokemon.com uses various class names + # Try multiple selectors for robustness + article_containers = soup.select('article, .news-article, .article-item, .content-block') + + if not article_containers: + # Fallback: look for links in the main content area + article_containers = soup.select('.main-content a, .news-list a, section a') + + for container in article_containers[:limit]: + try: + article = self._parse_article(container) + if article: + articles.append(article) + except Exception as e: + logger.debug(f"Failed to parse article: {e}") + continue + + logger.info(f"Fetched {len(articles)} Pokemon.com news articles") + return articles + + def _parse_article(self, container) -> Optional[NewsArticle]: + """Parse a single article container""" + # Extract title + title_elem = container.select_one('h1, h2, h3, h4, .title, .headline') + if not title_elem: + # Check if container itself is a link with text + title = container.get_text(strip=True) + if not title or len(title) < 10: + return None + else: + title = title_elem.get_text(strip=True) + + # Extract URL + link = container.select_one('a[href]') + if not link: + if container.name == 'a': + link = container + else: + return None + + url = link.get('href', '') + if url and not url.startswith('http'): + url = f"https://www.pokemon.com{url}" + + # Skip non-TCG articles + if url and '/pokemon-tcg' not in url.lower() and 'tcg' not in title.lower(): + return None + + # Extract summary/description + summary_elem = container.select_one('p, .summary, .description, .excerpt') + summary = summary_elem.get_text(strip=True) if summary_elem else title + + # Extract image + img_elem = container.select_one('img') + image_url = None + if img_elem: + image_url = img_elem.get('src') or img_elem.get('data-src') + if image_url and not image_url.startswith('http'): + image_url = f"https://www.pokemon.com{image_url}" + + # Extract date (if available) + date_elem = container.select_one('time, .date, .published-date') + published_at = datetime.now() + if date_elem: + date_str = date_elem.get('datetime') or date_elem.get_text(strip=True) + published_at = self._parse_date(date_str) or datetime.now() + + # Generate external ID from URL + external_id = f"pokemon_{self._url_to_id(url)}" + + # Analyze sentiment + analysis = analyze_article(summary, title) + + article = NewsArticle( + source='pokemon_official', + source_account='Pokemon.com', + external_id=external_id, + title=title, + content=summary, + url=url, + author='The Pokemon Company', + image_url=image_url, + published_at=published_at, + sentiment_score=analysis['sentiment_score'], + sentiment_label=analysis['sentiment_label'], + keywords=analysis['keywords'], + is_drop_related=analysis['is_drop_related'], + is_restock_related=analysis['is_restock_related'] + ) + + return article + + def _parse_date(self, date_str: str) -> Optional[datetime]: + """Parse various date formats""" + if not date_str: + return None + + # Try ISO format first + try: + return datetime.fromisoformat(date_str.replace('Z', '+00:00')) + except ValueError: + pass + + # Try common formats + formats = [ + '%B %d, %Y', # January 15, 2024 + '%b %d, %Y', # Jan 15, 2024 + '%m/%d/%Y', # 01/15/2024 + '%Y-%m-%d', # 2024-01-15 + ] + + for fmt in formats: + try: + return datetime.strptime(date_str.strip(), fmt) + except ValueError: + continue + + return None + + def _url_to_id(self, url: str) -> str: + """Convert URL to a simple ID""" + if not url: + return str(hash(datetime.now())) + + # Extract the last path segment + parts = url.rstrip('/').split('/') + slug = parts[-1] if parts else '' + + # Remove query params + slug = slug.split('?')[0] + + # If slug is too long or not useful, use hash + if not slug or len(slug) > 100: + return str(abs(hash(url))) + + return slug + + +def fetch_pokemon_news(limit: int = 20) -> List[NewsArticle]: + """ + Convenience function to fetch Pokemon.com news. + + Args: + limit: Maximum number of articles to fetch + + Returns: + List of NewsArticle objects + """ + fetcher = PokemonNewsFetcher() + return fetcher.fetch_latest(limit) + + +# Allow running as a standalone script for testing +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + + print("Fetching Pokemon.com TCG news...") + articles = fetch_pokemon_news(limit=10) + + for i, article in enumerate(articles, 1): + print(f"\n{i}. {article.title}") + print(f" Source: {article.source_account}") + print(f" Sentiment: {article.sentiment_label} ({article.sentiment_score:.2f})") + print(f" Keywords: {', '.join(article.keywords) if article.keywords else 'None'}") + print(f" URL: {article.url}") diff --git a/src/news/sentiment.py b/src/news/sentiment.py new file mode 100644 index 0000000..405404a --- /dev/null +++ b/src/news/sentiment.py @@ -0,0 +1,218 @@ +""" +Sentiment analysis for Pokemon TCG news articles. +Uses VADER (Valence Aware Dictionary and sEntiment Reasoner) from NLTK. +""" + +import re +import logging +from typing import Tuple, List, Optional + +logger = logging.getLogger(__name__) + +# Try to import NLTK's VADER +try: + import nltk + from nltk.sentiment.vader import SentimentIntensityAnalyzer + NLTK_AVAILABLE = True +except ImportError: + NLTK_AVAILABLE = False + logger.warning("NLTK not installed. Sentiment analysis will use fallback method.") + + +# Keywords that indicate stock-related news +STOCK_KEYWORDS = { + 'positive': [ + 'restock', 'restocked', 'restocking', + 'live', 'in stock', 'instock', 'in-stock', + 'available', 'back in stock', 'just dropped', + 'drop', 'dropped', 'dropping', 'new drop', + 'go go go', 'hurry', 'quick', 'fast', + 'preorder', 'pre-order', 'pre order', + 'release', 'released', 'launching' + ], + 'negative': [ + 'sold out', 'soldout', 'oos', 'out of stock', + 'unavailable', 'gone', 'missed', 'too late', + 'cancelled', 'canceled', 'delayed', + 'scalper', 'scalped', 'bot', 'botted' + ], + 'neutral': [ + 'announcement', 'announced', 'reveal', 'revealed', + 'coming soon', 'upcoming', 'rumor', 'leak', + 'spoiler', 'preview', 'teaser' + ] +} + +# Pokemon TCG product keywords for extraction +PRODUCT_KEYWORDS = [ + 'etb', 'elite trainer box', 'elite trainer', + 'booster box', 'booster bundle', 'booster pack', + 'collection', 'premium collection', 'super premium', + 'tin', 'blister', 'pack', 'box', + 'prismatic evolutions', 'surging sparks', 'stellar crown', + 'twilight masquerade', 'temporal forces', 'paldean fates', + 'obsidian flames', 'scarlet violet', '151', 'sv', + 'charizard', 'pikachu', 'mewtwo', 'mew', + 'pokemon center', 'target', 'walmart', 'best buy', 'gamestop' +] + + +def _ensure_vader_data(): + """Download VADER lexicon if not present""" + if not NLTK_AVAILABLE: + return False + try: + nltk.data.find('sentiment/vader_lexicon.zip') + return True + except LookupError: + try: + logger.info("Downloading VADER lexicon...") + nltk.download('vader_lexicon', quiet=True) + return True + except Exception as e: + logger.error(f"Failed to download VADER lexicon: {e}") + return False + + +def analyze_sentiment(text: str) -> Tuple[float, str]: + """ + Analyze sentiment of text using VADER. + + Args: + text: The text to analyze + + Returns: + Tuple of (compound_score, label) + - compound_score: float from -1.0 (negative) to 1.0 (positive) + - label: 'positive', 'negative', or 'neutral' + """ + if not text: + return 0.0, 'neutral' + + # Try VADER first + if NLTK_AVAILABLE and _ensure_vader_data(): + try: + sia = SentimentIntensityAnalyzer() + scores = sia.polarity_scores(text) + compound = scores['compound'] + + if compound >= 0.05: + label = 'positive' + elif compound <= -0.05: + label = 'negative' + else: + label = 'neutral' + + return compound, label + except Exception as e: + logger.warning(f"VADER analysis failed: {e}, using fallback") + + # Fallback: simple keyword-based sentiment + return _fallback_sentiment(text) + + +def _fallback_sentiment(text: str) -> Tuple[float, str]: + """Simple keyword-based sentiment analysis as fallback""" + text_lower = text.lower() + + positive_count = sum(1 for kw in STOCK_KEYWORDS['positive'] if kw in text_lower) + negative_count = sum(1 for kw in STOCK_KEYWORDS['negative'] if kw in text_lower) + + # Simple scoring + if positive_count > negative_count: + score = min(0.5 + (positive_count * 0.1), 1.0) + return score, 'positive' + elif negative_count > positive_count: + score = max(-0.5 - (negative_count * 0.1), -1.0) + return score, 'negative' + else: + return 0.0, 'neutral' + + +def extract_keywords(text: str) -> List[str]: + """ + Extract relevant Pokemon TCG keywords from text. + + Args: + text: The text to analyze + + Returns: + List of found keywords + """ + if not text: + return [] + + text_lower = text.lower() + found = [] + + for keyword in PRODUCT_KEYWORDS: + if keyword in text_lower: + found.append(keyword) + + # Remove duplicates while preserving order + seen = set() + unique = [] + for kw in found: + if kw not in seen: + seen.add(kw) + unique.append(kw) + + return unique + + +def detect_stock_status(text: str) -> Tuple[bool, bool]: + """ + Detect if news is related to drops or restocks. + + Args: + text: The text to analyze + + Returns: + Tuple of (is_drop_related, is_restock_related) + """ + if not text: + return False, False + + text_lower = text.lower() + + # Check for drop-related keywords + drop_keywords = ['new drop', 'just dropped', 'dropping', 'release', 'launched', 'preorder', 'pre-order'] + is_drop = any(kw in text_lower for kw in drop_keywords) + + # Check for restock-related keywords + restock_keywords = ['restock', 'back in stock', 'restocked', 'restocking', 'available again'] + is_restock = any(kw in text_lower for kw in restock_keywords) + + # Generic "in stock" or "live" could be either + if not is_drop and not is_restock: + if any(kw in text_lower for kw in ['live', 'in stock', 'available', 'go go']): + # Default to restock if not clearly a new release + is_restock = True + + return is_drop, is_restock + + +def analyze_article(text: str, title: str = None) -> dict: + """ + Full analysis of a news article. + + Args: + text: Article content + title: Optional title for additional context + + Returns: + Dictionary with analysis results + """ + full_text = f"{title or ''} {text}".strip() + + sentiment_score, sentiment_label = analyze_sentiment(full_text) + keywords = extract_keywords(full_text) + is_drop, is_restock = detect_stock_status(full_text) + + return { + 'sentiment_score': sentiment_score, + 'sentiment_label': sentiment_label, + 'keywords': keywords, + 'is_drop_related': is_drop, + 'is_restock_related': is_restock + } diff --git a/src/product_tracker.py b/src/product_tracker.py new file mode 100644 index 0000000..cf5989b --- /dev/null +++ b/src/product_tracker.py @@ -0,0 +1,246 @@ +""" +Product tracker - keeps track of known products to detect new drops and restocks +Now integrates with SQLite database for historical tracking. +""" + +import json +import logging +from pathlib import Path +from typing import Dict, List, Set, Optional, Tuple +from dataclasses import asdict +from datetime import datetime + +from scrapers.base import Product +from .database import get_database +from .favorites import get_favorites_manager + +logger = logging.getLogger(__name__) + +# File to store known products (kept for backwards compatibility) +PRODUCTS_FILE = Path(__file__).parent.parent / "data" / "products.json" + + +class ProductTracker: + """Tracks known products to detect new listings and stock changes""" + + def __init__(self, products_file: Path = PRODUCTS_FILE): + self.products_file = products_file + self.products: Dict[str, dict] = {} # URL -> product data + self.db = get_database() + self.favorites = get_favorites_manager() + self.load() + + def load(self): + """Load products from file""" + if self.products_file.exists(): + try: + with open(self.products_file, "r", encoding="utf-8") as f: + self.products = json.load(f) + logger.info(f"Loaded {len(self.products)} tracked products") + except (json.JSONDecodeError, IOError) as e: + logger.error(f"Error loading products file: {e}") + self.products = {} + else: + self.products = {} + logger.info("No existing products file, starting fresh") + + def save(self): + """Save products to file""" + try: + with open(self.products_file, "w", encoding="utf-8") as f: + json.dump(self.products, f, indent=2, ensure_ascii=False) + logger.debug(f"Saved {len(self.products)} products") + except IOError as e: + logger.error(f"Error saving products file: {e}") + + def process_products(self, products: List[Product]) -> Tuple[List[Product], List[Product]]: + """ + Process a list of scraped products and detect changes. + Also logs events to database for stats tracking. + + Args: + products: List of products from scraper + + Returns: + Tuple of (new_products, restocked_products) + """ + new_products = [] + restocked_products = [] + + for product in products: + url = product.url + now = datetime.now().isoformat() + + # Get or create product in database + db_product_id = self.db.get_or_create_product( + url=url, + name=product.name, + site=product.site, + product_id=product.product_id, + image_url=product.image_url, + price=product.price, + in_stock=product.in_stock + ) + + if url not in self.products: + # New product! + new_products.append(product) + self.products[url] = { + "name": product.name, + "url": url, + "price": product.price, + "in_stock": product.in_stock, + "image_url": product.image_url, + "site": product.site, + "product_id": product.product_id, + "first_seen": now, + "last_seen": now, + "last_in_stock": now if product.in_stock else None, + "db_id": db_product_id, + } + logger.info(f"NEW PRODUCT: {product.name}") + + # Log new drop event to database + self.db.record_stock_event(db_product_id, "new_drop") + if product.in_stock: + self.db.record_stock_event(db_product_id, "in_stock") + + # Record initial price + if product.price: + self.db.record_price(db_product_id, product.price) + + else: + # Existing product - check for changes + existing = self.products[url] + was_in_stock = existing.get("in_stock", False) + old_price = existing.get("price") + + # Update last seen + existing["last_seen"] = now + existing["image_url"] = product.image_url or existing.get("image_url") + existing["db_id"] = db_product_id + + # Check for price change + if product.price and product.price != old_price: + existing["price"] = product.price + self.db.record_price(db_product_id, product.price) + logger.debug(f"Price change for {product.name}: {old_price} -> {product.price}") + + # Check for restock + if product.in_stock and not was_in_stock: + # RESTOCK! + restocked_products.append(product) + existing["last_in_stock"] = now + logger.info(f"RESTOCK: {product.name}") + + # Log restock event to database + self.db.record_stock_event(db_product_id, "restock") + self.db.record_stock_event(db_product_id, "in_stock") + + # Check for out of stock + elif not product.in_stock and was_in_stock: + # Just went out of stock + self.db.record_stock_event(db_product_id, "out_of_stock") + logger.debug(f"Out of stock: {product.name}") + + existing["in_stock"] = product.in_stock + self.products[url] = existing + + self.save() + return new_products, restocked_products + + def process_products_with_priority(self, products: List[Product]) -> Tuple[List[Tuple[Product, dict]], List[Tuple[Product, dict]]]: + """ + Process products and return with priority/favorite info. + + Returns: + Tuple of (new_products_with_priority, restocked_products_with_priority) + Each item is a tuple of (Product, notification_settings) + """ + new_products, restocked_products = self.process_products(products) + + def add_priority(product: Product) -> Tuple[Product, dict]: + # Get product from DB for category + db_product = self.db.get_product_by_url(product.url) + category = db_product.get('category') if db_product else None + + settings = self.favorites.get_notification_settings( + url=product.url, + name=product.name, + category=category + ) + return (product, settings) + + new_with_priority = [add_priority(p) for p in new_products] + restocked_with_priority = [add_priority(p) for p in restocked_products] + + return new_with_priority, restocked_with_priority + + def log_check(self, site: str, products_found: int, new_count: int, + restock_count: int, duration_ms: int, success: bool = True, + error_message: str = None): + """Log a monitoring check to database""" + self.db.log_check( + site=site, + products_found=products_found, + new_products=new_count, + restocks=restock_count, + duration_ms=duration_ms, + success=success, + error_message=error_message + ) + + def get_known_urls(self) -> Set[str]: + """Get all known product URLs""" + return set(self.products.keys()) + + def get_product(self, url: str) -> Optional[dict]: + """Get a specific product by URL""" + return self.products.get(url) + + def mark_out_of_stock(self, url: str): + """Mark a product as out of stock""" + if url in self.products: + self.products[url]["in_stock"] = False + + # Also update database + db_product = self.db.get_product_by_url(url) + if db_product: + self.db.record_stock_event(db_product['id'], "out_of_stock") + + self.save() + + def clear(self): + """Clear all tracked products (JSON only, database preserved for history)""" + self.products = {} + self.save() + logger.info("Cleared all tracked products from JSON") + + def get_stats(self) -> dict: + """Get tracking statistics from both JSON and database""" + # JSON stats + total = len(self.products) + in_stock = sum(1 for p in self.products.values() if p.get("in_stock", False)) + out_of_stock = total - in_stock + + # Database stats + db_stats = self.db.get_dashboard_stats() + + return { + "total_products": total, + "in_stock": in_stock, + "out_of_stock": out_of_stock, + # From database + "new_drops_today": db_stats.get("new_drops_today", 0), + "new_drops_week": db_stats.get("new_drops_week", 0), + "restocks_today": db_stats.get("restocks_today", 0), + "restocks_week": db_stats.get("restocks_week", 0), + "last_check": db_stats.get("last_check"), + "favorites_count": db_stats.get("favorites_count", 0), + } + + def migrate_to_database(self): + """Migrate existing products.json to database""" + if self.products_file.exists(): + self.db.migrate_from_json(str(self.products_file)) + logger.info("Migration to database complete") diff --git a/src/scraper_state.py b/src/scraper_state.py new file mode 100644 index 0000000..6fb9c18 --- /dev/null +++ b/src/scraper_state.py @@ -0,0 +1,264 @@ +""" +Scraper State Manager +Manages runtime state for scrapers - enabling/disabling, running status, etc. +""" + +import json +import os +import subprocess +import sys +import threading +import time +from datetime import datetime +from pathlib import Path + +STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json" + +# Default state +DEFAULT_STATE = { + "scrapers": { + "pokemoncenter": {"enabled": False, "running": False, "last_run": None, "last_error": None}, + "target": {"enabled": True, "running": False, "last_run": None, "last_error": None}, + "gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None}, + "walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None}, + "bestbuy": {"enabled": False, "running": False, "last_run": None, "last_error": None}, + }, + "monitor_running": False, + "monitor_pid": None, + "check_interval": 60, + "last_check": None, +} + + +class ScraperStateManager: + """Manages scraper state with file persistence""" + + _instance = None + _lock = threading.Lock() + _monitor_process = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + self._initialized = True + self.state = self._load_state() + + def _load_state(self) -> dict: + """Load state from file or return defaults""" + try: + if STATE_FILE.exists(): + with open(STATE_FILE, 'r') as f: + saved = json.load(f) + # Merge with defaults in case new fields added + state = DEFAULT_STATE.copy() + state.update(saved) + # Ensure all scrapers exist + for scraper in DEFAULT_STATE["scrapers"]: + if scraper not in state["scrapers"]: + state["scrapers"][scraper] = DEFAULT_STATE["scrapers"][scraper] + return state + except Exception: + pass + return DEFAULT_STATE.copy() + + def _save_state(self): + """Save state to file""" + with self._lock: + try: + with open(STATE_FILE, 'w') as f: + json.dump(self.state, f, indent=2, default=str) + except Exception as e: + print(f"Error saving scraper state: {e}") + + def get_state(self) -> dict: + """Get current state""" + # Check if monitor is still running + if self.state["monitor_pid"]: + try: + # Check if process exists + import psutil + if not psutil.pid_exists(self.state["monitor_pid"]): + self.state["monitor_running"] = False + self.state["monitor_pid"] = None + for scraper in self.state["scrapers"]: + self.state["scrapers"][scraper]["running"] = False + self._save_state() + except ImportError: + pass # psutil not installed, skip check + + return self.state.copy() + + def set_scraper_enabled(self, scraper: str, enabled: bool) -> bool: + """Enable or disable a scraper""" + if scraper not in self.state["scrapers"]: + return False + + self.state["scrapers"][scraper]["enabled"] = enabled + self._save_state() + return True + + def get_enabled_scrapers(self) -> list: + """Get list of enabled scraper names""" + return [ + name for name, data in self.state["scrapers"].items() + if data.get("enabled", False) + ] + + def set_check_interval(self, seconds: int) -> bool: + """Set check interval""" + if seconds < 10: + return False + self.state["check_interval"] = seconds + self._save_state() + return True + + def record_check(self, scraper: str, success: bool, error: str = None): + """Record a scraper check""" + if scraper in self.state["scrapers"]: + self.state["scrapers"][scraper]["last_run"] = datetime.now().isoformat() + if not success: + self.state["scrapers"][scraper]["last_error"] = error + else: + self.state["scrapers"][scraper]["last_error"] = None + self.state["last_check"] = datetime.now().isoformat() + self._save_state() + + def start_monitor(self) -> dict: + """Start the monitor process""" + if self.state["monitor_running"]: + return {"success": False, "error": "Monitor already running"} + + enabled = self.get_enabled_scrapers() + if not enabled: + return {"success": False, "error": "No scrapers enabled"} + + try: + # Start main.py as a subprocess + project_dir = Path(__file__).parent.parent + main_script = project_dir / "main.py" + + # Create a modified config for enabled scrapers + self._update_config_file(enabled) + + # Start the process + if sys.platform == 'win32': + # Windows: use CREATE_NEW_PROCESS_GROUP for proper handling + process = subprocess.Popen( + [sys.executable, str(main_script)], + cwd=str(project_dir), + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + process = subprocess.Popen( + [sys.executable, str(main_script)], + cwd=str(project_dir), + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + self._monitor_process = process + self.state["monitor_running"] = True + self.state["monitor_pid"] = process.pid + + for scraper in enabled: + self.state["scrapers"][scraper]["running"] = True + + self._save_state() + + return {"success": True, "pid": process.pid, "scrapers": enabled} + + except Exception as e: + return {"success": False, "error": str(e)} + + def stop_monitor(self) -> dict: + """Stop the monitor process""" + if not self.state["monitor_running"] and not self.state["monitor_pid"]: + return {"success": False, "error": "Monitor not running"} + + try: + pid = self.state["monitor_pid"] + + if pid: + # On Windows, use taskkill for reliable process termination + if sys.platform == 'win32': + try: + # Kill the process tree forcefully + subprocess.run( + ['taskkill', '/F', '/T', '/PID', str(pid)], + capture_output=True, + timeout=10 + ) + except Exception: + pass + else: + # On Unix, use psutil + try: + import psutil + proc = psutil.Process(pid) + # Kill child processes first + children = proc.children(recursive=True) + for child in children: + try: + child.kill() # Use kill instead of terminate + except: + pass + # Kill main process + proc.kill() + proc.wait(timeout=5) + except: + pass + + self.state["monitor_running"] = False + self.state["monitor_pid"] = None + for scraper in self.state["scrapers"]: + self.state["scrapers"][scraper]["running"] = False + self._save_state() + + return {"success": True} + + except Exception as e: + # Still update state even if kill failed + self.state["monitor_running"] = False + self.state["monitor_pid"] = None + for scraper in self.state["scrapers"]: + self.state["scrapers"][scraper]["running"] = False + self._save_state() + return {"success": True, "warning": f"Process may need manual cleanup: {str(e)}"} + + def _update_config_file(self, enabled_scrapers: list): + """Update config.py with enabled scrapers""" + config_path = Path(__file__).parent.parent / "config.py" + + try: + with open(config_path, 'r') as f: + content = f.read() + + # Find and replace SITES_ENABLED line + import re + new_line = f'SITES_ENABLED = {json.dumps(enabled_scrapers)}' + pattern = r'SITES_ENABLED\s*=\s*\[.*?\]' + content = re.sub(pattern, new_line, content) + + # Also update CHECK_INTERVAL_SECONDS + interval_line = f'CHECK_INTERVAL_SECONDS = {self.state["check_interval"]}' + interval_pattern = r'CHECK_INTERVAL_SECONDS\s*=\s*\d+' + content = re.sub(interval_pattern, interval_line, content) + + with open(config_path, 'w') as f: + f.write(content) + + except Exception as e: + print(f"Error updating config: {e}") + + +# Singleton instance +scraper_state = ScraperStateManager() diff --git a/src/store_locator.py b/src/store_locator.py new file mode 100644 index 0000000..d493353 --- /dev/null +++ b/src/store_locator.py @@ -0,0 +1,382 @@ +""" +Store Locator Module + +Finds Pokemon TCG products at local retail stores. +Supports Target, Best Buy, GameStop, Walmart. +""" + +import json +import requests +import logging +from typing import List, Dict, Optional +from dataclasses import dataclass +from pathlib import Path + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# User location config +CONFIG_FILE = Path(__file__).parent / "location_config.json" + + +@dataclass +class Store: + name: str + retailer: str + address: str + city: str + state: str + zip_code: str + distance_miles: float + phone: Optional[str] = None + store_id: Optional[str] = None + + +@dataclass +class StoreStock: + store: Store + product_name: str + product_url: str + in_stock: bool + quantity: Optional[int] = None + price: Optional[str] = None + + +class StoreLocator: + """Base class for store locators""" + + def __init__(self, zip_code: str, radius_miles: int = 25): + self.zip_code = zip_code + self.radius_miles = radius_miles + self.session = requests.Session() + self.session.headers.update({ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + }) + + def find_stores(self) -> List[Store]: + """Find nearby stores - override in subclasses""" + raise NotImplementedError + + def check_stock(self, product_id: str) -> List[StoreStock]: + """Check stock at nearby stores - override in subclasses""" + raise NotImplementedError + + +class TargetLocator(StoreLocator): + """Target store locator and stock checker""" + + BASE_URL = "https://redsky.target.com" + + def find_stores(self) -> List[Store]: + """Find nearby Target stores""" + url = f"{self.BASE_URL}/v3/stores/nearby/{self.zip_code}" + params = { + "limit": 20, + "within": self.radius_miles, + "key": "9f36aeafbe60771e321a7cc95a78140772ab3e96" # Public API key + } + + try: + response = self.session.get(url, params=params, timeout=10) + if response.status_code == 200: + data = response.json() + stores = [] + for loc in data.get("locations", []): + addr = loc.get("address", {}) + stores.append(Store( + name=loc.get("location_name", "Target"), + retailer="Target", + address=addr.get("address_line1", ""), + city=addr.get("city", ""), + state=addr.get("state", ""), + zip_code=addr.get("postal_code", ""), + distance_miles=loc.get("distance", 0), + phone=loc.get("telephone", ""), + store_id=str(loc.get("location_id", "")) + )) + return stores + except Exception as e: + logger.error(f"Error finding Target stores: {e}") + + return [] + + def check_stock(self, tcin: str, store_ids: List[str] = None) -> List[StoreStock]: + """ + Check Target stock for a product (TCIN). + + Popular Pokemon TCINs: + - 89776270: Pokemon TCG general + - Search for specific products on Target.com and get TCIN from URL + """ + if not store_ids: + stores = self.find_stores() + store_ids = [s.store_id for s in stores[:5]] # Check closest 5 + + results = [] + + for store_id in store_ids: + url = f"{self.BASE_URL}/v3/fulfillment/stores/{store_id}/products/{tcin}" + params = { + "key": "9f36aeafbe60771e321a7cc95a78140772ab3e96" + } + + try: + response = self.session.get(url, params=params, timeout=10) + if response.status_code == 200: + data = response.json() + product = data.get("product", {}) + avail = product.get("fulfillment", {}).get("store_options", []) + + for opt in avail: + if opt.get("location_id") == store_id: + in_stock = opt.get("in_store_only", {}).get("availability_status") == "IN_STOCK" + results.append(StoreStock( + store=Store( + name="Target", + retailer="Target", + address="", + city="", + state="", + zip_code="", + distance_miles=0, + store_id=store_id + ), + product_name=product.get("item", {}).get("product_description", {}).get("title", "Unknown"), + product_url=f"https://www.target.com/p/-/A-{tcin}", + in_stock=in_stock, + price=product.get("price", {}).get("current_retail_min") + )) + except Exception as e: + logger.error(f"Error checking Target stock: {e}") + + return results + + +class BestBuyLocator(StoreLocator): + """Best Buy store locator and stock checker""" + + def find_stores(self) -> List[Store]: + """Find nearby Best Buy stores""" + url = "https://www.bestbuy.com/site/store-locator/v1/stores" + params = { + "postalCode": self.zip_code, + "radius": self.radius_miles + } + + try: + response = self.session.get(url, params=params, timeout=10) + if response.status_code == 200: + data = response.json() + stores = [] + for store in data.get("stores", []): + stores.append(Store( + name=store.get("name", "Best Buy"), + retailer="Best Buy", + address=store.get("address", ""), + city=store.get("city", ""), + state=store.get("state", ""), + zip_code=store.get("postalCode", ""), + distance_miles=store.get("distance", 0), + phone=store.get("phone", ""), + store_id=str(store.get("storeId", "")) + )) + return stores + except Exception as e: + logger.error(f"Error finding Best Buy stores: {e}") + + return [] + + def check_stock(self, sku: str) -> List[StoreStock]: + """ + Check Best Buy stock for a product (SKU). + + Example: Search for Pokemon on bestbuy.com, get SKU from product page + """ + stores = self.find_stores()[:5] + store_ids = [s.store_id for s in stores] + + if not store_ids: + return [] + + url = f"https://www.bestbuy.com/fulfillment/v1/stores/{','.join(store_ids)}/sku/{sku}" + + try: + response = self.session.get(url, timeout=10) + if response.status_code == 200: + data = response.json() + results = [] + for store_data in data.get("stores", []): + store = next((s for s in stores if s.store_id == str(store_data.get("storeId"))), None) + if store: + results.append(StoreStock( + store=store, + product_name=data.get("productName", "Unknown"), + product_url=f"https://www.bestbuy.com/site/-/{sku}.p", + in_stock=store_data.get("storeAvailability", {}).get("inStoreAvailable", False), + quantity=store_data.get("storeAvailability", {}).get("quantity") + )) + return results + except Exception as e: + logger.error(f"Error checking Best Buy stock: {e}") + + return [] + + +class GameStopLocator(StoreLocator): + """GameStop store locator""" + + def find_stores(self) -> List[Store]: + """Find nearby GameStop stores""" + url = "https://www.gamestop.com/api/location/v1/stores/search" + params = { + "postalCode": self.zip_code, + "radius": self.radius_miles, + "count": 20 + } + + try: + response = self.session.get(url, params=params, timeout=10) + if response.status_code == 200: + data = response.json() + stores = [] + for store in data.get("stores", []): + stores.append(Store( + name=store.get("storeName", "GameStop"), + retailer="GameStop", + address=store.get("address1", ""), + city=store.get("city", ""), + state=store.get("state", ""), + zip_code=store.get("postalCode", ""), + distance_miles=store.get("distance", 0), + phone=store.get("phone", ""), + store_id=str(store.get("storeNumber", "")) + )) + return stores + except Exception as e: + logger.error(f"Error finding GameStop stores: {e}") + + return [] + + +class WalmartLocator(StoreLocator): + """Walmart store locator""" + + def find_stores(self) -> List[Store]: + """Find nearby Walmart stores""" + url = "https://www.walmart.com/store/finder/electrode/api/stores" + params = { + "singleLineAddr": self.zip_code, + "distance": self.radius_miles + } + + try: + response = self.session.get(url, params=params, timeout=10) + if response.status_code == 200: + data = response.json() + stores = [] + for store in data.get("payload", {}).get("storesData", {}).get("stores", []): + addr = store.get("address", {}) + stores.append(Store( + name=store.get("displayName", "Walmart"), + retailer="Walmart", + address=addr.get("streetAddress", ""), + city=addr.get("city", ""), + state=addr.get("state", ""), + zip_code=addr.get("postalCode", ""), + distance_miles=store.get("distance", 0), + phone=store.get("phone", ""), + store_id=str(store.get("id", "")) + )) + return stores + except Exception as e: + logger.error(f"Error finding Walmart stores: {e}") + + return [] + + +# Pokemon TCG product IDs for various retailers +POKEMON_PRODUCTS = { + "target": { + "pokemon_tcg_general": "89776270", + "etb_search": "pokemon+elite+trainer+box", + }, + "bestbuy": { + "pokemon_cards": "6542167", + }, + "gamestop": { + "pokemon_tcg": "pokemon-trading-cards", + }, + "walmart": { + "pokemon_tcg": "pokemon-trading-card-game", + } +} + + +def load_location_config() -> Dict: + """Load user location configuration""" + if CONFIG_FILE.exists(): + with open(CONFIG_FILE, 'r') as f: + return json.load(f) + return {"zip_code": None, "radius_miles": 25} + + +def save_location_config(zip_code: str, radius_miles: int = 25): + """Save user location configuration""" + config = { + "zip_code": zip_code, + "radius_miles": radius_miles + } + with open(CONFIG_FILE, 'w') as f: + json.dump(config, f, indent=2) + logger.info(f"Location saved: {zip_code}, radius: {radius_miles} miles") + + +def find_all_nearby_stores(zip_code: str, radius_miles: int = 25) -> Dict[str, List[Store]]: + """Find all nearby stores across all supported retailers""" + results = {} + + locators = [ + ("Target", TargetLocator(zip_code, radius_miles)), + ("Best Buy", BestBuyLocator(zip_code, radius_miles)), + ("GameStop", GameStopLocator(zip_code, radius_miles)), + ("Walmart", WalmartLocator(zip_code, radius_miles)), + ] + + for name, locator in locators: + try: + stores = locator.find_stores() + results[name] = stores + logger.info(f"Found {len(stores)} {name} stores near {zip_code}") + except Exception as e: + logger.error(f"Error with {name}: {e}") + results[name] = [] + + return results + + +def format_stores_for_discord(stores_by_retailer: Dict[str, List[Store]]) -> str: + """Format store list for Discord message""" + lines = ["**Nearby Stores:**\n"] + + for retailer, stores in stores_by_retailer.items(): + if stores: + lines.append(f"\n**{retailer}** ({len(stores)} stores)") + for store in stores[:3]: # Show top 3 + lines.append(f" - {store.name} ({store.distance_miles:.1f} mi)") + lines.append(f" {store.address}, {store.city}, {store.state}") + + return "\n".join(lines) + + +if __name__ == "__main__": + # Test with a sample zip code + test_zip = "90210" + print(f"Finding stores near {test_zip}...") + + stores = find_all_nearby_stores(test_zip, 25) + + for retailer, store_list in stores.items(): + print(f"\n{retailer}: {len(store_list)} stores found") + for store in store_list[:3]: + print(f" - {store.name} ({store.distance_miles:.1f} mi)") + print(f" {store.address}, {store.city}, {store.state}") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..374728e --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Test modules diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..c688072 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,68 @@ +""" +Test Pokemon Center API endpoints directly +""" + +import requests +import json + +# Headers extracted from HAR capture +HEADERS = { + "Accept": "application/json", + "Accept-Version": "1", + "Content-Type": "application/json", + "X-Store-Locale": "en-us", + "X-Store-Scope": "pokemon", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + "Referer": "https://www.pokemoncenter.com/category/tcg-cards", +} + +# Test endpoints +ENDPOINTS = [ + # Category listing + ("GET", "https://www.pokemoncenter.com/site/resourceapi/category/new-releases"), + + # Product details (example SKU) + ("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/699-17113"), + + # Product status + ("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/status/qgqvbkjwhe4s2mjxgeytg="), +] + +print("=" * 70) +print("Pokemon Center API Test") +print("=" * 70) +print() + +for method, url in ENDPOINTS: + print(f"[{method}] {url[:80]}...") + + try: + if method == "GET": + response = requests.get(url, headers=HEADERS, timeout=10) + else: + response = requests.post(url, headers=HEADERS, timeout=10) + + print(f" Status: {response.status_code}") + print(f" Content-Type: {response.headers.get('Content-Type', 'N/A')}") + + if response.status_code == 200: + try: + data = response.json() + print(f" Response keys: {list(data.keys())[:5]}...") + + # Show a preview + preview = json.dumps(data, indent=2)[:500] + print(f" Preview:\n{preview}") + except: + print(f" Raw: {response.text[:200]}") + else: + print(f" Response: {response.text[:200]}") + + except Exception as e: + print(f" Error: {e}") + + print() + +print("=" * 70) +print("If APIs return 200, we can monitor without a browser!") +print("=" * 70) diff --git a/tests/test_dashboard_api.py b/tests/test_dashboard_api.py new file mode 100644 index 0000000..3272a72 --- /dev/null +++ b/tests/test_dashboard_api.py @@ -0,0 +1,407 @@ +""" +Unit tests for Dashboard REST API endpoints. +""" + +import pytest +import tempfile +import os +import sys +import json + +# Add project root to path +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, project_root) + +from flask import Flask +from src.database import Database + +# Import api module directly to avoid circular imports via dashboard/__init__.py +import importlib.util +spec = importlib.util.spec_from_file_location( + "dashboard_api", + os.path.join(project_root, "dashboard", "api.py") +) +dashboard_api = importlib.util.module_from_spec(spec) +sys.modules["dashboard_api"] = dashboard_api +spec.loader.exec_module(dashboard_api) +api_bp = dashboard_api.api_bp + + +@pytest.fixture +def app(): + """Create test Flask app""" + app = Flask(__name__) + app.config['TESTING'] = True + app.register_blueprint(api_bp, url_prefix='/api') + + # Use temp database + with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f: + db_path = f.name + + # Monkey-patch the database + import src.database as db_module + db_module._db = Database(db_path) + + # Also reset the favorites manager so it picks up the new database + import src.favorites as fav_module + fav_module._favorites = None # Reset so it gets recreated with new db + + yield app + + # Cleanup + os.unlink(db_path) + + +@pytest.fixture +def client(app): + """Create test client""" + return app.test_client() + + +class TestStatsEndpoints: + """Tests for stats and health endpoints""" + + def test_get_stats(self, client): + """Test /api/stats endpoint""" + response = client.get('/api/stats') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'total_products' in data + assert 'in_stock_count' in data + assert 'new_drops_today' in data + + def test_get_health(self, client): + """Test /api/health endpoint""" + response = client.get('/api/health') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert data['status'] == 'ok' + + +class TestProductsEndpoints: + """Tests for product endpoints""" + + def test_get_products_empty(self, client): + """Test getting products when empty""" + response = client.get('/api/products') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'products' in data + assert 'total' in data + + def test_get_products_with_filters(self, client): + """Test product filtering""" + response = client.get('/api/products?site=pokemoncenter&in_stock=true') + + assert response.status_code == 200 + + def test_get_product_not_found(self, client): + """Test getting non-existent product""" + response = client.get('/api/products/99999') + + assert response.status_code == 404 + + +class TestEventsEndpoints: + """Tests for events endpoints""" + + def test_get_events(self, client): + """Test /api/events endpoint""" + response = client.get('/api/events') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'events' in data + + def test_get_events_with_type_filter(self, client): + """Test filtering events by type""" + response = client.get('/api/events?type=restock&type=new_drop') + + assert response.status_code == 200 + + +class TestAnalyticsEndpoints: + """Tests for analytics endpoints""" + + def test_get_drop_timing(self, client): + """Test /api/analytics/drops endpoint""" + response = client.get('/api/analytics/drops') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'drop_timing' in data + + def test_get_stock_duration(self, client): + """Test /api/analytics/stock endpoint""" + response = client.get('/api/analytics/stock') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'stock_duration' in data + + def test_get_selling_rates(self, client): + """Test /api/analytics/selling-rates endpoint""" + response = client.get('/api/analytics/selling-rates') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'products' in data + + def test_get_site_stats(self, client): + """Test /api/analytics/sites endpoint""" + response = client.get('/api/analytics/sites') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'sites' in data + + +class TestExtensionSyncEndpoint: + """Tests for Chrome extension sync endpoint""" + + def test_sync_empty_data(self, client): + """Test syncing with no data returns error""" + response = client.post('/api/extension/sync', + content_type='application/json') + + assert response.status_code == 400 + + def test_sync_skus(self, client): + """Test syncing SKUs from extension""" + data = { + 'skus': ['699-17113', '191-85953', '100-12345'], + 'products': [], + 'events': [] + } + + response = client.post('/api/extension/sync', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + result = json.loads(response.data) + + assert result['success'] is True + assert result['total_skus'] == 3 + assert result['new_skus_added'] == 3 + + def test_sync_products(self, client): + """Test syncing products from extension""" + data = { + 'skus': [], + 'products': [ + { + 'url': 'https://www.pokemoncenter.com/product/699-17113', + 'name': 'Pokemon ETB Prismatic Evolutions', + 'price': '$49.99', + 'inStock': True, + 'imageUrl': 'https://example.com/image.jpg', + 'productId': '699-17113' + } + ], + 'events': [] + } + + response = client.post('/api/extension/sync', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 200 + result = json.loads(response.data) + + assert result['success'] is True + assert result['total_products'] >= 1 + + def test_sync_events(self, client): + """Test syncing events (restocks, drops) from extension""" + # First sync a product + product_data = { + 'skus': [], + 'products': [ + { + 'url': 'https://www.pokemoncenter.com/product/test-123', + 'name': 'Test Product', + 'price': '$29.99', + 'inStock': True + } + ], + 'events': [] + } + client.post('/api/extension/sync', + data=json.dumps(product_data), + content_type='application/json') + + # Now sync events + event_data = { + 'skus': [], + 'products': [], + 'events': [ + { + 'type': 'restock', + 'url': 'https://www.pokemoncenter.com/product/test-123', + 'name': 'Test Product', + 'price': '$29.99', + 'timestamp': '2024-01-15T10:00:00Z' + }, + { + 'type': 'out_of_stock', + 'url': 'https://www.pokemoncenter.com/product/test-123', + 'name': 'Test Product', + 'timestamp': '2024-01-15T10:30:00Z' + } + ] + } + + response = client.post('/api/extension/sync', + data=json.dumps(event_data), + content_type='application/json') + + assert response.status_code == 200 + result = json.loads(response.data) + + assert result['success'] is True + assert result['events_processed'] == 2 + + def test_sync_price_change_event(self, client): + """Test syncing price change events""" + # First create the product + client.post('/api/extension/sync', + data=json.dumps({ + 'products': [{ + 'url': 'https://www.pokemoncenter.com/product/price-test', + 'name': 'Price Test Product', + 'price': '$39.99', + 'inStock': True + }] + }), + content_type='application/json') + + # Now send price change event + event_data = { + 'events': [ + { + 'type': 'price_change', + 'url': 'https://www.pokemoncenter.com/product/price-test', + 'oldPrice': '$39.99', + 'newPrice': '$34.99', + 'timestamp': '2024-01-15T12:00:00Z' + } + ] + } + + response = client.post('/api/extension/sync', + data=json.dumps(event_data), + content_type='application/json') + + assert response.status_code == 200 + result = json.loads(response.data) + assert result['events_processed'] == 1 + + +class TestExtensionDataEndpoints: + """Tests for extension data retrieval endpoints""" + + def test_get_extension_skus(self, client): + """Test getting extension SKUs""" + response = client.get('/api/extension/skus') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'skus' in data + assert 'total' in data + + def test_get_extension_products(self, client): + """Test getting extension products""" + response = client.get('/api/extension/products') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'products' in data + assert 'total' in data + + def test_get_extension_stats(self, client): + """Test getting extension stats""" + response = client.get('/api/extension/stats') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'total_skus' in data + assert 'total_products' in data + assert 'source' in data + assert data['source'] == 'chrome_extension' + + def test_clear_extension_data(self, client): + """Test clearing extension data""" + response = client.post('/api/extension/clear') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert data['success'] is True + + +class TestFavoritesEndpoints: + """Tests for favorites CRUD endpoints""" + + def test_add_favorite(self, client): + """Test adding a favorite""" + data = { + 'type': 'product', + 'value': 'https://example.com/fav-product', + 'display_name': 'My Favorite ETB', + 'priority': 'high' + } + + response = client.post('/api/favorites', + data=json.dumps(data), + content_type='application/json') + + assert response.status_code == 201 + result = json.loads(response.data) + + assert result['success'] is True + assert 'id' in result + + def test_get_favorites(self, client): + """Test getting all favorites""" + response = client.get('/api/favorites') + + assert response.status_code == 200 + data = json.loads(response.data) + + assert 'favorites' in data + + def test_delete_favorite(self, client): + """Test deleting a favorite""" + # First add one + add_response = client.post('/api/favorites', + data=json.dumps({ + 'type': 'category', + 'value': 'ETB' + }), + content_type='application/json') + + fav_id = json.loads(add_response.data)['id'] + + # Now delete it + response = client.delete(f'/api/favorites/{fav_id}') + + assert response.status_code == 200 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..346ec50 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,300 @@ +""" +Unit tests for database operations. +""" + +import pytest +import tempfile +import os +from datetime import datetime, timedelta + +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.database import Database + + +@pytest.fixture +def db(): + """Create a temporary database for testing""" + with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f: + db_path = f.name + + database = Database(db_path) + yield database + + # Cleanup + os.unlink(db_path) + + +class TestProductOperations: + """Tests for product CRUD operations""" + + def test_create_product(self, db): + """Test creating a new product""" + product_id = db.get_or_create_product( + url="https://example.com/product/123", + name="Test Pokemon Card", + site="pokemoncenter", + product_id="123", + price="$19.99", + in_stock=True + ) + + assert product_id is not None + assert product_id > 0 + + def test_get_product_by_url(self, db): + """Test retrieving product by URL""" + url = "https://example.com/product/456" + db.get_or_create_product( + url=url, + name="Pikachu ETB", + site="target", + in_stock=True + ) + + product = db.get_product_by_url(url) + + assert product is not None + assert product['name'] == "Pikachu ETB" + assert product['site'] == "target" + + def test_update_product_price(self, db): + """Test updating product price""" + product_id = db.get_or_create_product( + url="https://example.com/product/789", + name="Booster Box", + site="bestbuy", + price="$99.99", + in_stock=True + ) + + # Update price + db.update_product_price(product_id, "$89.99") + + product = db.get_product(product_id) + assert product['current_price'] == "$89.99" + + # Check price history was recorded + history = db.get_price_history(product_id, days=1) + assert len(history) >= 1 + + def test_update_product_stock(self, db): + """Test updating product stock status""" + product_id = db.get_or_create_product( + url="https://example.com/product/stock-test", + name="Limited Edition", + site="pokemoncenter", + in_stock=True + ) + + # Mark out of stock + db.update_product_stock(product_id, False) + + product = db.get_product(product_id) + assert product['in_stock'] == 0 # SQLite stores as 0/1 + + def test_category_detection(self, db): + """Test automatic category detection from product name""" + test_cases = [ + ("Pokemon Scarlet Elite Trainer Box", "ETB"), + ("Charizard Booster Bundle", "Booster Bundle"), + ("Display Booster Box 36 Packs", "Booster Box"), + ("Sleeved Booster Pack", "Booster Pack"), + ("Premium Collection Box", "Collection Box"), + ("Pokemon Trading Card Tin", "Tin"), + ("Card Binder Album", "Accessories"), + ("Random Pokemon Item", "Other"), + ] + + for name, expected_category in test_cases: + product_id = db.get_or_create_product( + url=f"https://example.com/{name.replace(' ', '-')}", + name=name, + site="test", + in_stock=True + ) + product = db.get_product(product_id) + assert product['category'] == expected_category, f"Failed for: {name}" + + +class TestStockEvents: + """Tests for stock event tracking""" + + def test_record_stock_event(self, db): + """Test recording stock events""" + product_id = db.get_or_create_product( + url="https://example.com/event-test", + name="Event Test Product", + site="test", + in_stock=True + ) + + db.record_stock_event(product_id, "new_drop") + db.record_stock_event(product_id, "out_of_stock") + db.record_stock_event(product_id, "restock") + + events = db.get_recent_events(limit=10) + + # Should have 3 events + product_events = [e for e in events if e['product_id'] == product_id] + assert len(product_events) == 3 + + def test_events_today_count(self, db): + """Test counting events from today""" + product_id = db.get_or_create_product( + url="https://example.com/today-test", + name="Today Test", + site="test", + in_stock=True + ) + + db.record_stock_event(product_id, "new_drop") + db.record_stock_event(product_id, "restock") + + new_drops = db.get_events_today("new_drop") + restocks = db.get_events_today("restock") + + assert new_drops >= 1 + assert restocks >= 1 + + def test_selling_rate_calculation(self, db): + """Test that selling rate can be calculated from events""" + product_id = db.get_or_create_product( + url="https://example.com/sellrate-test", + name="Fast Seller", + site="pokemoncenter", + in_stock=True + ) + + # Simulate: drop -> sold out + db.record_stock_event(product_id, "new_drop") + db.record_stock_event(product_id, "out_of_stock") + + # Get stock duration stats + stats = db.get_stock_duration_stats() + # Should return at least an empty list (stats depend on timing) + assert isinstance(stats, list) + + +class TestPriceHistory: + """Tests for price history tracking""" + + def test_record_price_change(self, db): + """Test that price changes are recorded""" + product_id = db.get_or_create_product( + url="https://example.com/price-test", + name="Price Test", + site="test", + price="$49.99", + in_stock=True + ) + + # Record initial price + db.record_price(product_id, "$49.99") + + # Change price + db.record_price(product_id, "$39.99") + + history = db.get_price_history(product_id, days=1) + + # Should have at least 2 price points + assert len(history) >= 1 + + def test_duplicate_price_not_recorded(self, db): + """Test that same price is not recorded multiple times""" + product_id = db.get_or_create_product( + url="https://example.com/dup-price-test", + name="Dup Price Test", + site="test", + in_stock=True + ) + + # Record same price multiple times + db.record_price(product_id, "$29.99") + db.record_price(product_id, "$29.99") + db.record_price(product_id, "$29.99") + + history = db.get_price_history(product_id, days=1) + + # Should only have 1 entry + assert len(history) == 1 + + +class TestDashboardStats: + """Tests for dashboard statistics""" + + def test_get_dashboard_stats(self, db): + """Test getting dashboard stats summary""" + # Add some test data + for i in range(5): + product_id = db.get_or_create_product( + url=f"https://example.com/stats-{i}", + name=f"Stats Product {i}", + site="pokemoncenter", + in_stock=(i % 2 == 0) + ) + db.record_stock_event(product_id, "new_drop") + + stats = db.get_dashboard_stats() + + assert 'total_products' in stats + assert 'in_stock_count' in stats + assert 'new_drops_today' in stats + assert stats['total_products'] >= 5 + + def test_get_site_stats(self, db): + """Test per-site statistics""" + sites = ['target', 'bestbuy', 'pokemoncenter'] + + for site in sites: + db.get_or_create_product( + url=f"https://{site}.com/test", + name=f"{site} Product", + site=site, + in_stock=True + ) + + stats = db.get_site_stats() + + assert len(stats) == 3 + site_names = [s['site'] for s in stats] + assert 'target' in site_names + assert 'bestbuy' in site_names + + +class TestFavorites: + """Tests for favorites functionality""" + + def test_add_favorite(self, db): + """Test adding a favorite""" + fav_id = db.add_favorite( + fav_type="product", + value="https://example.com/fav-product", + display_name="My Favorite Card", + priority="high" + ) + + assert fav_id is not None + + favorite = db.get_favorite(fav_id) + assert favorite['display_name'] == "My Favorite Card" + + def test_check_is_favorite(self, db): + """Test checking if product is favorited""" + url = "https://example.com/check-fav" + + # Not a favorite yet + result = db.check_is_favorite(url=url) + assert result is None + + # Add as favorite + db.add_favorite(fav_type="product", value=url) + + # Now should be found + result = db.check_is_favorite(url=url) + assert result is not None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_product_filter.py b/tests/test_product_filter.py new file mode 100644 index 0000000..a0155b2 --- /dev/null +++ b/tests/test_product_filter.py @@ -0,0 +1,175 @@ +""" +Unit tests for Pokemon product validation filter. +""" + +import pytest +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scrapers.base import BaseScraper, Product + + +def make_product(name: str, url: str = "", site: str = "test") -> Product: + """Helper to create Product with required fields""" + return Product(name=name, url=url, price=None, in_stock=True, site=site) + + +class MockScraper(BaseScraper): + """Mock scraper for testing base class methods""" + + site_name = "test" + + def scrape_category_page(self, url: str): + return [] + + def check_product_stock(self, product): + return True, "$19.99" + + +@pytest.fixture +def scraper(): + return MockScraper() + + +class TestPokemonProductFilter: + """Tests for is_pokemon_product validation""" + + def test_valid_pokemon_products(self, scraper): + """Test that valid Pokemon products are accepted""" + valid_products = [ + make_product("Pokemon Scarlet & Violet Elite Trainer Box"), + make_product("Pikachu V Collection Box"), + make_product("TCG Booster Pack Prismatic Evolutions"), + make_product("Charizard Premium Collection"), + make_product("Pokemon ETB Paldean Fates"), + make_product("Pokémon Trading Card Game Tin"), # Unicode é + make_product("Mewtwo VMAX Box Set"), + make_product("Eevee Heroes Booster Box"), + ] + + for product in valid_products: + assert scraper.is_pokemon_product(product), f"Should accept: {product.name}" + + def test_invalid_non_pokemon_products(self, scraper): + """Test that non-Pokemon products are rejected""" + invalid_products = [ + make_product("Ice Cube Tray Silicone Mold"), + make_product("Barbie Dream House Playset"), + make_product("Hot Wheels Track Builder"), + make_product("LEGO Star Wars Set"), + make_product("Kitchen Appliance Blender"), + make_product("Transformers Action Figure"), + make_product("Room Essentials Bedding Set"), + make_product("Threshold Furniture Table"), + make_product("Random Gaming Accessory"), + ] + + for product in invalid_products: + assert not scraper.is_pokemon_product(product), f"Should reject: {product.name}" + + def test_exclusion_takes_priority(self, scraper): + """Test that exclusion terms override Pokemon terms""" + # This has "pokemon" but also "ice cube" - should be rejected + product = make_product("Pokemon Ice Cube Tray Silicone") + assert not scraper.is_pokemon_product(product) + + def test_case_insensitive(self, scraper): + """Test that matching is case insensitive""" + products = [ + make_product("POKEMON ELITE TRAINER BOX"), + make_product("pokemon scarlet booster"), + make_product("PoKeMoN ChArIzArD"), + ] + + for product in products: + assert scraper.is_pokemon_product(product), f"Should accept: {product.name}" + + +class TestFilterPokemonProducts: + """Tests for filter_pokemon_products batch filtering""" + + def test_filter_mixed_products(self, scraper): + """Test filtering a mix of valid and invalid products""" + products = [ + make_product("Pokemon Booster Pack", url="1"), + make_product("Ice Cube Tray", url="2"), + make_product("Charizard Collection", url="3"), + make_product("Barbie Doll", url="4"), + make_product("Pikachu Tin", url="5"), + ] + + filtered = scraper.filter_pokemon_products(products) + + assert len(filtered) == 3 + urls = [p.url for p in filtered] + assert "1" in urls + assert "3" in urls + assert "5" in urls + assert "2" not in urls + assert "4" not in urls + + def test_filter_empty_list(self, scraper): + """Test filtering empty list returns empty""" + filtered = scraper.filter_pokemon_products([]) + assert filtered == [] + + def test_filter_all_valid(self, scraper): + """Test filtering when all products are valid""" + products = [ + make_product("Pokemon ETB", url="1"), + make_product("Pokemon Booster", url="2"), + ] + + filtered = scraper.filter_pokemon_products(products) + assert len(filtered) == 2 + + def test_filter_all_invalid(self, scraper): + """Test filtering when all products are invalid""" + products = [ + make_product("Random Item", url="1"), + make_product("Another Thing", url="2"), + ] + + filtered = scraper.filter_pokemon_products(products) + assert len(filtered) == 0 + + +class TestProductDataclass: + """Tests for Product dataclass""" + + def test_product_creation(self): + """Test creating a Product with all fields""" + product = Product( + name="Test Product", + url="https://example.com/test", + price="$29.99", + in_stock=True, + site="pokemoncenter", + image_url="https://example.com/image.jpg", + product_id="123" + ) + + assert product.name == "Test Product" + assert product.price == "$29.99" + assert product.in_stock is True + + def test_product_minimal(self): + """Test Product with only required fields""" + product = Product( + name="Minimal Product", + url="https://example.com", + price=None, + in_stock=False + ) + + assert product.name == "Minimal Product" + assert product.price is None + assert product.in_stock is False + assert product.image_url is None + assert product.product_id is None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_stealth.py b/tests/test_stealth.py new file mode 100644 index 0000000..707af6c --- /dev/null +++ b/tests/test_stealth.py @@ -0,0 +1,111 @@ +""" +Test script for the stealth browser module +Run this to verify undetected-chromedriver is working +""" + +import sys +import time + +print("=" * 60) +print("Stealth Browser Test") +print("=" * 60) +print() + +# Check dependencies +print("1. Checking dependencies...") +try: + import undetected_chromedriver as uc + print(" [OK] undetected-chromedriver installed") +except ImportError: + print(" [ERROR] undetected-chromedriver not installed!") + print(" Run: pip install undetected-chromedriver") + sys.exit(1) + +try: + from selenium.webdriver.common.action_chains import ActionChains + print(" [OK] selenium installed") +except ImportError: + print(" [ERROR] selenium not installed!") + print(" Run: pip install selenium") + sys.exit(1) + +# Import our stealth browser +print() +print("2. Importing stealth browser module...") +try: + from stealth_browser import StealthBrowser, get_stealth_browser + print(" [OK] stealth_browser module loaded") +except Exception as e: + print(f" [ERROR] Failed to import: {e}") + sys.exit(1) + +# Test browser launch +print() +print("3. Launching stealth browser...") +print(" (A Chrome window should open)") +print() + +browser = None +try: + browser = StealthBrowser( + headless=False, + session_name="test_session" + ) + browser.start() + print(" [OK] Browser started successfully!") + + # Test navigation + print() + print("4. Testing navigation to Pokemon Center...") + print(" (Watch the browser window)") + + url = "https://www.pokemoncenter.com/category/tcg-cards" + html = browser.get_page(url, wait_time=5) + + print(f" [OK] Page loaded - {len(html)} bytes") + + # Check for CAPTCHA + print() + print("5. Checking for bot detection...") + if browser.check_for_captcha(): + print(" [!] CAPTCHA/Challenge detected!") + print(" The browser window is open - solve it manually if needed") + print(" Waiting up to 60 seconds...") + browser.wait_for_captcha_solve(timeout=60) + else: + print(" [OK] No CAPTCHA detected! Stealth mode working.") + + # Show some page info + print() + print("6. Page analysis...") + from bs4 import BeautifulSoup + soup = BeautifulSoup(html, 'html.parser') + title = soup.title.string if soup.title else "No title" + print(f" Page title: {title}") + + # Count potential product elements + products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']") + print(f" Potential product elements: {len(products)}") + + print() + print("=" * 60) + print("TEST COMPLETE") + print("=" * 60) + print() + print("The browser window will stay open for 10 seconds so you can inspect it.") + print("Press Ctrl+C to close early.") + + time.sleep(10) + +except KeyboardInterrupt: + print("\n Interrupted by user") +except Exception as e: + print(f" [ERROR] {e}") + import traceback + traceback.print_exc() +finally: + if browser: + print() + print("Closing browser...") + browser.stop() + print("Done!") diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..fa05096 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +# Utility and experimental scripts diff --git a/tools/analyze_har.py b/tools/analyze_har.py new file mode 100644 index 0000000..95ec100 --- /dev/null +++ b/tools/analyze_har.py @@ -0,0 +1,336 @@ +""" +HAR File Analyzer for Pokemon Center API Discovery + +Analyzes a HAR (HTTP Archive) file exported from Chrome DevTools +to find API endpoints used by Pokemon Center. + +Usage: + 1. Export HAR from Chrome DevTools Network tab + 2. Save as pokemon_network.har in this folder + 3. Run: python analyze_har.py + +Or specify a different file: + python analyze_har.py my_capture.har +""" + +import json +import sys +import re +from pathlib import Path +from urllib.parse import urlparse, parse_qs +from collections import defaultdict + +# Patterns that indicate interesting API endpoints +INTERESTING_PATTERNS = [ + r'/api/', + r'/graphql', + r'/v\d+/', + r'product', + r'catalog', + r'search', + r'inventory', + r'stock', + r'\.json$', + r'algolia', + r'contentful', + r'commercetools', + r'demandware', + r'sfcc', +] + +# Skip these resource types +SKIP_TYPES = [ + 'image', 'stylesheet', 'font', 'script', 'media', + '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', + '.css', '.woff', '.woff2', '.ttf', + '.js', # Usually not APIs, but we'll catch fetch calls +] + +def should_skip(url, mime_type=''): + """Check if we should skip this request""" + url_lower = url.lower() + mime_lower = mime_type.lower() + + for skip in SKIP_TYPES: + if skip in url_lower or skip in mime_lower: + return True + return False + +def is_interesting(url): + """Check if URL matches patterns we care about""" + url_lower = url.lower() + for pattern in INTERESTING_PATTERNS: + if re.search(pattern, url_lower): + return True + return False + +def extract_json_preview(content, max_length=500): + """Try to extract and preview JSON content""" + if not content: + return None + + try: + # HAR stores content as text or base64 + text = content.get('text', '') + if not text: + return None + + # Try to parse as JSON + data = json.loads(text) + + # Return a preview + preview = json.dumps(data, indent=2) + if len(preview) > max_length: + return preview[:max_length] + "\n... (truncated)" + return preview + except: + return None + +def analyze_har(har_path): + """Analyze a HAR file for API endpoints""" + + print(f"\nLoading {har_path}...") + + with open(har_path, 'r', encoding='utf-8') as f: + har_data = json.load(f) + + entries = har_data.get('log', {}).get('entries', []) + print(f"Found {len(entries)} network requests") + + # Categorize requests + api_calls = [] + json_responses = [] + graphql_calls = [] + third_party_apis = [] + + domains = defaultdict(int) + + for entry in entries: + request = entry.get('request', {}) + response = entry.get('response', {}) + + url = request.get('url', '') + method = request.get('method', 'GET') + status = response.get('status', 0) + mime_type = response.get('content', {}).get('mimeType', '') + + # Track domains + parsed = urlparse(url) + domains[parsed.netloc] += 1 + + # Skip resources we don't care about + if should_skip(url, mime_type): + continue + + # Check for JSON responses + if 'json' in mime_type.lower(): + content_preview = extract_json_preview(response.get('content', {})) + json_responses.append({ + 'url': url, + 'method': method, + 'status': status, + 'preview': content_preview, + 'headers': {h['name']: h['value'] for h in request.get('headers', [])}, + 'response_headers': {h['name']: h['value'] for h in response.get('headers', [])} + }) + + # Check for GraphQL + if 'graphql' in url.lower(): + post_data = request.get('postData', {}) + graphql_calls.append({ + 'url': url, + 'method': method, + 'body': post_data.get('text', ''), + 'status': status + }) + + # Check for interesting patterns + if is_interesting(url): + api_calls.append({ + 'url': url, + 'method': method, + 'status': status, + 'mime': mime_type + }) + + # Check for third-party APIs + if any(tp in url.lower() for tp in ['algolia', 'contentful', 'commercetools', 'sfcc']): + third_party_apis.append({ + 'url': url, + 'method': method, + 'status': status + }) + + # Print analysis + print("\n" + "=" * 70) + print("HAR ANALYSIS RESULTS") + print("=" * 70) + + # Domains summary + print("\n[DOMAINS CONTACTED]") + print("-" * 40) + for domain, count in sorted(domains.items(), key=lambda x: -x[1])[:15]: + print(f" {count:4d} requests {domain}") + + # GraphQL calls + if graphql_calls: + print("\n[GRAPHQL ENDPOINTS]") + print("-" * 40) + for call in graphql_calls: + print(f"\n [{call['method']}] {call['url']}") + print(f" Status: {call['status']}") + if call['body']: + try: + body = json.loads(call['body']) + if 'query' in body: + query_preview = body['query'][:200].replace('\n', ' ') + print(f" Query: {query_preview}...") + if 'operationName' in body: + print(f" Operation: {body['operationName']}") + except: + print(f" Body: {call['body'][:200]}...") + + # JSON responses (potential APIs) + if json_responses: + print("\n[JSON RESPONSES - Potential APIs]") + print("-" * 40) + + # Filter to most interesting ones + product_related = [r for r in json_responses if 'product' in r['url'].lower()] + other_json = [r for r in json_responses if 'product' not in r['url'].lower()][:10] + + for resp in product_related + other_json: + print(f"\n [{resp['method']}] {resp['url'][:100]}") + print(f" Status: {resp['status']}") + + if resp['preview']: + print(f" Preview:") + for line in resp['preview'].split('\n')[:10]: + print(f" {line}") + + # Third-party APIs + if third_party_apis: + print("\n[THIRD-PARTY APIs]") + print("-" * 40) + for api in third_party_apis: + print(f" [{api['method']}] {api['url'][:100]}") + + # Interesting endpoints + if api_calls: + print("\n[OTHER INTERESTING ENDPOINTS]") + print("-" * 40) + seen = set() + for call in api_calls: + # Dedupe by base URL + parsed = urlparse(call['url']) + base = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + if base in seen: + continue + seen.add(base) + + print(f" [{call['method']}] {call['url'][:100]}") + print(f" Status: {call['status']}, Type: {call['mime']}") + + # Save detailed results + output_file = Path(har_path).stem + "_analysis.json" + output_path = Path(har_path).parent / output_file + + with open(output_path, 'w') as f: + json.dump({ + 'summary': { + 'total_requests': len(entries), + 'json_responses': len(json_responses), + 'graphql_calls': len(graphql_calls), + 'third_party_apis': len(third_party_apis), + 'interesting_endpoints': len(api_calls) + }, + 'domains': dict(domains), + 'json_responses': json_responses, + 'graphql_calls': graphql_calls, + 'third_party_apis': third_party_apis, + 'api_calls': api_calls + }, f, indent=2) + + print(f"\n[SAVED] Detailed results saved to: {output_path}") + + # Recommendations + print("\n" + "=" * 70) + print("RECOMMENDATIONS") + print("=" * 70) + + if graphql_calls: + print(""" +[OK] GraphQL endpoint found! This is likely the main data source. + Next steps: + 1. Examine the query structure in the analysis JSON + 2. Test if the endpoint works without cookies/auth + 3. Build a monitor that polls this endpoint +""") + + if any('algolia' in api['url'].lower() for api in third_party_apis): + print(""" +[OK] Algolia search detected! This is often used for product search. + Next steps: + 1. Find the Algolia App ID and Search API Key (usually in page source) + 2. Query Algolia directly - very fast and low detection risk + 3. Search for "apiKey" or "applicationId" in the page source +""") + + if json_responses and not graphql_calls: + print(""" +[INFO] JSON responses found but no GraphQL. Check the analysis JSON for: + - Responses containing product arrays + - URLs with /api/ or /v1/, /v2/ patterns + - Look for pagination parameters (page, limit, offset) +""") + + if not json_responses and not graphql_calls: + print(""" +[WARNING] No obvious API endpoints found. Possible reasons: + - Server-side rendering (data embedded in HTML) + - API calls blocked by bot protection + - Need to scroll/interact more to trigger lazy loading + +Try: + - Scrolling more on the page before exporting HAR + - Clicking on product filters + - Looking at individual product pages +""") + +def main(): + # Default HAR file location + default_har = Path(__file__).parent / "pokemon_network.har" + + # Check command line args + if len(sys.argv) > 1: + har_path = Path(sys.argv[1]) + else: + har_path = default_har + + if not har_path.exists(): + print("=" * 70) + print("HAR File Analyzer") + print("=" * 70) + print(f""" +No HAR file found at: {har_path} + +To capture a HAR file: +1. Open Chrome and go to: https://www.pokemoncenter.com/category/tcg-cards +2. Press F12 to open DevTools +3. Go to the Network tab +4. Check "Preserve log" checkbox +5. Refresh the page (F5) +6. Scroll down to load more products +7. Right-click in Network panel → "Save all as HAR with content" +8. Save as: {default_har} +9. Run this script again + +Or specify a HAR file: + python analyze_har.py path/to/your/file.har +""") + return + + analyze_har(har_path) + +if __name__ == "__main__": + main() diff --git a/tools/api_discovery.py b/tools/api_discovery.py new file mode 100644 index 0000000..9aec25a --- /dev/null +++ b/tools/api_discovery.py @@ -0,0 +1,267 @@ +""" +API Discovery Tool for Pokemon Center +Attempts to find and test API endpoints that could be used instead of browser scraping. +""" + +import requests +import json +from urllib.parse import urljoin + +# Common API patterns to try +BASE_URL = "https://www.pokemoncenter.com" + +# Headers to mimic a real browser +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "application/json, text/plain, */*", + "Accept-Language": "en-US,en;q=0.9", + "Referer": "https://www.pokemoncenter.com/", + "Origin": "https://www.pokemoncenter.com", +} + +# Common API endpoint patterns for e-commerce sites +API_PATTERNS = [ + # REST API patterns + "/api/products", + "/api/v1/products", + "/api/v2/products", + "/api/catalog/products", + "/api/search", + "/api/inventory", + + # GraphQL + "/graphql", + "/api/graphql", + + # Common e-commerce platforms + "/rest/V1/products", # Magento + "/_api/products", # Wix + "/cdn/shop/products.json", # Shopify pattern + "/products.json", # Shopify + + # Search APIs + "/api/search/products", + "/search/suggest", + "/api/autocomplete", + + # Algolia (very common for e-commerce search) + # Note: Algolia requires app ID and API key from the page +] + +# Pokemon TCG specific search terms +SEARCH_TERMS = ["pokemon", "tcg", "cards", "booster", "etb"] + + +def test_endpoint(url: str, method: str = "GET", data: dict = None) -> dict: + """Test an API endpoint""" + try: + if method == "GET": + response = requests.get(url, headers=HEADERS, timeout=10) + else: + response = requests.post(url, headers=HEADERS, json=data, timeout=10) + + return { + "url": url, + "status": response.status_code, + "content_type": response.headers.get("content-type", ""), + "size": len(response.content), + "sample": response.text[:500] if response.status_code == 200 else None + } + except Exception as e: + return { + "url": url, + "status": "error", + "error": str(e) + } + + +def discover_apis(): + """Attempt to discover API endpoints""" + print("=" * 60) + print("Pokemon Center API Discovery") + print("=" * 60) + print() + + results = [] + + # Test common patterns + print("Testing common API patterns...") + for pattern in API_PATTERNS: + url = urljoin(BASE_URL, pattern) + result = test_endpoint(url) + results.append(result) + + if result["status"] == 200: + print(f" [OK] {url}") + print(f" Content-Type: {result['content_type']}") + print(f" Size: {result['size']} bytes") + elif result["status"] != "error" and result["status"] < 500: + print(f" [{result['status']}] {url}") + + # Test search with query params + print() + print("Testing search endpoints...") + search_patterns = [ + "/api/search?q=pokemon", + "/api/products?search=tcg", + "/api/catalog?category=tcg-cards", + "/search?q=pokemon+tcg", + ] + + for pattern in search_patterns: + url = urljoin(BASE_URL, pattern) + result = test_endpoint(url) + results.append(result) + + if result["status"] == 200: + print(f" [OK] {url}") + + # Look for Algolia configuration + print() + print("Checking for Algolia search...") + # Algolia is often exposed in page source + try: + response = requests.get(BASE_URL, headers=HEADERS, timeout=15) + if "algolia" in response.text.lower(): + print(" [!] Algolia detected in page source") + # Try to extract app ID and search key + import re + app_id = re.search(r'["\']?algolia[_-]?app[_-]?id["\']?\s*[:=]\s*["\']([A-Z0-9]+)["\']', response.text, re.I) + api_key = re.search(r'["\']?algolia[_-]?(?:search[_-]?)?(?:api[_-]?)?key["\']?\s*[:=]\s*["\']([a-f0-9]+)["\']', response.text, re.I) + + if app_id: + print(f" App ID: {app_id.group(1)}") + if api_key: + print(f" Search Key: {api_key.group(1)}") + + # Check for other API clues + if "graphql" in response.text.lower(): + print(" [!] GraphQL detected in page source") + + if "__NEXT_DATA__" in response.text: + print(" [!] Next.js detected - may have API routes at /api/*") + + if "window.__INITIAL_STATE__" in response.text or "window.__PRELOADED_STATE__" in response.text: + print(" [!] Pre-rendered state detected - data may be in page source") + + except Exception as e: + print(f" Error checking main page: {e}") + + # Summary + print() + print("=" * 60) + print("Summary") + print("=" * 60) + + working = [r for r in results if r.get("status") == 200] + if working: + print(f"Found {len(working)} potentially working endpoints:") + for r in working: + print(f" - {r['url']}") + else: + print("No direct API endpoints found.") + print() + print("Alternative approaches to consider:") + print(" 1. Monitor sitemap.xml for new products") + print(" 2. Use Google Shopping API or similar aggregators") + print(" 3. Check if they have an RSS feed") + print(" 4. Use a service like Distill.io for change detection") + print(" 5. Proxy rotation with residential IPs") + print(" 6. Lower check frequency + add human-like delays") + + return results + + +def check_sitemap(): + """Check sitemap for product URLs""" + print() + print("Checking sitemap...") + + sitemap_urls = [ + "/sitemap.xml", + "/sitemap_index.xml", + "/sitemaps/sitemap.xml", + "/robots.txt", # Often contains sitemap location + ] + + for pattern in sitemap_urls: + url = urljoin(BASE_URL, pattern) + result = test_endpoint(url) + + if result["status"] == 200: + print(f" [OK] {url}") + if "sitemap" in result.get("sample", "").lower(): + print(f" Contains sitemap references") + if "product" in result.get("sample", "").lower(): + print(f" Contains product references") + + +def check_rss(): + """Check for RSS feeds""" + print() + print("Checking for RSS/Atom feeds...") + + feed_urls = [ + "/feed", + "/rss", + "/feed.xml", + "/rss.xml", + "/atom.xml", + "/blog/feed", + "/news/feed", + ] + + for pattern in feed_urls: + url = urljoin(BASE_URL, pattern) + result = test_endpoint(url) + + if result["status"] == 200 and ("xml" in result.get("content_type", "") or "rss" in result.get("content_type", "")): + print(f" [OK] {url}") + + +if __name__ == "__main__": + discover_apis() + check_sitemap() + check_rss() + + print() + print("=" * 60) + print("Next Steps") + print("=" * 60) + print(""" +To avoid bot detection, consider these strategies: + +1. API-BASED MONITORING (if endpoints found): + - Call API endpoints directly with requests + - Much faster and less detectable than browser + - Can check more frequently + +2. SITEMAP MONITORING: + - Parse sitemap.xml periodically + - Detect new product URLs without visiting pages + - Very low detection risk + +3. HASH-BASED CHANGE DETECTION: + - Fetch page, hash content + - Only alert when hash changes + - Reduces unnecessary processing + +4. RESIDENTIAL PROXY ROTATION: + - Use services like Bright Data, Oxylabs + - Rotate IPs to avoid blocks + - More expensive but reliable + +5. HUMAN-LIKE BEHAVIOR: + - Random delays between 60-180 seconds + - Vary user agent strings + - Add mouse movements and scrolling + - Use real browser cookies + +6. THIRD-PARTY ALERTS: + - Discord servers that track Pokemon Center + - Stock alert services (NowInStock, etc.) + - Browser extensions like Distill.io + +Run this script to see what APIs are available: + python api_discovery.py +""") diff --git a/tools/api_monitor.py b/tools/api_monitor.py new file mode 100644 index 0000000..cda2b12 --- /dev/null +++ b/tools/api_monitor.py @@ -0,0 +1,239 @@ +""" +Pokemon Center API Monitor + +Uses cookies from a browser session to make API calls directly. +Much lighter than full browser scraping once cookies are established. + +Workflow: +1. Run warmup_session.py first to get valid cookies +2. This script uses those cookies to call APIs directly +3. Falls back to browser if cookies expire +""" + +import json +import time +import pickle +import logging +import requests +from pathlib import Path +from datetime import datetime +from typing import Optional, Dict, List, Any + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Session/cookie storage +SESSION_DIR = Path(__file__).parent / "sessions" +COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl" + +# API endpoints discovered from HAR analysis +API_BASE = "https://www.pokemoncenter.com" +ENDPOINTS = { + "product": "/tpci-ecommweb-api/product/{sku}", + "status": "/tpci-ecommweb-api/product/status/{encoded_id}", + "category": "/site/resourceapi/category/{category}", + "reviews": "/tpci-ecommweb-api/review/get-product-scores", +} + +# Required headers from HAR capture +BASE_HEADERS = { + "Accept": "application/json", + "Accept-Version": "1", + "Content-Type": "application/json", + "X-Store-Locale": "en-us", + "X-Store-Scope": "pokemon", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + "Referer": "https://www.pokemoncenter.com/category/tcg-cards", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", +} + +# Known product SKUs for TCG (build this list over time) +KNOWN_TCG_SKUS_FILE = Path(__file__).parent / "known_tcg_skus.json" + + +class PokemonCenterAPI: + """Direct API client for Pokemon Center using session cookies""" + + def __init__(self): + self.session = requests.Session() + self.session.headers.update(BASE_HEADERS) + self.cookies_loaded = False + self.last_cookie_refresh = None + + def load_cookies(self) -> bool: + """Load cookies from the saved session file""" + if not COOKIE_FILE.exists(): + logger.warning(f"No cookie file found at {COOKIE_FILE}") + logger.info("Run warmup_session.py first to create a valid session") + return False + + try: + with open(COOKIE_FILE, 'rb') as f: + cookies = pickle.load(f) + + # Add cookies to session + for cookie in cookies: + self.session.cookies.set( + cookie['name'], + cookie['value'], + domain=cookie.get('domain', '.pokemoncenter.com'), + path=cookie.get('path', '/') + ) + + logger.info(f"Loaded {len(cookies)} cookies from session file") + self.cookies_loaded = True + self.last_cookie_refresh = datetime.now() + return True + + except Exception as e: + logger.error(f"Failed to load cookies: {e}") + return False + + def _make_request(self, method: str, url: str, **kwargs) -> Optional[requests.Response]: + """Make a request with error handling""" + try: + if method.upper() == "GET": + response = self.session.get(url, timeout=15, **kwargs) + else: + response = self.session.post(url, timeout=15, **kwargs) + + # Check for blocking + if response.status_code == 403: + if "captcha" in response.text.lower() or "blocked" in response.text.lower(): + logger.warning("Request blocked - cookies may have expired") + self.cookies_loaded = False + return None + + return response + + except requests.RequestException as e: + logger.error(f"Request failed: {e}") + return None + + def get_product(self, sku: str) -> Optional[Dict]: + """Get product details by SKU""" + if not self.cookies_loaded: + if not self.load_cookies(): + return None + + url = f"{API_BASE}{ENDPOINTS['product'].format(sku=sku)}" + response = self._make_request("GET", url) + + if response and response.status_code == 200: + return response.json() + return None + + def get_product_status(self, encoded_id: str) -> Optional[Dict]: + """Get product availability status""" + if not self.cookies_loaded: + if not self.load_cookies(): + return None + + url = f"{API_BASE}{ENDPOINTS['status'].format(encoded_id=encoded_id)}" + response = self._make_request("GET", url) + + if response and response.status_code == 200: + return response.json() + return None + + def get_category(self, category: str = "new-releases") -> Optional[Dict]: + """Get category listing (potential goldmine for new drops!)""" + if not self.cookies_loaded: + if not self.load_cookies(): + return None + + url = f"{API_BASE}{ENDPOINTS['category'].format(category=category)}" + response = self._make_request("GET", url) + + if response and response.status_code == 200: + return response.json() + return None + + def get_review_scores(self, sku_list: List[str]) -> Optional[Dict]: + """Get review scores for multiple SKUs""" + if not self.cookies_loaded: + if not self.load_cookies(): + return None + + skus = ",".join(sku_list) + url = f"{API_BASE}{ENDPOINTS['reviews']}?skuList={skus}" + response = self._make_request("GET", url) + + if response and response.status_code == 200: + return response.json() + return None + + +def load_known_skus() -> List[str]: + """Load list of known TCG SKUs""" + if KNOWN_TCG_SKUS_FILE.exists(): + with open(KNOWN_TCG_SKUS_FILE, 'r') as f: + return json.load(f) + return [] + + +def save_known_skus(skus: List[str]): + """Save list of known TCG SKUs""" + with open(KNOWN_TCG_SKUS_FILE, 'w') as f: + json.dump(skus, f, indent=2) + + +def test_api(): + """Test the API client""" + print("=" * 70) + print("Pokemon Center API Monitor Test") + print("=" * 70) + print() + + api = PokemonCenterAPI() + + if not api.load_cookies(): + print("\nNo valid cookies found!") + print("Please run: python warmup_session.py") + print("Then try again.") + return + + print("\n[1] Testing category endpoint (new-releases)...") + category_data = api.get_category("new-releases") + if category_data: + print(" SUCCESS! Category data retrieved.") + print(f" Keys: {list(category_data.keys())[:5]}") + + # Save for analysis + with open("category_response.json", "w") as f: + json.dump(category_data, f, indent=2) + print(" Saved to category_response.json") + else: + print(" FAILED - cookies may have expired") + + print("\n[2] Testing product endpoint...") + # Try a known SKU from our HAR capture + product_data = api.get_product("699-17113") + if product_data: + print(" SUCCESS! Product data retrieved.") + print(f" Keys: {list(product_data.keys())[:5]}") + else: + print(" FAILED - cookies may have expired") + + print("\n[3] Testing review scores endpoint...") + reviews = api.get_review_scores(["699-17113", "191-85953"]) + if reviews: + print(" SUCCESS! Review scores retrieved.") + print(f" Data: {reviews}") + else: + print(" FAILED - cookies may have expired") + + print() + print("=" * 70) + if category_data or product_data: + print("API access working! Can monitor without full browser scraping.") + print("Cookies will expire eventually - re-run warmup when needed.") + else: + print("API blocked. Need fresh cookies from browser session.") + print("=" * 70) + + +if __name__ == "__main__": + test_api() diff --git a/tools/backend_monitor.py b/tools/backend_monitor.py new file mode 100644 index 0000000..26bdbd2 --- /dev/null +++ b/tools/backend_monitor.py @@ -0,0 +1,370 @@ +""" +Pokemon Center Backend Monitor + +Detects NEW products added to the API before they're publicly announced. +This is how accounts like @pokepullzhq detect drops early. + +Strategy: +1. Maintain a list of all known product SKUs +2. Periodically check the API for current products +3. Compare: Any new SKUs = potential silent drop +4. Alert immediately on new detections + +Usage: +1. First run warmup_session.py to get valid cookies +2. Then run: python backend_monitor.py +""" + +import json +import time +import pickle +import logging +import requests +from pathlib import Path +from datetime import datetime +from typing import Optional, Dict, List, Set + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# File paths +DATA_DIR = Path(__file__).parent.parent / "data" +SESSION_DIR = DATA_DIR / "sessions" +COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl" +KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json" +DETECTIONS_LOG = DATA_DIR / "detections.json" + +# API configuration +API_BASE = "https://www.pokemoncenter.com" + +# Categories to monitor for new products +MONITOR_CATEGORIES = [ + "new-releases", + "tcg-cards", + # Add more as needed +] + +# Headers required for API calls +HEADERS = { + "Accept": "application/json", + "Accept-Version": "1", + "Content-Type": "application/json", + "X-Store-Locale": "en-us", + "X-Store-Scope": "pokemon", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + "Referer": "https://www.pokemoncenter.com/", +} + +# Check interval (seconds) +CHECK_INTERVAL = 60 + + +class BackendMonitor: + """Monitors Pokemon Center API for new product drops""" + + def __init__(self): + self.session = requests.Session() + self.session.headers.update(HEADERS) + self.known_skus: Set[str] = set() + self.cookies_valid = False + self.detections: List[Dict] = [] + + def load_cookies(self) -> bool: + """Load cookies from warmup session""" + if not COOKIE_FILE.exists(): + logger.error(f"No cookie file found at {COOKIE_FILE}") + logger.info("Run warmup_session.py first!") + return False + + try: + with open(COOKIE_FILE, 'rb') as f: + cookies = pickle.load(f) + + for cookie in cookies: + self.session.cookies.set( + cookie['name'], + cookie['value'], + domain=cookie.get('domain', '.pokemoncenter.com'), + path=cookie.get('path', '/') + ) + + logger.info(f"Loaded {len(cookies)} cookies") + self.cookies_valid = True + return True + + except Exception as e: + logger.error(f"Failed to load cookies: {e}") + return False + + def load_known_skus(self): + """Load previously seen SKUs""" + if KNOWN_SKUS_FILE.exists(): + with open(KNOWN_SKUS_FILE, 'r') as f: + data = json.load(f) + self.known_skus = set(data.get('skus', [])) + logger.info(f"Loaded {len(self.known_skus)} known SKUs") + else: + logger.info("No known SKUs file - will create on first run") + self.known_skus = set() + + def save_known_skus(self): + """Save known SKUs to file""" + with open(KNOWN_SKUS_FILE, 'w') as f: + json.dump({ + 'skus': list(self.known_skus), + 'last_updated': datetime.now().isoformat(), + 'count': len(self.known_skus) + }, f, indent=2) + + def log_detection(self, sku: str, product_info: Dict): + """Log a new product detection""" + detection = { + 'sku': sku, + 'detected_at': datetime.now().isoformat(), + 'product_info': product_info + } + self.detections.append(detection) + + # Append to detections log file + detections = [] + if DETECTIONS_LOG.exists(): + with open(DETECTIONS_LOG, 'r') as f: + detections = json.load(f) + + detections.append(detection) + + with open(DETECTIONS_LOG, 'w') as f: + json.dump(detections, f, indent=2) + + def get_category_products(self, category: str) -> Optional[Dict]: + """Fetch products from a category endpoint""" + url = f"{API_BASE}/site/resourceapi/category/{category}" + + try: + response = self.session.get(url, timeout=15) + + if response.status_code == 403: + logger.warning("API blocked - cookies may have expired") + self.cookies_valid = False + return None + + if response.status_code == 200: + return response.json() + + logger.warning(f"Unexpected status {response.status_code} for {category}") + return None + + except Exception as e: + logger.error(f"Error fetching {category}: {e}") + return None + + def get_product_details(self, sku: str) -> Optional[Dict]: + """Get full details for a specific product""" + url = f"{API_BASE}/tpci-ecommweb-api/product/{sku}" + + try: + response = self.session.get(url, timeout=15) + + if response.status_code == 200: + return response.json() + + return None + + except Exception as e: + logger.error(f"Error fetching product {sku}: {e}") + return None + + def extract_skus_from_response(self, data: Dict) -> Set[str]: + """Extract product SKUs from API response""" + skus = set() + + # The response structure varies - try multiple approaches + # This will need adjustment based on actual API response + + def find_skus(obj, depth=0): + """Recursively find SKU-like values""" + if depth > 10: # Prevent infinite recursion + return + + if isinstance(obj, dict): + # Look for SKU fields + for key in ['sku', 'skuCode', 'productId', 'id', 'code']: + if key in obj: + value = obj[key] + if isinstance(value, str) and self._looks_like_sku(value): + skus.add(value) + + # Look in nested objects + for value in obj.values(): + find_skus(value, depth + 1) + + elif isinstance(obj, list): + for item in obj: + find_skus(item, depth + 1) + + find_skus(data) + return skus + + def _looks_like_sku(self, value: str) -> bool: + """Check if a string looks like a Pokemon Center SKU""" + # SKUs we've seen: 699-17113, 191-85953, 10-10191-109 + if not value: + return False + + # Must contain digits and possibly hyphens + has_digit = any(c.isdigit() for c in value) + reasonable_length = 5 <= len(value) <= 20 + + return has_digit and reasonable_length + + def check_for_new_products(self) -> List[Dict]: + """Main check - look for new SKUs across all categories""" + if not self.cookies_valid: + if not self.load_cookies(): + return [] + + new_products = [] + current_skus = set() + + for category in MONITOR_CATEGORIES: + logger.debug(f"Checking category: {category}") + + data = self.get_category_products(category) + if data: + skus = self.extract_skus_from_response(data) + current_skus.update(skus) + logger.debug(f" Found {len(skus)} SKUs in {category}") + + if not current_skus: + logger.warning("No SKUs found - API may not be returning data") + return [] + + # Find new SKUs + new_skus = current_skus - self.known_skus + + if new_skus: + logger.info(f"!!! DETECTED {len(new_skus)} NEW SKU(s) !!!") + + for sku in new_skus: + # Get full product details + details = self.get_product_details(sku) + + product_info = { + 'sku': sku, + 'details': details, + 'detected_at': datetime.now().isoformat() + } + + # Try to extract name from details + name = "Unknown Product" + if details: + # Look for name in various places + name = ( + details.get('name') or + details.get('displayName') or + details.get('definition', {}).get('display-name') or + sku + ) + + logger.info(f" NEW: {sku} - {name}") + self.log_detection(sku, product_info) + new_products.append(product_info) + + # Add to known SKUs + self.known_skus.add(sku) + + # Save updated known SKUs + self.save_known_skus() + + else: + logger.info(f"Check complete - {len(current_skus)} products, no new drops") + + return new_products + + def send_discord_alert(self, product: Dict): + """Send Discord notification for new product""" + # Import from your existing discord_notifier + try: + from src.discord_notifier import send_notification + # You'd format and send the alert here + logger.info(f"Discord alert sent for {product['sku']}") + except ImportError: + logger.warning("Discord notifier not available") + + def run(self): + """Main monitoring loop""" + print("=" * 60) + print("Pokemon Center Backend Monitor") + print("=" * 60) + print() + print("This monitors for NEW products added to the API.") + print("New SKUs = potential silent drops before announcement!") + print() + print(f"Check interval: {CHECK_INTERVAL} seconds") + print(f"Monitoring categories: {', '.join(MONITOR_CATEGORIES)}") + print() + print("Press Ctrl+C to stop") + print("=" * 60) + print() + + # Load known SKUs + self.load_known_skus() + + # Load cookies + if not self.load_cookies(): + print("\nERROR: No valid cookies!") + print("Run: python warmup_session.py") + return + + # First check - populate known SKUs if empty + if not self.known_skus: + logger.info("First run - building initial SKU database...") + self.check_for_new_products() + logger.info(f"Baseline established with {len(self.known_skus)} products") + print() + + # Main loop + check_count = 0 + while True: + try: + check_count += 1 + logger.info(f"--- Check #{check_count} ---") + + new_products = self.check_for_new_products() + + if new_products: + print() + print("!" * 60) + print("!!! NEW PRODUCT DETECTED !!!") + print("!" * 60) + for p in new_products: + print(f" SKU: {p['sku']}") + # Send Discord alert + self.send_discord_alert(p) + print("!" * 60) + print() + + # Wait for next check + time.sleep(CHECK_INTERVAL) + + except KeyboardInterrupt: + print("\nStopping monitor...") + self.save_known_skus() + break + + except Exception as e: + logger.error(f"Error in main loop: {e}") + time.sleep(CHECK_INTERVAL) + + +def main(): + monitor = BackendMonitor() + monitor.run() + + +if __name__ == "__main__": + main() diff --git a/tools/capture_api_calls.py b/tools/capture_api_calls.py new file mode 100644 index 0000000..dd1700d --- /dev/null +++ b/tools/capture_api_calls.py @@ -0,0 +1,242 @@ +""" +API Call Capture Tool for Pokemon Center + +This script opens Pokemon Center and logs ALL network requests, +helping identify backend APIs used for product data. + +Usage: + python capture_api_calls.py + +Output: + - api_captures.json: All captured API calls + - Console output with interesting endpoints +""" + +import json +import time +import re +from datetime import datetime +from pathlib import Path + +print("=" * 70) +print("Pokemon Center API Capture Tool") +print("=" * 70) +print() + +# Check for selenium +try: + from selenium import webdriver + from selenium.webdriver.chrome.options import Options + from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +except ImportError: + print("ERROR: selenium not installed") + print("Run: pip install selenium") + exit(1) + +# Storage for captured requests +captured_requests = [] +interesting_patterns = [ + r'api', + r'graphql', + r'product', + r'catalog', + r'search', + r'inventory', + r'stock', + r'algolia', + r'contentful', + r'commercetools', + r'\.json', +] + +def is_interesting(url): + """Check if URL matches patterns we care about""" + url_lower = url.lower() + for pattern in interesting_patterns: + if re.search(pattern, url_lower): + return True + return False + +def setup_browser(): + """Setup Chrome with network logging enabled""" + options = Options() + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_argument("--window-size=1920,1080") + + # Enable performance logging to capture network requests + options.set_capability('goog:loggingPrefs', {'performance': 'ALL'}) + + driver = webdriver.Chrome(options=options) + return driver + +def extract_network_requests(driver): + """Extract network requests from Chrome performance logs""" + logs = driver.get_log('performance') + requests = [] + + for entry in logs: + try: + log = json.loads(entry['message'])['message'] + + # We want Network.requestWillBeSent and Network.responseReceived + if log['method'] == 'Network.requestWillBeSent': + request = log['params']['request'] + requests.append({ + 'type': 'request', + 'url': request.get('url', ''), + 'method': request.get('method', ''), + 'headers': request.get('headers', {}), + 'postData': request.get('postData', None), + 'timestamp': entry['timestamp'] + }) + + elif log['method'] == 'Network.responseReceived': + response = log['params']['response'] + requests.append({ + 'type': 'response', + 'url': response.get('url', ''), + 'status': response.get('status', 0), + 'mimeType': response.get('mimeType', ''), + 'headers': response.get('headers', {}), + 'timestamp': entry['timestamp'] + }) + + except Exception: + pass + + return requests + +def analyze_requests(requests): + """Analyze captured requests for interesting patterns""" + api_calls = [] + seen_urls = set() + + for req in requests: + url = req.get('url', '') + + # Skip if already seen or not interesting + if url in seen_urls: + continue + if not is_interesting(url): + continue + + # Skip common non-API resources + if any(ext in url for ext in ['.png', '.jpg', '.gif', '.css', '.woff', '.svg', '.ico']): + continue + + seen_urls.add(url) + api_calls.append(req) + + return api_calls + +def main(): + driver = None + + try: + print("Starting Chrome with network logging...") + driver = setup_browser() + + print("Navigating to Pokemon Center TCG page...") + print("(This may trigger a CAPTCHA - solve it if needed)") + print() + + # Navigate to the TCG category + driver.get("https://www.pokemoncenter.com/category/tcg-cards") + + print("Waiting for page to load...") + time.sleep(10) + + # Scroll down to trigger lazy loading + print("Scrolling to load more content...") + for i in range(3): + driver.execute_script("window.scrollBy(0, 800);") + time.sleep(2) + + # Wait a bit more + time.sleep(5) + + print() + print("Extracting network requests...") + requests = extract_network_requests(driver) + print(f"Captured {len(requests)} total network events") + + # Analyze for interesting APIs + api_calls = analyze_requests(requests) + print(f"Found {len(api_calls)} potentially interesting API calls") + + # Save all captures + output_file = Path(__file__).parent / "api_captures.json" + with open(output_file, 'w') as f: + json.dump({ + 'captured_at': datetime.now().isoformat(), + 'page_url': driver.current_url, + 'total_requests': len(requests), + 'interesting_calls': api_calls, + 'all_requests': requests + }, f, indent=2) + + print(f"\nSaved full capture to: {output_file}") + + # Display interesting findings + print() + print("=" * 70) + print("INTERESTING API CALLS FOUND") + print("=" * 70) + + if not api_calls: + print("No obvious API calls detected.") + print("This could mean:") + print(" 1. Data is server-rendered (no client API)") + print(" 2. API calls use non-standard paths") + print(" 3. Bot protection blocked the content") + else: + for call in api_calls: + url = call.get('url', '') + method = call.get('method', 'GET') + status = call.get('status', '-') + mime = call.get('mimeType', '') + + # Truncate long URLs + display_url = url[:100] + '...' if len(url) > 100 else url + + print(f"\n[{method}] {display_url}") + if status != '-': + print(f" Status: {status}, Type: {mime}") + + # Check for key patterns + if 'graphql' in url.lower(): + print(" ⚡ GraphQL endpoint!") + if 'algolia' in url.lower(): + print(" 🔍 Algolia search!") + if 'product' in url.lower(): + print(" 📦 Product data!") + + print() + print("=" * 70) + print("NEXT STEPS") + print("=" * 70) + print(""" +1. Review api_captures.json for the full data +2. Look for JSON responses containing product info +3. Test promising endpoints with curl/requests +4. Note any required headers (auth tokens, API keys) + +To test an endpoint manually: + curl -H "User-Agent: Mozilla/5.0..." "https://api.example.com/endpoint" +""") + + # Keep browser open for manual inspection + print() + input("Browser is still open. Press ENTER to close and exit...") + + except Exception as e: + print(f"Error: {e}") + import traceback + traceback.print_exc() + finally: + if driver: + driver.quit() + print("Browser closed.") + +if __name__ == "__main__": + main() diff --git a/tools/smart_monitor.py b/tools/smart_monitor.py new file mode 100644 index 0000000..ea91c3a --- /dev/null +++ b/tools/smart_monitor.py @@ -0,0 +1,415 @@ +""" +Smart Pokemon Center Monitor + +Uses stealth browser with adaptive modes and human-like behavior. +Detects new products before they're announced. + +Modes: +- STEALTH: Normal monitoring (~1 min intervals, human-like) +- ALERT: Fast checking when new product detected (~20 sec) +- COOLDOWN: Gradual return to stealth after alert + +Usage: + python smart_monitor.py +""" + +import json +import time +import random +import logging +from pathlib import Path +from datetime import datetime, timedelta +from typing import Optional, Set, List, Dict +from enum import Enum + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + datefmt='%H:%M:%S' +) +logger = logging.getLogger(__name__) + +# File paths +DATA_DIR = Path(__file__).parent.parent / "data" +KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json" +DETECTIONS_FILE = DATA_DIR / "detections.json" + +# URLs to monitor +MONITOR_URLS = [ + "https://www.pokemoncenter.com/category/tcg-cards?sort=newest", + "https://www.pokemoncenter.com/category/new-releases", +] + + +class MonitorMode(Enum): + STEALTH = "stealth" + ALERT = "alert" + COOLDOWN = "cooldown" + + +class SmartMonitor: + """Adaptive monitor with human-like behavior""" + + def __init__(self): + self.browser = None + self.known_skus: Set[str] = set() + self.mode = MonitorMode.STEALTH + self.alert_triggered_at: Optional[datetime] = None + self.check_count = 0 + self.last_detection: Optional[Dict] = None + + # Mode timing settings + self.timing = { + MonitorMode.STEALTH: (45, 90), # 45-90 seconds + MonitorMode.ALERT: (15, 30), # 15-30 seconds + MonitorMode.COOLDOWN: (60, 120), # 60-120 seconds + } + + # Alert mode duration + self.alert_duration = timedelta(minutes=30) + self.cooldown_duration = timedelta(minutes=15) + + def start_browser(self): + """Start the stealth browser""" + if self.browser: + return + + logger.info("Starting stealth browser...") + + try: + from stealth_browser import StealthBrowser + self.browser = StealthBrowser( + headless=False, + session_name="smart_monitor" + ) + self.browser.start() + logger.info("Browser started successfully") + except Exception as e: + logger.error(f"Failed to start browser: {e}") + raise + + def stop_browser(self): + """Stop the browser""" + if self.browser: + logger.info("Stopping browser...") + self.browser.stop() + self.browser = None + + def load_known_skus(self): + """Load previously seen SKUs""" + if KNOWN_SKUS_FILE.exists(): + with open(KNOWN_SKUS_FILE, 'r') as f: + data = json.load(f) + self.known_skus = set(data.get('skus', [])) + logger.info(f"Loaded {len(self.known_skus)} known SKUs") + else: + self.known_skus = set() + logger.info("No known SKUs file - starting fresh") + + def save_known_skus(self): + """Save known SKUs""" + with open(KNOWN_SKUS_FILE, 'w') as f: + json.dump({ + 'skus': list(self.known_skus), + 'updated': datetime.now().isoformat(), + 'count': len(self.known_skus) + }, f, indent=2) + + def log_detection(self, sku: str, name: str, url: str): + """Log a new product detection""" + detection = { + 'sku': sku, + 'name': name, + 'url': url, + 'detected_at': datetime.now().isoformat(), + 'mode': self.mode.value + } + + self.last_detection = detection + + # Load existing detections + detections = [] + if DETECTIONS_FILE.exists(): + with open(DETECTIONS_FILE, 'r') as f: + detections = json.load(f) + + detections.append(detection) + + with open(DETECTIONS_FILE, 'w') as f: + json.dump(detections, f, indent=2) + + logger.info(f"Detection logged: {sku}") + + def get_interval(self) -> float: + """Get randomized check interval based on current mode""" + min_sec, max_sec = self.timing[self.mode] + + # Add gaussian jitter for more natural timing + base = (min_sec + max_sec) / 2 + jitter = random.gauss(0, (max_sec - min_sec) / 4) + interval = base + jitter + + # Clamp to bounds + return max(min_sec * 0.8, min(max_sec * 1.2, interval)) + + def maybe_do_human_action(self): + """Occasionally do something human-like""" + if not self.browser or not self.browser.driver: + return + + action = random.random() + + if action < 0.3: + # Scroll randomly + self.browser.human_scroll() + elif action < 0.4: + # Small mouse movement + self.browser.human_mouse_move() + elif action < 0.45: + # Longer pause (human distraction) + pause = random.uniform(3, 8) + logger.debug(f"Human pause: {pause:.1f}s") + time.sleep(pause) + + def maybe_take_break(self) -> bool: + """Occasionally take a longer break""" + # 3% chance of a break + if random.random() < 0.03: + break_time = random.randint(120, 300) # 2-5 minutes + logger.info(f"Taking a break for {break_time}s (human-like pause)") + time.sleep(break_time) + return True + return False + + def extract_skus_from_page(self) -> Set[str]: + """Extract product SKUs from the current page""" + if not self.browser or not self.browser.driver: + return set() + + skus = set() + + try: + # Get page source and parse + from bs4 import BeautifulSoup + html = self.browser.driver.page_source + soup = BeautifulSoup(html, 'html.parser') + + # Look for product links - Pokemon Center format: /product/SKU/name + import re + product_links = soup.select('a[href*="/product/"]') + + for link in product_links: + href = link.get('href', '') + # Extract SKU from URL like /product/699-17113/product-name + match = re.search(r'/product/([0-9]+-?[0-9]+)', href) + if match: + skus.add(match.group(1)) + + # Also look for data attributes + for elem in soup.select('[data-sku], [data-product-id]'): + sku = elem.get('data-sku') or elem.get('data-product-id') + if sku and re.match(r'^[0-9]+-?[0-9]+', sku): + skus.add(sku) + + except Exception as e: + logger.error(f"Error extracting SKUs: {e}") + + return skus + + def check_page(self, url: str) -> Set[str]: + """Load a page and extract SKUs""" + if not self.browser: + self.start_browser() + + try: + # Human-like delay before navigation + self.browser.human_delay(0.5, 2.0) + + # Navigate + logger.debug(f"Loading: {url}") + self.browser.driver.get(url) + + # Wait for page load + time.sleep(random.uniform(3, 6)) + + # Check for CAPTCHA + if self.browser.check_for_captcha(): + logger.warning("CAPTCHA detected!") + print("\n" + "!" * 50) + print("CAPTCHA DETECTED - Please solve it in the browser") + print("!" * 50 + "\n") + self.browser.wait_for_captcha_solve(timeout=120) + + # Human actions + self.maybe_do_human_action() + + # Extract SKUs + skus = self.extract_skus_from_page() + + return skus + + except Exception as e: + logger.error(f"Error checking page: {e}") + return set() + + def update_mode(self): + """Update monitoring mode based on state""" + now = datetime.now() + + if self.mode == MonitorMode.ALERT: + # Check if alert period is over + if self.alert_triggered_at: + elapsed = now - self.alert_triggered_at + if elapsed > self.alert_duration: + logger.info("Alert period over, entering cooldown") + self.mode = MonitorMode.COOLDOWN + + elif self.mode == MonitorMode.COOLDOWN: + # Check if cooldown is over + if self.alert_triggered_at: + elapsed = now - self.alert_triggered_at + if elapsed > (self.alert_duration + self.cooldown_duration): + logger.info("Cooldown over, returning to stealth mode") + self.mode = MonitorMode.STEALTH + self.alert_triggered_at = None + + def trigger_alert_mode(self): + """Switch to alert mode""" + logger.info("!!! ENTERING ALERT MODE - Faster checks !!!") + self.mode = MonitorMode.ALERT + self.alert_triggered_at = datetime.now() + + def send_discord_alert(self, sku: str, name: str, url: str): + """Send Discord notification""" + try: + from src.discord_notifier import DiscordNotifier + from config import DISCORD_WEBHOOK_URL + + if DISCORD_WEBHOOK_URL and DISCORD_WEBHOOK_URL != "YOUR_WEBHOOK_URL_HERE": + notifier = DiscordNotifier(DISCORD_WEBHOOK_URL) + # Create a simple product dict + product = { + 'name': name, + 'url': f"https://www.pokemoncenter.com/product/{sku}", + 'price': 'Check site', + 'site': 'pokemoncenter', + } + notifier.send_stock_alert(product, "NEW BACKEND DETECTION") + logger.info("Discord alert sent!") + except Exception as e: + logger.warning(f"Could not send Discord alert: {e}") + + def run_check(self) -> List[str]: + """Run a single check across all monitored URLs""" + self.check_count += 1 + all_skus = set() + new_skus = [] + + logger.info(f"Check #{self.check_count} | Mode: {self.mode.value.upper()}") + + for url in MONITOR_URLS: + skus = self.check_page(url) + all_skus.update(skus) + + # Brief pause between pages + if url != MONITOR_URLS[-1]: + time.sleep(random.uniform(2, 5)) + + logger.info(f"Found {len(all_skus)} total SKUs") + + # Find new SKUs + if self.known_skus: # Only check if we have a baseline + new = all_skus - self.known_skus + if new: + for sku in new: + logger.info(f"!!! NEW SKU DETECTED: {sku}") + new_skus.append(sku) + + # Log and alert + url = f"https://www.pokemoncenter.com/product/{sku}" + self.log_detection(sku, f"New Product {sku}", url) + self.send_discord_alert(sku, f"New Product {sku}", url) + + # Trigger alert mode + self.trigger_alert_mode() + + # Update known SKUs + self.known_skus.update(all_skus) + self.save_known_skus() + + return new_skus + + def run(self): + """Main monitoring loop""" + print() + print("=" * 60) + print(" SMART POKEMON CENTER MONITOR") + print("=" * 60) + print() + print("Modes:") + print(f" STEALTH: {self.timing[MonitorMode.STEALTH]} sec (normal)") + print(f" ALERT: {self.timing[MonitorMode.ALERT]} sec (after detection)") + print(f" COOLDOWN: {self.timing[MonitorMode.COOLDOWN]} sec (transition)") + print() + print("Press Ctrl+C to stop") + print("=" * 60) + print() + + try: + # Initialize + self.load_known_skus() + self.start_browser() + + # First check - build baseline if needed + if not self.known_skus: + logger.info("First run - building SKU baseline...") + self.run_check() + logger.info(f"Baseline: {len(self.known_skus)} products") + print() + + # Main loop + while True: + # Maybe take a break + if self.maybe_take_break(): + continue + + # Run check + new_skus = self.run_check() + + if new_skus: + print() + print("!" * 60) + print("!!! NEW PRODUCT(S) DETECTED !!!") + for sku in new_skus: + print(f" -> {sku}") + print("!" * 60) + print() + + # Update mode + self.update_mode() + + # Get interval and wait + interval = self.get_interval() + logger.info(f"Next check in {interval:.0f}s") + time.sleep(interval) + + except KeyboardInterrupt: + print("\nStopping monitor...") + except Exception as e: + logger.error(f"Monitor error: {e}") + import traceback + traceback.print_exc() + finally: + self.save_known_skus() + self.stop_browser() + print("Monitor stopped.") + + +def main(): + monitor = SmartMonitor() + monitor.run() + + +if __name__ == "__main__": + main() diff --git a/tools/stealth_browser.py b/tools/stealth_browser.py new file mode 100644 index 0000000..728b683 --- /dev/null +++ b/tools/stealth_browser.py @@ -0,0 +1,539 @@ +""" +Stealth Browser Module +Uses undetected-chromedriver to bypass bot detection (Imperva, Cloudflare, etc.) +Includes session persistence, proxy rotation, and human-like behavior. +""" + +import os +import json +import time +import random +import logging +import pickle +from pathlib import Path +from typing import Optional, List, Dict, Any +from datetime import datetime, timedelta + +logger = logging.getLogger(__name__) + +# Data directory +DATA_DIR = Path(__file__).parent.parent / "data" + +# Session storage directory +SESSION_DIR = DATA_DIR / "sessions" +SESSION_DIR.mkdir(parents=True, exist_ok=True) + +# Proxy configuration file +PROXY_FILE = DATA_DIR / "proxies.json" + + +class ProxyRotator: + """Manages proxy rotation for requests""" + + def __init__(self, proxy_file: Path = PROXY_FILE): + self.proxies: List[Dict] = [] + self.current_index = 0 + self.failed_proxies: Dict[str, datetime] = {} + self.cooldown_minutes = 30 + + self._load_proxies(proxy_file) + + def _load_proxies(self, proxy_file: Path): + """Load proxies from configuration file""" + if proxy_file.exists(): + try: + with open(proxy_file, 'r') as f: + data = json.load(f) + self.proxies = data.get("proxies", []) + logger.info(f"Loaded {len(self.proxies)} proxies") + except Exception as e: + logger.warning(f"Failed to load proxies: {e}") + + def get_proxy(self) -> Optional[Dict]: + """Get next available proxy""" + if not self.proxies: + return None + + # Clean up expired cooldowns + now = datetime.now() + self.failed_proxies = { + k: v for k, v in self.failed_proxies.items() + if now - v < timedelta(minutes=self.cooldown_minutes) + } + + # Find next working proxy + attempts = 0 + while attempts < len(self.proxies): + proxy = self.proxies[self.current_index] + proxy_key = f"{proxy.get('host')}:{proxy.get('port')}" + + self.current_index = (self.current_index + 1) % len(self.proxies) + + if proxy_key not in self.failed_proxies: + return proxy + + attempts += 1 + + # All proxies in cooldown, return first one anyway + return self.proxies[0] if self.proxies else None + + def mark_failed(self, proxy: Dict): + """Mark a proxy as failed (temporary cooldown)""" + if proxy: + proxy_key = f"{proxy.get('host')}:{proxy.get('port')}" + self.failed_proxies[proxy_key] = datetime.now() + logger.warning(f"Proxy {proxy_key} marked as failed") + + +class StealthBrowser: + """ + Stealth browser using undetected-chromedriver. + Designed to bypass Imperva/Incapsula and similar bot protection. + """ + + def __init__( + self, + headless: bool = False, + proxy: Optional[Dict] = None, + user_data_dir: Optional[str] = None, + session_name: str = "default" + ): + self.headless = headless + self.proxy = proxy + self.user_data_dir = user_data_dir + self.session_name = session_name + self.driver = None + self._setup_complete = False + + def _get_chrome_options(self): + """Configure Chrome options for stealth""" + import undetected_chromedriver as uc + + options = uc.ChromeOptions() + + # Basic stealth settings - compatible with newer Chrome versions + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--no-sandbox") + options.add_argument("--disable-infobars") + + # Window size (realistic resolution) + options.add_argument("--window-size=1920,1080") + + # User data directory for session persistence + if self.user_data_dir: + options.add_argument(f"--user-data-dir={self.user_data_dir}") + + # Proxy configuration + if self.proxy: + proxy_str = self._format_proxy(self.proxy) + if proxy_str: + options.add_argument(f"--proxy-server={proxy_str}") + + # Headless mode (note: more detectable) + if self.headless: + options.add_argument("--headless=new") + + return options + + def _format_proxy(self, proxy: Dict) -> Optional[str]: + """Format proxy dict into Chrome proxy string""" + if not proxy: + return None + + host = proxy.get("host") + port = proxy.get("port") + + if not host or not port: + return None + + protocol = proxy.get("protocol", "http") + return f"{protocol}://{host}:{port}" + + def start(self): + """Start the browser""" + if self.driver: + return + + try: + import undetected_chromedriver as uc + + options = self._get_chrome_options() + + # Create driver with version_main to match installed Chrome version + # This prevents "ChromeDriver only supports Chrome version X" errors + self.driver = uc.Chrome( + options=options, + use_subprocess=True, + version_main=146, # Match user's Chrome version + ) + + # Set realistic viewport + self.driver.set_window_size(1920, 1080) + + # Load saved cookies if they exist + self._load_cookies() + + self._setup_complete = True + logger.info("Stealth browser started successfully") + + except Exception as e: + logger.error(f"Failed to start stealth browser: {e}") + raise + + def stop(self): + """Stop the browser and save session""" + if self.driver: + try: + self._save_cookies() + self.driver.quit() + except Exception as e: + logger.warning(f"Error stopping browser: {e}") + finally: + self.driver = None + self._setup_complete = False + + def _get_cookie_file(self) -> Path: + """Get path to cookie file for this session""" + return SESSION_DIR / f"{self.session_name}_cookies.pkl" + + def _save_cookies(self): + """Save cookies to file for session persistence""" + if not self.driver: + return + + try: + cookies = self.driver.get_cookies() + cookie_file = self._get_cookie_file() + + with open(cookie_file, 'wb') as f: + pickle.dump(cookies, f) + + logger.debug(f"Saved {len(cookies)} cookies to {cookie_file}") + except Exception as e: + logger.warning(f"Failed to save cookies: {e}") + + def _load_cookies(self): + """Load cookies from file""" + cookie_file = self._get_cookie_file() + + if not cookie_file.exists(): + return + + try: + with open(cookie_file, 'rb') as f: + cookies = pickle.load(f) + + # Need to visit domain first before adding cookies + # This will be done when navigating to the actual page + + self._pending_cookies = cookies + logger.debug(f"Loaded {len(cookies)} cookies from {cookie_file}") + except Exception as e: + logger.warning(f"Failed to load cookies: {e}") + self._pending_cookies = [] + + def _apply_pending_cookies(self, domain: str): + """Apply loaded cookies after visiting domain""" + if not hasattr(self, '_pending_cookies') or not self._pending_cookies: + return + + for cookie in self._pending_cookies: + try: + # Only add cookies for matching domain + if domain in cookie.get('domain', ''): + self.driver.add_cookie(cookie) + except Exception: + pass # Some cookies may fail, that's ok + + self._pending_cookies = [] + + def human_delay(self, min_seconds: float = 1.0, max_seconds: float = 3.0): + """Add human-like random delay""" + delay = random.uniform(min_seconds, max_seconds) + time.sleep(delay) + + def human_scroll(self): + """Scroll like a human would""" + if not self.driver: + return + + # Random scroll amount + scroll_amount = random.randint(200, 600) + + # Smooth scroll + self.driver.execute_script(f""" + window.scrollBy({{ + top: {scroll_amount}, + behavior: 'smooth' + }}); + """) + + self.human_delay(0.5, 1.5) + + def human_mouse_move(self): + """Simulate mouse movement (basic)""" + if not self.driver: + return + + # Move mouse to random position + try: + from selenium.webdriver.common.action_chains import ActionChains + + actions = ActionChains(self.driver) + + # Random coordinates within viewport + x = random.randint(100, 800) + y = random.randint(100, 600) + + # Move by offset from current position + actions.move_by_offset(x, y).perform() + + # Reset position + actions.move_by_offset(-x, -y).perform() + except Exception: + pass # Mouse movement is optional + + def get_page(self, url: str, wait_time: float = 5.0) -> str: + """ + Navigate to URL with human-like behavior. + + Args: + url: URL to navigate to + wait_time: Time to wait for page load + + Returns: + Page HTML content + """ + if not self.driver: + self.start() + + try: + # Pre-navigation delay + self.human_delay(0.5, 1.5) + + # Navigate + self.driver.get(url) + + # Apply any pending cookies + from urllib.parse import urlparse + domain = urlparse(url).netloc + self._apply_pending_cookies(domain) + + # Wait for page load + time.sleep(wait_time) + + # Human-like behavior + self.human_scroll() + self.human_delay(1, 2) + self.human_mouse_move() + + # Get page content + html = self.driver.page_source + + # Save cookies after successful page load + self._save_cookies() + + return html + + except Exception as e: + logger.error(f"Error getting page {url}: {e}") + raise + + def check_for_captcha(self) -> bool: + """Check if page has a CAPTCHA challenge blocking content""" + if not self.driver: + return False + + page_source = self.driver.page_source.lower() + + # First check: Did the page load actual content? + # If we see product elements, it's NOT a CAPTCHA page + content_loaded_indicators = [ + 'class="product-card', + 'data-product-id', + 'data-sku', + '/product/', + 'add to cart', + 'product-grid', + 'product-list', + ] + + for indicator in content_loaded_indicators: + if indicator in page_source: + # Page has actual content - no CAPTCHA + return False + + # Only flag as CAPTCHA if we see blocking indicators AND no content + captcha_indicators = [ + "verify you are human", + "press & hold", + "press and hold", + "checking your browser", + "just a moment", + "enable javascript and cookies", + "access denied", + "blocked", + "challenge-running", + "cf-browser-verification", + "ddos-guard", + ] + + for indicator in captcha_indicators: + if indicator in page_source: + return True + + # Also check if page is suspiciously empty (might be blocked) + if len(page_source) < 5000 and "pokemoncenter" not in page_source: + return True + + return False + + def wait_for_captcha_solve(self, timeout: int = 120): + """ + Wait for user to solve CAPTCHA manually. + Only works in non-headless mode. + """ + if self.headless: + logger.warning("Cannot solve CAPTCHA in headless mode") + return False + + logger.info("CAPTCHA detected! Please solve it manually...") + print("\n" + "=" * 50) + print("CAPTCHA DETECTED!") + print("Please solve the CAPTCHA in the browser window.") + print("=" * 50 + "\n") + + start_time = time.time() + + while time.time() - start_time < timeout: + if not self.check_for_captcha(): + logger.info("CAPTCHA solved!") + self._save_cookies() # Save session after solving + return True + + time.sleep(2) + + logger.warning("CAPTCHA solve timeout") + return False + + def screenshot(self, filename: str = "screenshot.png"): + """Take a screenshot for debugging""" + if self.driver: + try: + self.driver.save_screenshot(filename) + logger.info(f"Screenshot saved to {filename}") + except Exception as e: + logger.warning(f"Failed to save screenshot: {e}") + + +class StealthBrowserPool: + """ + Manages multiple stealth browser instances with proxy rotation. + """ + + def __init__( + self, + pool_size: int = 1, + use_proxies: bool = False, + headless: bool = False + ): + self.pool_size = pool_size + self.use_proxies = use_proxies + self.headless = headless + self.browsers: List[StealthBrowser] = [] + self.proxy_rotator = ProxyRotator() if use_proxies else None + self.current_index = 0 + + def get_browser(self) -> StealthBrowser: + """Get a browser from the pool""" + # Create browser if pool is empty + if not self.browsers: + proxy = self.proxy_rotator.get_proxy() if self.proxy_rotator else None + browser = StealthBrowser( + headless=self.headless, + proxy=proxy, + session_name=f"pool_{self.current_index}" + ) + browser.start() + self.browsers.append(browser) + return browser + + # Rotate through browsers + browser = self.browsers[self.current_index] + self.current_index = (self.current_index + 1) % len(self.browsers) + return browser + + def shutdown_all(self): + """Shutdown all browsers in pool""" + for browser in self.browsers: + try: + browser.stop() + except Exception: + pass + self.browsers = [] + + +# Global instances +_stealth_browser: Optional[StealthBrowser] = None +_browser_pool: Optional[StealthBrowserPool] = None + + +def get_stealth_browser( + headless: bool = False, + session_name: str = "pokemoncenter" +) -> StealthBrowser: + """Get or create the global stealth browser instance""" + global _stealth_browser + + if _stealth_browser is None: + _stealth_browser = StealthBrowser( + headless=headless, + session_name=session_name + ) + + if not _stealth_browser._setup_complete: + _stealth_browser.start() + + return _stealth_browser + + +def shutdown_stealth_browser(): + """Shutdown the global stealth browser""" + global _stealth_browser + + if _stealth_browser: + _stealth_browser.stop() + _stealth_browser = None + + +def create_proxy_config_template(): + """Create a template proxies.json file""" + template = { + "proxies": [ + { + "host": "proxy1.example.com", + "port": 8080, + "protocol": "http", + "username": "user", + "password": "pass" + }, + { + "host": "proxy2.example.com", + "port": 8080, + "protocol": "http", + "username": "user", + "password": "pass" + } + ], + "_comment": "Add your residential proxies here. Recommended providers: Bright Data, Oxylabs, Smartproxy" + } + + if not PROXY_FILE.exists(): + with open(PROXY_FILE, 'w') as f: + json.dump(template, f, indent=2) + logger.info(f"Created proxy template at {PROXY_FILE}") + + +# Create template on import +create_proxy_config_template() diff --git a/tools/warmup.py b/tools/warmup.py new file mode 100644 index 0000000..fba06fe --- /dev/null +++ b/tools/warmup.py @@ -0,0 +1,74 @@ +""" +Warmup tool for sites with CAPTCHA/bot protection. +Run this before the main monitor to solve CAPTCHAs manually. + +Usage: + python tools/warmup.py gamestop + python tools/warmup.py all +""" + +import sys +import os + +# Add parent directory to path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +from scrapers.gamestop import warmup_gamestop + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +def warmup_all(): + """Warm up all sites that need it""" + results = {} + + logger.info("=" * 50) + logger.info("Starting warmup for GameStop...") + logger.info("=" * 50) + results["gamestop"] = warmup_gamestop() + + # Add more sites here as needed + # results["pokemoncenter"] = warmup_pokemoncenter() + + logger.info("=" * 50) + logger.info("Warmup Results:") + for site, success in results.items(): + status = "OK" if success else "FAILED" + logger.info(f" {site}: {status}") + logger.info("=" * 50) + + return all(results.values()) + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + print("\nAvailable sites: gamestop, all") + sys.exit(1) + + site = sys.argv[1].lower() + + if site == "gamestop": + success = warmup_gamestop() + elif site == "all": + success = warmup_all() + else: + print(f"Unknown site: {site}") + print("Available sites: gamestop, all") + sys.exit(1) + + if success: + logger.info("Warmup completed successfully!") + sys.exit(0) + else: + logger.error("Warmup failed or timed out") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/warmup_session.py b/tools/warmup_session.py new file mode 100644 index 0000000..271269e --- /dev/null +++ b/tools/warmup_session.py @@ -0,0 +1,119 @@ +""" +Session Warmup Tool for Pokemon Center + +Run this BEFORE starting the monitor to: +1. Open a stealth browser +2. Let you manually browse and solve any CAPTCHAs +3. Save cookies/session for the monitor to reuse + +Usage: + python warmup_session.py + +After running: +1. Browse Pokemon Center normally for a few minutes +2. Add items to cart, look at products, etc. +3. Solve any CAPTCHAs that appear +4. Press Enter in this terminal when done +5. The session will be saved and reused by the monitor +""" + +import sys +import time + +print("=" * 60) +print("Pokemon Center Session Warmup") +print("=" * 60) +print() + +# Import stealth browser +try: + from .stealth_browser import StealthBrowser +except ImportError as e: + print(f"Error importing stealth browser: {e}") + sys.exit(1) + +print("Starting stealth browser...") +print("This will open a Chrome window.") +print() + +browser = StealthBrowser( + headless=False, + session_name="pokemoncenter" # Same name used by monitor +) + +try: + browser.start() + print("[OK] Browser started!") + print() + + # Navigate to Pokemon Center + print("Navigating to Pokemon Center...") + browser.driver.get("https://www.pokemoncenter.com") + + time.sleep(3) + + print() + print("=" * 60) + print("WARMUP INSTRUCTIONS") + print("=" * 60) + print(""" +1. If you see a CAPTCHA or "Pardon Our Interruption": + - Solve it in the browser window + - Wait for the page to load + +2. Browse naturally for 2-3 minutes: + - Click on some products + - Look at different categories + - Add something to cart (you don't have to buy) + - This builds a legitimate browsing profile + +3. Navigate to the TCG section: + - https://www.pokemoncenter.com/category/tcg-cards + +4. Once you're browsing normally without issues: + - Come back here + - Press ENTER to save the session +""") + print("=" * 60) + print() + + input("Press ENTER when you're done browsing to save the session...") + + print() + print("Saving session cookies...") + browser._save_cookies() + + # Check how many cookies we got + cookies = browser.driver.get_cookies() + print(f"[OK] Saved {len(cookies)} cookies") + + # Get current URL for reference + current_url = browser.driver.current_url + print(f"[OK] Final URL: {current_url}") + + print() + print("=" * 60) + print("SESSION SAVED!") + print("=" * 60) + print(""" +Your session has been saved. The monitor will now use these +cookies when checking Pokemon Center. + +Tips for best results: +- Run the monitor with 3-5 minute check intervals +- Keep USE_STEALTH_BROWSER = True in config.py +- If you get blocked again, run this warmup again + +Session file: data/sessions/pokemoncenter_cookies.pkl +""") + +except KeyboardInterrupt: + print("\nInterrupted by user") +except Exception as e: + print(f"Error: {e}") + import traceback + traceback.print_exc() +finally: + print("Closing browser...") + browser.stop() + print("Done!")