Cleanups for scrapers

.claude/settings.local.json

@@ -15,7 +15,57 @@
       "Bash(PYTHON=\"/c/Users/hocke/Documents/GitHub/pokemon-stock-checker/.venv/Scripts/python.exe\")",
       "Bash(\"$PYTHON\" -c \":*)",
       "Bash(1 grep:*)",
-      "Bash(taskkill /F /IM python.exe)"
+      "Bash(taskkill /F /IM python.exe)",
+      "Bash(python -m http.server 8080)",
+      "Bash(py -m http.server 8080 --directory \"C:\\\\Users\\\\hocke\\\\Documents\\\\GitHub\\\\pokemon-stock-checker\")",
+      "Bash(where python:*)",
+      "Bash(where python3:*)",
+      "Bash(where py:*)",
+      "Read(//c/Users/hocke/AppData/Local/Programs/**)",
+      "Read(//c/Users/hocke/AppData/Local/Microsoft/WindowsApps/**)",
+      "Read(//c/Users/hocke/AppData/Local//**)",
+      "Read(//c/ProgramData//**)",
+      "Read(//c/Program Files//**)",
+      "Read(//c/Program Files \\(x86\\)//**)",
+      "Read(//proc/**)",
+      "Bash(echo \"PATH: $PATH\")",
+      "Bash(netstat -ano)",
+      "Read(//c/Users/hocke/Documents/GitHub/pokemon-stock-checker/**)",
+      "Bash(taskkill /PID 21060 /F)",
+      "Bash(cmd /c \"taskkill /PID 21060 /F\")",
+      "Bash(cmd /c taskkill /PID 21060 /F)",
+      "Bash(where python3.13)",
+      "Bash(where python3.12)",
+      "Bash(ls /c/Python*)",
+      "Bash(cmd //c \"where py\")",
+      "Bash(cmd //c \"py --version\")",
+      "Bash(cmd //c \"python --version\")",
+      "Bash(\"/c/Users/hocke/.local/bin/python3.14.exe\" -c \":*)",
+      "Bash(find /c/Users/hocke -maxdepth 6 -type d -name *undetected*)",
+      "Bash(find /c/Users/hocke -maxdepth 6 -type d -name *chrome*profile*)",
+      "Bash(grep -v \"^$\")",
+      "Bash(cmd /c \".venv\\\\Scripts\\\\python.exe -c \"\"import sqlite3; conn = sqlite3.connect\\(''stats.db''\\); conn.row_factory = sqlite3.Row; cur = conn.cursor\\(\\); cur.execute\\(\\\\\"\"SELECT name, site, image_url FROM products WHERE site IN \\(''target'', ''gamestop''\\) LIMIT 10\\\\\"\"\\); [print\\(f''[{r[\\\\\"\"site\\\\\"\"]}] {r[\\\\\"\"name\\\\\"\"][:40]} -> {r[\\\\\"\"image_url\\\\\"\"]}''\\) for r in cur.fetchall\\(\\)]\"\"\")",
+      "Bash(python3 -c \":*)",
+      "Bash(python -c \":*)",
+      "Bash(cmd /c \".venv\\\\Scripts\\\\python.exe -c \"\"import json; f=open\\(''data/products.json''\\); p=json.load\\(f\\); gs=[\\(v[''name''][:50],v.get\\(''image_url''\\)\\) for v in p.values\\(\\) if v.get\\(''site''\\)==''gamestop'']; [print\\(n,''->'',i\\) for n,i in gs[:5]]\"\"\")",
+      "WebSearch",
+      "WebFetch(domain:scrapfly.io)",
+      "WebFetch(domain:scrape.do)",
+      "WebFetch(domain:docs.unwrangle.com)",
+      "WebFetch(domain:oxylabs.io)",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:www.bestbuy.com)",
+      "WebFetch(domain:scrapeops.io)",
+      "Bash(find /c/Users/hocke -maxdepth 8 -name python*.exe)",
+      "Bash(\"/c/Users/hocke/.local/bin/python3.14.exe\":*)",
+      "Bash(.venv/Scripts/python.exe -c \":*)",
+      "Bash(unzip -q \"3.73 Obf FREE.zip\" -d /tmp/extension_temp)",
+      "Read(//tmp/**)",
+      "Bash(find /c/Users/hocke/Downloads -name *3.73* -o -name *Obf*)",
+      "Bash(unzip -q \"/c/Users/hocke/Downloads/3.73 Obf FREE.zip\")",
+      "Bash(wc -l \"/tmp/3.73 Obf FREE/background.js\" \"/tmp/3.73 Obf FREE/common\"/*.js \"/tmp/3.73 Obf FREE/monitors\"/*.js \"/tmp/3.73 Obf FREE/sites/target\"/*.js \"/tmp/3.73 Obf FREE/ui/popup\"/*.js \"/tmp/3.73 Obf FREE/ui/options\"/*.js)",
+      "Bash(ls -lh \"/tmp/3.73 Obf FREE\"/*.js \"/tmp/3.73 Obf FREE/common\"/*.js)",
+      "Bash(grep -o \"pac_script\\\\|pacScript\\\\|''''PROXY \\\\|proxy:\" \"/tmp/3.73 Obf FREE/\"*.js \"/tmp/3.73 Obf FREE/common/\"*.js)"
     ]
   }
 }

chrome-extension/background.js

@@ -18,9 +18,11 @@ const DEFAULT_CONFIG = {
 let knownProducts = {};
 let config = DEFAULT_CONFIG;
 let persistentTabId = null; // Keep tab open for faster refreshes
-let checkLoopRunning = false; // Prevent multiple loops
 let isChecking = false; // Prevent overlapping checks
 let lastNotificationTime = 0; // Rate limiting for Discord
+let botBackoffUntil = 0; // Timestamp (ms) until which checks are paused after bot detection
+let consecutiveBotDetections = 0; // Track consecutive failures for exponential backoff
+let captchaTabId = null; // Tab currently showing a captcha — kept open for user to solve
 
 // API monitoring - track SKUs seen from backend API calls
 let knownSkus = new Set();
@@ -34,6 +36,7 @@ let apiStats = {
 // Initialize
 chrome.runtime.onInstalled.addListener(() => {
   console.log("Pokemon Stock Monitor installed");
+  chrome.storage.local.remove(["persistentTabId", "captchaTabId"]);
   loadConfig().then(() => {
     loadProducts();
     loadKnownSkus();
@@ -42,51 +45,60 @@ chrome.runtime.onInstalled.addListener(() => {
 });
 
 // Load on startup
-chrome.runtime.onStartup.addListener(() => {
-  loadConfig().then(() => {
+chrome.runtime.onStartup.addListener(async () => {
+  await loadConfig();
+  await loadPersistentTabId(); // must complete before any check runs
   loadProducts();
   loadKnownSkus();
   startCheckLoop();
-  });
 });
 
-// Start the check loop (uses setTimeout to bypass Chrome's 30-sec alarm minimum)
+const ALARM_NAME = "stockCheck";
+
+// Start the check loop using chrome.alarms so it survives service worker restarts
+// Adds ±15s jitter to the first delay to avoid predictable bot-like cadence
 function startCheckLoop() {
-  if (checkLoopRunning) {
-    console.log("Check loop already running");
-    return;
+  const intervalMinutes = Math.max(1, config.checkIntervalSeconds / 60);
+  const jitterMinutes = (Math.random() - 0.5) * 0.5; // ±15 seconds
+  const delayMinutes = Math.max(0.1, intervalMinutes + jitterMinutes);
+  chrome.alarms.get(ALARM_NAME, (existing) => {
+    if (!existing) {
+      chrome.alarms.create(ALARM_NAME, { delayInMinutes: delayMinutes, periodInMinutes: intervalMinutes });
+      console.log(`Check alarm created: every ${intervalMinutes} min (first in ~${delayMinutes.toFixed(2)} min)`);
     }
-  checkLoopRunning = true;
-  console.log(`Starting check loop: every ${config.checkIntervalSeconds} seconds`);
-  scheduleNextCheck();
+  });
 }
 
-function scheduleNextCheck() {
-  if (!config.enabled || !checkLoopRunning) {
-    checkLoopRunning = false;
-    console.log("Check loop stopped");
+function stopCheckLoop() {
+  chrome.alarms.clear(ALARM_NAME);
+  console.log("Check alarm cleared");
+}
+
+// Handle alarms
+chrome.alarms.onAlarm.addListener(async (alarm) => {
+  if (alarm.name === ALARM_NAME) {
+    if (!isChecking && config.enabled) {
+      // Skip check if we're still in bot-detection backoff window
+      if (Date.now() < botBackoffUntil) {
+        const remainingSec = Math.round((botBackoffUntil - Date.now()) / 1000);
+        console.log(`[Backoff] Skipping check — bot backoff active for ${remainingSec}s more`);
         return;
       }
-
-  const intervalMs = Math.max(60, config.checkIntervalSeconds) * 1000; // Min 60 seconds to avoid bot detection
-
-  setTimeout(async () => {
-    if (!isChecking && config.enabled) {
       isChecking = true;
       try {
+        // Service worker may have been killed and restarted since last check —
+        // restore state from storage before doing anything with tabs or config.
+        await loadConfig();
+        await loadPersistentTabId();
+        await loadBackoffState();
+        await loadCaptchaTabState();
         await runStockCheck();
       } finally {
         isChecking = false;
       }
     }
-    scheduleNextCheck();
-  }, intervalMs);
-}
-
-function stopCheckLoop() {
-  checkLoopRunning = false;
-  console.log("Check loop stopped");
-}
+  }
+});
 
 // Load config from storage
 async function loadConfig() {
@@ -100,7 +112,7 @@ async function loadConfig() {
 // Save config to storage
 async function saveConfig() {
   await chrome.storage.local.set({ config });
-  // Restart loop with new interval
+  // Recreate alarm with updated interval
   stopCheckLoop();
   if (config.enabled) {
     startCheckLoop();
@@ -121,6 +133,114 @@ async function saveProducts() {
   await chrome.storage.local.set({ knownProducts });
 }
 
+// Load/save persistent tab ID from storage so it survives service worker restarts
+async function loadPersistentTabId() {
+  const stored = await chrome.storage.local.get("persistentTabId");
+  if (stored.persistentTabId) {
+    try {
+      await chrome.tabs.get(stored.persistentTabId);
+      persistentTabId = stored.persistentTabId;
+      console.log(`Restored persistent tab ${persistentTabId}`);
+    } catch (e) {
+      persistentTabId = null;
+      await chrome.storage.local.remove("persistentTabId");
+    }
+  }
+}
+
+async function savePersistentTabId(tabId) {
+  persistentTabId = tabId;
+  await chrome.storage.local.set({ persistentTabId: tabId });
+}
+
+async function clearPersistentTab() {
+  const idToRemove = persistentTabId;
+  // Null out state immediately so a concurrent call doesn't double-remove
+  persistentTabId = null;
+  await chrome.storage.local.remove("persistentTabId");
+  if (idToRemove) {
+    try {
+      await chrome.tabs.remove(idToRemove);
+      console.log(`Closed persistent tab ${idToRemove}`);
+    } catch (e) {
+      // Tab was already closed externally — that's fine
+      console.log(`Tab ${idToRemove} already gone: ${e.message}`);
+    }
+  }
+}
+
+// Backoff state survives service worker restarts via storage
+async function loadBackoffState() {
+  const stored = await chrome.storage.local.get(["botBackoffUntil", "consecutiveBotDetections"]);
+  botBackoffUntil = stored.botBackoffUntil || 0;
+  consecutiveBotDetections = stored.consecutiveBotDetections || 0;
+}
+
+async function saveBackoffState() {
+  await chrome.storage.local.set({ botBackoffUntil, consecutiveBotDetections });
+}
+
+async function applyBotBackoff() {
+  consecutiveBotDetections++;
+  // Exponential backoff: 2^n minutes, capped at 30 minutes, with ±30s jitter
+  const backoffMinutes = Math.min(30, Math.pow(2, consecutiveBotDetections));
+  const jitterMs = (Math.random() - 0.5) * 60000; // ±30 seconds
+  botBackoffUntil = Date.now() + backoffMinutes * 60000 + jitterMs;
+  await saveBackoffState();
+  console.log(`[Backoff] Bot detected (${consecutiveBotDetections}x) — pausing checks for ~${backoffMinutes} min`);
+}
+
+function resetBotBackoff() {
+  if (consecutiveBotDetections > 0) {
+    console.log("[Backoff] Successful check — resetting bot detection counter");
+    consecutiveBotDetections = 0;
+    botBackoffUntil = 0;
+    chrome.storage.local.remove(["botBackoffUntil", "consecutiveBotDetections"]);
+  }
+}
+
+// --- Captcha tab tracking ---
+// Persists the tab ID of the open captcha page across service worker restarts.
+
+async function loadCaptchaTabState() {
+  const stored = await chrome.storage.local.get("captchaTabId");
+  if (stored.captchaTabId) {
+    try {
+      await chrome.tabs.get(stored.captchaTabId);
+      captchaTabId = stored.captchaTabId;
+      console.log(`[Captcha] Restored captcha tab ${captchaTabId}`);
+    } catch (e) {
+      captchaTabId = null;
+      await chrome.storage.local.remove("captchaTabId");
+    }
+  }
+}
+
+async function saveCaptchaTabId(tabId) {
+  captchaTabId = tabId;
+  if (tabId) {
+    await chrome.storage.local.set({ captchaTabId: tabId });
+  } else {
+    await chrome.storage.local.remove("captchaTabId");
+  }
+}
+
+// Returns 'solved' | 'still_captcha' | 'gone'
+async function checkCaptchaTabStatus() {
+  if (!captchaTabId) return 'gone';
+  try {
+    const tab = await chrome.tabs.get(captchaTabId);
+    // If still loading or on a known captcha URL, not solved yet
+    if (!tab.url || tab.status === 'loading' || isCaptchaUrl(tab.url)) {
+      return 'still_captcha';
+    }
+    // Tab is back on pokemoncenter — user likely solved it
+    return 'solved';
+  } catch (e) {
+    return 'gone'; // Tab was closed
+  }
+}
+
 // Load known SKUs from storage (for API monitoring)
 async function loadKnownSkus() {
   const stored = await chrome.storage.local.get("knownSkus");
@@ -219,20 +339,60 @@ async function runStockCheck() {
     return;
   }
 
+  // If we have a captcha tab open, check whether the user has solved it
+  // before opening any new tabs or sending more warnings.
+  if (captchaTabId) {
+    const status = await checkCaptchaTabStatus();
+    if (status === 'still_captcha') {
+      console.log(`[Captcha] Tab ${captchaTabId} still showing captcha — waiting for user`);
+      await applyBotBackoff(); // Re-arm backoff silently; no Discord ping
+      return;
+    }
+    if (status === 'solved') {
+      console.log(`[Captcha] Tab ${captchaTabId} looks resolved — resuming normal checks`);
+      // Reclaim the solved tab as the persistent tab so we don't open a new one
+      await savePersistentTabId(captchaTabId);
+      await saveCaptchaTabId(null);
+      resetBotBackoff();
+    } else { // 'gone' — user closed the tab themselves
+      console.log('[Captcha] Captcha tab was closed — resuming normal checks');
+      await saveCaptchaTabId(null);
+      resetBotBackoff();
+    }
+  }
+
   console.log("Running stock check...");
 
   for (const url of config.urls) {
     try {
       const products = await fetchAndParseProducts(url);
 
-      // Alert if 0 products found - likely bot protection
+      // Alert if 0 products found — likely captcha or bot protection
       if (products.length === 0) {
-        console.warn("WARNING: 0 products found - possible bot protection!");
+        console.warn("WARNING: 0 products found - possible bot protection or captcha!");
+        botProtectionDetected = true;
+        const isFirstDetection = consecutiveBotDetections === 0;
+        await applyBotBackoff();
+        if (isFirstDetection) {
+          // Only ping Discord on the first detection — not on every retry
+          const backoffMin = Math.round((botBackoffUntil - Date.now()) / 60000);
           await sendWarningNotification("Bot Protection Detected",
-          "Found 0 products - you may need to manually verify you're human on PokemonCenter.com");
+            `Found 0 products — captcha may be active. Pausing checks for ~${backoffMin} min. Check the open PokemonCenter tab and solve the captcha if prompted.`);
+        }
+        // Keep the tab open so the user can solve the captcha.
+        // If fetchAndParseProducts already saved it as captchaTabId (URL-detected),
+        // use that; otherwise promote the current persistentTabId so we track it.
+        if (!captchaTabId && persistentTabId) {
+          await saveCaptchaTabId(persistentTabId);
+        }
+        persistentTabId = null;
+        await chrome.storage.local.remove("persistentTabId");
         continue;
       }
 
+      // Successful product load — reset any backoff state
+      resetBotBackoff();
+
       const { newProducts, restockedProducts } = processProducts(products);
 
       // Send notifications with rate limiting
@@ -275,37 +435,48 @@ async function runStockCheck() {
   }
 }
 
+// Detect captcha/challenge pages by URL patterns (all lowercase comparisons)
+function isCaptchaUrl(url) {
+  if (!url) return false;
+  const lower = url.toLowerCase();
+  // Known bot-protection URL signatures
+  if (lower.includes('_incapsula_resource')) return true;
+  if (lower.includes('/cdn-cgi/challenge')) return true;
+  if (lower.includes('imperva')) return true;
+  if (lower.includes('incapsula')) return true;
+  if (lower.includes('distil')) return true;
+  // "challenge" or "verify" only flag if we've left the pokemoncenter domain entirely
+  if (!lower.includes('pokemoncenter.com')) return true;
+  return false;
+}
+
 // Fetch and parse products from a URL using content script
 async function fetchAndParseProducts(url) {
-  return new Promise(async (resolve, reject) => {
-    try {
   let tabId;
-      let isNewTab = false;
-
-      // Reuse existing tab or create new one
+  try {
+    // Reuse existing tab or create a new one
     if (persistentTabId) {
       try {
-          const tab = await chrome.tabs.get(persistentTabId);
-          tabId = tab.id;
-          console.log(`Reusing tab ${tabId}, refreshing...`);
+        await chrome.tabs.get(persistentTabId);
+        tabId = persistentTabId;
+        console.log(`Reusing tab ${tabId}, navigating...`);
         await chrome.tabs.update(tabId, { url });
       } catch (e) {
-          // Tab was closed, create new one
-          persistentTabId = null;
+        // Tab is gone — clear it and fall through to create a new one
+        await clearPersistentTab();
+        tabId = null;
       }
     }
 
-      if (!persistentTabId) {
+    if (!tabId) {
       const tab = await chrome.tabs.create({ url, active: false });
       tabId = tab.id;
-        persistentTabId = tabId;
-        isNewTab = true;
+      await savePersistentTabId(tabId);
       console.log(`Created new tab ${tabId}`);
     }
 
     // Wait for tab to finish loading
-      const waitForLoad = () => {
-        return new Promise((res) => {
+    await new Promise((res) => {
       const listener = (tid, changeInfo) => {
         if (tid === tabId && changeInfo.status === "complete") {
           chrome.tabs.onUpdated.removeListener(listener);
@@ -318,26 +489,41 @@ async function fetchAndParseProducts(url) {
         res();
       }, 20000);
     });
-      };
 
-      await waitForLoad();
+    // Check if the tab landed on a captcha/challenge page
+    try {
+      const tab = await chrome.tabs.get(tabId);
+      if (isCaptchaUrl(tab.url)) {
+        console.warn(`[Captcha] Tab ${tabId} landed on challenge page: ${tab.url}`);
+        // Keep tab open for user; track it so we can poll for resolution
+        await saveCaptchaTabId(tabId);
+        persistentTabId = null;
+        await chrome.storage.local.remove("persistentTabId");
+        return []; // Caller will handle backoff + single ping
+      }
+    } catch (e) { /* tab may have been closed */ }
+
     console.log(`Tab ${tabId} loaded`);
 
-      // Smart wait: poll for products instead of fixed wait
+    // Poll for products — PokemonCenter is a React SPA so status:"complete" fires
+    // on the HTML shell before products are rendered via API calls.
+    // Inject content.js once, then poll via messaging.
     let products = [];
     const startTime = Date.now();
-      const maxWait = 8000; // Max 8 seconds
-      const pollInterval = 500; // Check every 500ms
+    const maxWait = 25000;
+    const pollInterval = 1000; // Reduced polling frequency
 
+    let injected = false;
     while (Date.now() - startTime < maxWait) {
       try {
-          // Inject content script if needed
+        if (!injected) {
           try {
-            await chrome.scripting.executeScript({
-              target: { tabId },
-              files: ["content.js"]
-            });
-          } catch (e) { /* Already injected */ }
+            await chrome.scripting.executeScript({ target: { tabId }, files: ["content.js"] });
+            injected = true;
+          } catch (e) {
+            injected = true; // Already injected — treat as done
+          }
+        }
 
         await new Promise(r => setTimeout(r, pollInterval));
 
@@ -354,26 +540,30 @@ async function fetchAndParseProducts(url) {
     }
 
     if (products.length === 0) {
-        console.log(`No products found after ${maxWait}ms`);
+      console.log(`No products found after ${maxWait}ms — assuming captcha/bot block`);
+      // Keep the tab open and track it so runStockCheck can poll for resolution
+      await saveCaptchaTabId(tabId);
+      persistentTabId = null;
+      await chrome.storage.local.remove("persistentTabId");
     }
 
     // Apply keyword filter
     if (config.keywords && config.keywords.length > 0) {
-        products = products.filter(p => {
-          const nameLower = p.name.toLowerCase();
-          return config.keywords.some(kw => nameLower.includes(kw.toLowerCase()));
-        });
+      products = products.filter(p =>
+        config.keywords.some(kw => p.name.toLowerCase().includes(kw.toLowerCase()))
+      );
       console.log(`After keyword filter: ${products.length} products`);
     }
 
-      // Don't close tab - keep for reuse
-      resolve(products);
+    // Keep tab open for reuse next check
+    return products;
 
   } catch (error) {
-      persistentTabId = null; // Reset on error
-      reject(error);
+    // On any error, close the tab so we don't leave an untracked orphan
+    await clearPersistentTab();
+    tabId = null;
+    throw error;
   }
-  });
 }
 
 // Note: Product parsing is now done by the content script (content.js)
@@ -548,6 +738,7 @@ async function sendDiscordNotification(product, alertType) {
 
 // Track pending events to sync (restocks, new drops)
 let pendingEvents = [];
+let botProtectionDetected = false;
 
 // Sync data to local dashboard
 async function syncToDashboard() {
@@ -566,7 +757,8 @@ async function syncToDashboard() {
     skus: Array.from(knownSkus),
     products: productsToSync,
     apiStats: apiStats,
-    events: pendingEvents  // Include pending events (restocks, new drops)
+    events: pendingEvents,
+    bot_protection_detected: botProtectionDetected
   };
 
   try {
@@ -579,8 +771,9 @@ async function syncToDashboard() {
     if (response.ok) {
       const result = await response.json();
       console.log(`[Dashboard Sync] Success - ${result.total_skus} SKUs, ${result.total_products} products`);
-      // Clear pending events after successful sync
+      // Clear pending events and flags after successful sync
       pendingEvents = [];
+      botProtectionDetected = false;
       return { success: true, ...result };
     } else {
       console.warn(`[Dashboard Sync] Failed: ${response.status}`);
@@ -600,12 +793,6 @@ function extractSkuFromUrl(url) {
   return match ? match[1] : null;
 }
 
-// Auto-sync to dashboard periodically (every 5 minutes)
-setInterval(() => {
-  if (config.syncToDashboard) {
-    syncToDashboard();
-  }
-}, 5 * 60 * 1000);
 
 
 // Listen for messages from popup and content scripts
@@ -648,16 +835,24 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
 });
 
 // Run initial check after a short delay
-setTimeout(() => {
-  loadConfig().then(() => {
+setTimeout(async () => {
+  await loadConfig();
+  await loadPersistentTabId(); // must complete before any check runs
+  await loadBackoffState();
+  await loadCaptchaTabState();
   loadProducts();
-    loadKnownSkus().then(() => {
+  await loadKnownSkus();
   console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center");
   if (config.enabled) {
     console.log(`Check interval: ${config.checkIntervalSeconds} seconds`);
-        runStockCheck();
     startCheckLoop();
+    // Skip initial check if still in backoff window
+    if (Date.now() < botBackoffUntil) {
+      const remainingSec = Math.round((botBackoffUntil - Date.now()) / 1000);
+      console.log(`[Backoff] Startup — skipping initial check, backoff active for ${remainingSec}s more`);
+    } else if (!isChecking) {
+      isChecking = true;
+      runStockCheck().finally(() => { isChecking = false; });
+    }
   }
-    });
-  });
 }, 3000);

chrome-extension/content.js

@@ -18,10 +18,14 @@
     if (event.data?.source !== 'pokemon-api-interceptor') return;
 
     // Forward API data to background script
+    try {
       chrome.runtime.sendMessage({
         type: 'apiData',
         data: event.data.data
       }).catch(() => {});
+    } catch (e) {
+      // Extension context invalidated (tab closing or extension reloaded)
+    }
   });
 
   // Inject the interceptor

config.py

@@ -3,10 +3,21 @@ Configuration for Pokemon Stock Monitor
 Update DISCORD_WEBHOOK_URL with your actual webhook URL
 """
 
+import os
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass  # python-dotenv not installed, fall back to env vars or hardcoded values
+
+def _env(key, default=""):
+    return os.environ.get(key) or default
+
 # Discord webhook URL - GET THIS FROM YOUR DISCORD SERVER
 # Server Settings -> Integrations -> Webhooks -> New Webhook -> Copy Webhook URL
 # This is the default/fallback webhook for anything not specifically routed below
-DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1488982276455796756/4HEzQDBF-Hs4a9Iyd7yvj7-SfP0BwrbQGdGBmQ9TL9urWzkTRaHelkvrqX4v9NTf2ZJ_"
+# Set DISCORD_WEBHOOK_DEFAULT in .env to override
+DISCORD_WEBHOOK_URL = _env("DISCORD_WEBHOOK_DEFAULT", "https://discord.com/api/webhooks/1488982276455796756/4HEzQDBF-Hs4a9Iyd7yvj7-SfP0BwrbQGdGBmQ9TL9urWzkTRaHelkvrqX4v9NTf2ZJ_")
 
 # =============================================================================
 # MULTI-CHANNEL WEBHOOK ROUTING (Optional)
@@ -21,24 +32,26 @@ DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1488982276455796756/4HEz
 #   2. sites (all alerts for a specific store)
 #   3. types (all alerts of a specific type)
 #   4. default (DISCORD_WEBHOOK_URL)
+#
+# Override any value by setting the corresponding env var in .env
 
 DISCORD_WEBHOOKS = {
     # Per-site channels - all alerts from a specific store go here first
     "sites": {
-        "target": "https://discord.com/api/webhooks/1488982720901025852/O0KkVFHrQ6-k6FmCqQE4QpuWF6YFJ8YyFgojBPKcQu9g0UHzTpnAU9G1gbhOftARS11R",
-        "bestbuy": "https://discord.com/api/webhooks/1488982786781216798/AtG6x7FWzJjOsmosq9oPMuwlPrSwNHZJJTUuB3AApjyT7LIOxmEcYeiMFP7HiA_Ov1pn",
-        "gamestop": "https://discord.com/api/webhooks/1488982854036619355/NWZtydrKEhj-kxKbFi78f1Gpf3h16W_1kAJEcSuIgtjJUoAO8Sk7inOI2T8fb9GXo9xY",
-        "walmart": "https://discord.com/api/webhooks/1488982929450340375/0g16tJXw8f9UtUrRrLd2jDPWWD__LTDYO_X2VT2WL4F5GBYB8ykVxrjV-bCjzKKjSA5C",
-        "pokemoncenter": "https://discord.com/api/webhooks/1488983050090975364/DwY9QQvBZa6V-zz8mIg302sukbISSbkXOhDwe1Q6yDG0zAwvaGQcn8iTzNB4D676e3cA",
+        "target":        _env("DISCORD_WEBHOOK_TARGET",        "https://discord.com/api/webhooks/1488982720901025852/O0KkVFHrQ6-k6FmCqQE4QpuWF6YFJ8YyFgojBPKcQu9g0UHzTpnAU9G1gbhOftARS11R") or None,
+        "bestbuy":       _env("DISCORD_WEBHOOK_BESTBUY",       "https://discord.com/api/webhooks/1488982786781216798/AtG6x7FWzJjOsmosq9oPMuwlPrSwNHZJJTUuB3AApjyT7LIOxmEcYeiMFP7HiA_Ov1pn") or None,
+        "gamestop":      _env("DISCORD_WEBHOOK_GAMESTOP",      "https://discord.com/api/webhooks/1488982854036619355/NWZtydrKEhj-kxKbFi78f1Gpf3h16W_1kAJEcSuIgtjJUoAO8Sk7inOI2T8fb9GXo9xY") or None,
+        "walmart":       _env("DISCORD_WEBHOOK_WALMART",       "https://discord.com/api/webhooks/1488982929450340375/0g16tJXw8f9UtUrRrLd2jDPWWD__LTDYO_X2VT2WL4F5GBYB8ykVxrjV-bCjzKKjSA5C") or None,
+        "pokemoncenter": _env("DISCORD_WEBHOOK_POKEMONCENTER", "https://discord.com/api/webhooks/1488983050090975364/DwY9QQvBZa6V-zz8mIg302sukbISSbkXOhDwe1Q6yDG0zAwvaGQcn8iTzNB4D676e3cA") or None,
     },
 
     # Per-type channels - used when no site-specific webhook is set
     "types": {
-        "restock": "https://discord.com/api/webhooks/1488982403048542269/s1J_p8R6IMGQGypPh_ZYTHHBD0cMnd_BjLGggVWaL2OcyDQAR42KS-CyuHcVr5s21vl3",
-        "new_drop": "https://discord.com/api/webhooks/1488982511613771898/RLDoHxNjsAx0PfHfT0kZe6-IW3g9PR_ckX_sI-CALcv4Lt0USslvI0rqePqW9gVP7D7v",
-        "price_change": "https://discord.com/api/webhooks/1488982573723025633/J864UL4vk_q9Rmj4i4wtkGfzB3nKAOc0y7jzBFDgUbhwg4XKm1Z7sWduqRc6x_ThWetc",
-        "error": "https://discord.com/api/webhooks/1488983128641900657/RP5FRSWwbVaRAf7RdXPs66ETbY8ziI1ekIx7lrHqxp8n200KpzUiX_EylCxXBH5nJD3B",
-        "captcha": "https://discord.com/api/webhooks/1488983128641900657/RP5FRSWwbVaRAf7RdXPs66ETbY8ziI1ekIx7lrHqxp8n200KpzUiX_EylCxXBH5nJD3B",
+        "restock":      _env("DISCORD_WEBHOOK_RESTOCK",       "https://discord.com/api/webhooks/1488982403048542269/s1J_p8R6IMGQGypPh_ZYTHHBD0cMnd_BjLGggVWaL2OcyDQAR42KS-CyuHcVr5s21vl3") or None,
+        "new_drop":     _env("DISCORD_WEBHOOK_NEW_DROP",      "https://discord.com/api/webhooks/1488982511613771898/RLDoHxNjsAx0PfHfT0kZe6-IW3g9PR_ckX_sI-CALcv4Lt0USslvI0rqePqW9gVP7D7v") or None,
+        "price_change": _env("DISCORD_WEBHOOK_PRICE_CHANGE",  "https://discord.com/api/webhooks/1488982573723025633/J864UL4vk_q9Rmj4i4wtkGfzB3nKAOc0y7jzBFDgUbhwg4XKm1Z7sWduqRc6x_ThWetc") or None,
+        "error":        _env("DISCORD_WEBHOOK_ERROR",         "https://discord.com/api/webhooks/1488983128641900657/RP5FRSWwbVaRAf7RdXPs66ETbY8ziI1ekIx7lrHqxp8n200KpzUiX_EylCxXBH5nJD3B") or None,
+        "captcha":      _env("DISCORD_WEBHOOK_ERROR",         "https://discord.com/api/webhooks/1488983128641900657/RP5FRSWwbVaRAf7RdXPs66ETbY8ziI1ekIx7lrHqxp8n200KpzUiX_EylCxXBH5nJD3B") or None,
     },
 
     # Override specific site + type combinations (highest priority)
@@ -62,7 +75,8 @@ RESTOCK_COOLDOWN_HOURS = 6  # Minimum hours between restock alerts for same prod
 # Discord Bot Token (for interactive bot commands)
 # Create a bot at https://discord.com/developers/applications
 # Required for !setlocation, !stores, !stock commands
-DISCORD_BOT_TOKEN = "MTQ4NjQ2ODM2NDkyMTczNzM3OQ.Gxl8pq.wIvK2eVbp1G0SJhW4XTYD3zopR1gohaNv5sxlQ"
+# Set DISCORD_BOT_TOKEN in .env to override
+DISCORD_BOT_TOKEN = _env("DISCORD_BOT_TOKEN", "MTQ4NjQ2ODM2NDkyMTczNzM3OQ.Gxl8pq.wIvK2eVbp1G0SJhW4XTYD3zopR1gohaNv5sxlQ")
 
 # How often to check each site (in seconds)
 CHECK_INTERVAL_SECONDS = 60
@@ -83,7 +97,7 @@ SORT_BY_NEWEST = True
 # Which sites to monitor
 # Note: PokemonCenter has strong bot protection (Imperva) - may need manual workarounds
 # Note: Walmart has aggressive bot protection (PerimeterX) - may need stealth mode
-SITES_ENABLED = ["target", "bestbuy", "gamestop"]  # Options: "pokemoncenter", "target", "gamestop", "bestbuy" (disabled - needs rework), "walmart"
+SITES_ENABLED = ["target", "gamestop", "bestbuy"]  # Options: "pokemoncenter", "target", "gamestop", "bestbuy" (disabled - needs rework), "walmart"
 
 # PokemonCenter URLs to monitor
 POKEMON_CENTER_URLS = [
@@ -146,6 +160,8 @@ USE_STEALTH_BROWSER = True
 
 # Stealth browser settings
 STEALTH_HEADLESS = True  # Set True for headless/no-display environments
+GAMESTOP_HEADLESS = False  # GameStop (Cloudflare) hard-blocks headless Chrome - must stay False
+BESTBUY_HEADLESS = False  # BestBuy detects headless and blocks GraphQL product loading - must stay False
 STEALTH_SESSION_NAME = "pokemoncenter"  # Name for cookie/session persistence
 
 # Human-like behavior settings

dashboard/api.py

@@ -13,6 +13,7 @@ from flask import Blueprint, jsonify, request
 from src.database import get_database
 from src.favorites import get_favorites_manager
 from src.scraper_state import scraper_state
+from src.discord_notifier import send_error_notification
 
 api_bp = Blueprint('api', __name__)
 
@@ -796,6 +797,14 @@ def sync_extension_data():
                         db.update_product_price(product_id, new_price)
                     events_processed += 1
 
+    # Check for bot protection - extension explicitly reports when a check returned 0 products
+    if data.get('bot_protection_detected'):
+        send_error_notification(
+            "Pokemon Center extension detected possible bot protection — 0 products found on last check. "
+            "Check the extension tab manually and verify you can browse PokemonCenter.com.",
+            site="pokemoncenter"
+        )
+
     # Update API stats
     if 'apiStats' in data:
         extension_data['api_stats'] = data['apiStats']

dashboard/static/app.js

@@ -878,7 +878,7 @@ function renderScrapersGrid(scrapers) {
               data.enabled ? '<span class="status-enabled">&#9679; Enabled</span>' :
               '<span class="status-disabled">&#9675; Disabled</span>'}
           </div>
-          ${data.last_run ? `<div class="scraper-last-run">Last: ${formatTime(new Date(data.last_run))}</div>` : ''}
+          <div class="scraper-last-run">Last: ${data.last_run ? formatTime(new Date(data.last_run)) : 'Never'}</div>
           ${data.last_error ? `<div class="scraper-error">${data.last_error}</div>` : ''}
         </div>
         <label class="toggle-switch">

main.py

@@ -28,6 +28,7 @@ from config import (
 from src.browser import get_browser, shutdown_browser
 from src.product_tracker import ProductTracker
 from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification, send_price_change_alert
+from src.scraper_state import scraper_state
 # Note: Pokemon Center uses Chrome Extension (chrome-extension/), not a Python scraper
 from scrapers import (
     TargetScraper,
@@ -139,6 +140,7 @@ def check_target():
         # Log successful check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("target", total_products, total_new, total_restocks, duration_ms, success=True)
+        scraper_state.record_check("target", success=True)
 
     except Exception as e:
         logger.error(f"Error checking Target: {e}")
@@ -146,6 +148,7 @@ def check_target():
         # Log failed check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("target", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
+        scraper_state.record_check("target", success=False, error=str(e))
 
 
 def check_gamestop():
@@ -228,6 +231,7 @@ def check_gamestop():
         # Log successful check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("gamestop", total_products, total_new, total_restocks, duration_ms, success=True)
+        scraper_state.record_check("gamestop", success=True)
 
     except Exception as e:
         logger.error(f"Error checking GameStop: {e}")
@@ -235,6 +239,7 @@ def check_gamestop():
         # Log failed check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("gamestop", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
+        scraper_state.record_check("gamestop", success=False, error=str(e))
 
 
 def check_bestbuy():
@@ -317,6 +322,7 @@ def check_bestbuy():
         # Log successful check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("bestbuy", total_products, total_new, total_restocks, duration_ms, success=True)
+        scraper_state.record_check("bestbuy", success=True)
 
     except Exception as e:
         logger.error(f"Error checking Best Buy: {e}")
@@ -324,6 +330,7 @@ def check_bestbuy():
         # Log failed check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("bestbuy", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
+        scraper_state.record_check("bestbuy", success=False, error=str(e))
 
 
 def check_walmart():
@@ -406,6 +413,7 @@ def check_walmart():
         # Log successful check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("walmart", total_products, total_new, total_restocks, duration_ms, success=True)
+        scraper_state.record_check("walmart", success=True)
 
     except Exception as e:
         logger.error(f"Error checking Walmart: {e}")
@@ -413,6 +421,7 @@ def check_walmart():
         # Log failed check to database
         duration_ms = int((time.time() - start_time) * 1000)
         tracker.log_check("walmart", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
+        scraper_state.record_check("walmart", success=False, error=str(e))
 
 
 def run_checks():
@@ -440,6 +449,11 @@ def graceful_shutdown(signum, frame):
     """Handle shutdown gracefully"""
     logger.info("Shutting down...")
     shutdown_browser()
+    scraper_state.state["monitor_running"] = False
+    scraper_state.state["monitor_pid"] = None
+    for site in scraper_state.state["scrapers"]:
+        scraper_state.state["scrapers"][site]["running"] = False
+    scraper_state._save_state()
     sys.exit(0)
 
 
@@ -469,6 +483,15 @@ def main():
         print("  playwright install chromium")
         return
 
+    # Mark monitor as running in state file
+    import os
+    scraper_state.state["monitor_running"] = True
+    scraper_state.state["monitor_pid"] = os.getpid()
+    for site in SITES_ENABLED:
+        if site in scraper_state.state["scrapers"]:
+            scraper_state.state["scrapers"][site]["running"] = True
+    scraper_state._save_state()
+
     # Send startup notification
     send_startup_notification()
 

requirements.txt

@@ -1,4 +1,5 @@
 playwright>=1.45.0
+python-dotenv>=1.0.0
 playwright-stealth>=1.0.6
 requests>=2.31.0
 schedule>=1.2.1

scrapers/bestbuy.py

@@ -12,6 +12,7 @@ from typing import List, Optional
 from bs4 import BeautifulSoup
 
 from .base import BaseScraper, Product
+from config import BESTBUY_HEADLESS
 
 # Add parent directory to path for tools import
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -37,7 +38,7 @@ class BestBuyScraper(BaseScraper):
         # Check if we need to create a new browser
         if self._stealth_browser is None:
             logger.info("Creating new Best Buy stealth browser...")
-            self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
+            self._stealth_browser = StealthBrowser(headless=BESTBUY_HEADLESS, session_name="bestbuy")
             self._stealth_browser.start()
             self._restart_attempts = 0
             return self._stealth_browser
@@ -63,7 +64,11 @@ class BestBuyScraper(BaseScraper):
         Returns True if browser was restarted and operation should be retried.
         """
         error_msg = str(error).lower()
-        session_errors = ["invalid session id", "session deleted", "no such session", "browser has closed"]
+        session_errors = [
+            "invalid session id", "session deleted", "no such session", "browser has closed",
+            "newconnectionerror", "connection refused", "winerror 10061",
+            "failed to establish a new connection", "max retries exceeded",
+        ]
 
         if any(err in error_msg for err in session_errors):
             logger.warning(f"Session error detected: {error}")
@@ -90,11 +95,6 @@ class BestBuyScraper(BaseScraper):
         """
         all_products = []
 
-        # Add sort by newest if not already in URL
-        if "sort=" not in url:
-            separator = "&" if "?" in url else "?"
-            url = f"{url}{separator}sp=-releasedate"
-
         # Use stealth browser
         try:
             browser = self._get_stealth_browser()
@@ -113,22 +113,26 @@ class BestBuyScraper(BaseScraper):
                 try:
                     logger.info(f"Navigating to: {page_url}")
                     browser.driver.get(page_url)
-                    time.sleep(8)  # Wait longer for initial page load
+                    time.sleep(15)  # Wait for React hydration and GraphQL fetches to complete
 
-                    # Scroll for lazy loading - wait longer between scrolls
-                    browser.driver.execute_script("window.scrollTo(0, 500)")
+                    # Scroll to trigger any remaining lazy-loaded products
+                    scroll_pause = 1.5
+                    scroll_step = 800
+                    current_pos = 0
+                    for _ in range(20):
+                        current_pos += scroll_step
+                        browser.driver.execute_script(f"window.scrollTo(0, {current_pos})")
+                        time.sleep(scroll_pause)
+                        page_height = browser.driver.execute_script("return document.body.scrollHeight")
+                        if current_pos >= page_height:
+                            break
+                    browser.driver.execute_script("window.scrollTo(0, 0)")
                     time.sleep(2)
-                    browser.driver.execute_script("window.scrollTo(0, 1500)")
-                    time.sleep(2)
-                    browser.driver.execute_script("window.scrollTo(0, 3000)")
-                    time.sleep(2)
-                    browser.driver.execute_script("window.scrollTo(0, 0)")  # Scroll back to top
-                    time.sleep(3)
 
                     # Wait for products to load (check for content)
                     for _ in range(10):
                         html = browser.driver.page_source
-                        if "sku-item" in html or "sku-title" in html or "priceView" in html:
+                        if any(kw in html for kw in ["sku-item", "sku-title", "priceView", "product-list-item", "ApolloSSRDataTransport", '"pdp":']):
                             logger.info("Product content detected")
                             break
                         time.sleep(1)
@@ -136,10 +140,12 @@ class BestBuyScraper(BaseScraper):
                     # Get HTML after waiting
                     html = browser.driver.page_source
 
-                    # Save debug screenshot
+                    # Save debug screenshot and HTML for analysis
                     try:
                         browser.driver.save_screenshot("debug_bestbuy.png")
-                        logger.info("Saved debug screenshot to debug_bestbuy.png")
+                        with open("debug_bestbuy.html", "w", encoding="utf-8") as f:
+                            f.write(html)
+                        logger.info("Saved debug screenshot and HTML to debug_bestbuy.*")
                     except:
                         pass
 
@@ -274,6 +280,45 @@ class BestBuyScraper(BaseScraper):
         logger.info(f"Scraped {len(unique_products)} unique products from Best Buy")
         return unique_products
 
+    def _extract_from_graphql_response(self, data: dict) -> List[Product]:
+        """Extract products from BestBuy's intercepted GraphQL search response"""
+        products = []
+        try:
+            search = data.get('searchResult') or data.get('searchResults') or {}
+            items = search.get('products') or search.get('items') or []
+            logger.info(f"GraphQL response has {len(items)} products")
+            for item in items:
+                product = item.get('product') or item
+                name = (product.get('name') or {}).get('short') or (product.get('name') or {}).get('title') or product.get('displayName') or ''
+                if not name or len(name) < 5:
+                    continue
+                sku_id = str(product.get('skuId') or '')
+                url_obj = product.get('url') or {}
+                url = url_obj.get('skuSpecificUrl') or url_obj.get('pdp') or (f"{self.base_url}/site/{sku_id}.p" if sku_id else None)
+                if not url:
+                    continue
+                image = (product.get('primaryImage') or {}).get('piscesHref')
+                price = None
+                price_info = product.get('priceInfo') or product.get('price') or {}
+                if isinstance(price_info, dict):
+                    raw = price_info.get('currentPrice') or price_info.get('regularPrice')
+                    if isinstance(raw, (int, float)):
+                        price = f"${raw:.2f}"
+                # Check stock via button state
+                in_stock = True
+                fulfillment = product.get('fulfillmentOptions') or {}
+                btn_states = fulfillment.get('buttonStates') or []
+                if btn_states:
+                    state = btn_states[0].get('buttonState', '')
+                    in_stock = state in ('ADD_TO_CART', 'PRE_ORDER', 'CHECK_STORES')
+                products.append(Product(
+                    name=name, url=url, price=price, in_stock=in_stock,
+                    image_url=image, site=self.site_name, product_id=sku_id,
+                ))
+        except Exception as e:
+            logger.error(f"Error parsing GraphQL response: {e}")
+        return products
+
     def _extract_from_page_data(self, soup, browser) -> List[Product]:
         """Extract products from page data attributes and evaluate JS if needed"""
         products = []
@@ -298,7 +343,58 @@ class BestBuyScraper(BaseScraper):
                 except (json.JSONDecodeError, TypeError) as e:
                     logger.debug(f"Could not parse __NEXT_DATA__ from HTML: {e}")
 
-        # Method 3: Try multiple JS state sources via browser execution
+        # Method 3: Extract from live Apollo client cache (captures client-side loaded products)
+        # BestBuy SSRs only ~4 featured products; the rest load via client-side GraphQL.
+        # The Apollo client cache holds all of them after hydration.
+        apollo_cache_script = """
+            try {
+                var client = window.__APOLLO_CLIENT__;
+                if (!client) return null;
+                var cache = client.cache.extract();
+                var products = [];
+                for (var key in cache) {
+                    var obj = cache[key];
+                    if (obj && obj.__typename === 'Product' && obj.skuId && obj.name && obj.name.short) {
+                        products.push({
+                            skuId: obj.skuId,
+                            name: obj.name.short || obj.name.title || '',
+                            pdp: obj.url ? obj.url.pdp || obj.url.skuSpecificUrl : null,
+                            price: obj.priceAndAvailabilityInfo ? obj.priceAndAvailabilityInfo.currentPrice : null,
+                            inStock: obj.fulfillmentOptions ? !!(obj.fulfillmentOptions.buttonStates || []).find(function(b){ return b.buttonState === 'ADD_TO_CART'; }) : true,
+                            image: obj.primaryImage ? obj.primaryImage.piscesHref : null
+                        });
+                    }
+                }
+                return products.length > 0 ? JSON.stringify(products) : null;
+            } catch(e) { return null; }
+        """
+        try:
+            apollo_json = browser.driver.execute_script(apollo_cache_script)
+            if apollo_json:
+                apollo_products = json.loads(apollo_json)
+                logger.info(f"Extracted {len(apollo_products)} products from Apollo client cache")
+                for item in apollo_products:
+                    if not item.get('name') or len(item.get('name', '')) < 5:
+                        continue
+                    sku_id = str(item.get('skuId', ''))
+                    pdp = item.get('pdp')
+                    url = pdp if pdp else (f"{self.base_url}/site/{sku_id}.p" if sku_id else None)
+                    if not url:
+                        continue
+                    price = item.get('price')
+                    products.append(Product(
+                        name=item['name'],
+                        url=url,
+                        price=f"${price:.2f}" if isinstance(price, (int, float)) else str(price) if price else None,
+                        in_stock=item.get('inStock', True),
+                        image_url=item.get('image'),
+                        site=self.site_name,
+                        product_id=sku_id,
+                    ))
+        except Exception as e:
+            logger.debug(f"Could not extract from Apollo client cache: {e}")
+
+        # Method 4: Try multiple JS state sources via browser execution
         if not products:
             state_scripts = [
                 "return window.__NEXT_DATA__ ? JSON.stringify(window.__NEXT_DATA__) : null",
@@ -428,21 +524,18 @@ class BestBuyScraper(BaseScraper):
                 if image_matches:
                     image_url = image_matches[0]
 
-                # Try to determine stock status from Apollo data context
-                in_stock = True  # Default to True if no stock info found
-                if url_pos > 0:
-                    # Look for availability info near this product's URL
-                    context_start = max(0, url_pos - 3000)
-                    context_end = min(len(script_content), url_pos + 500)
-                    product_context = script_content[context_start:context_end]
-
-                    # Check for explicit out of stock indicators
-                    if '"isAvailable":false' in product_context or '"available":false' in product_context:
-                        in_stock = False
-                    elif '"soldOut":true' in product_context or '"outOfStock":true' in product_context:
-                        in_stock = False
-                    elif 'sold out' in product_context.lower() or 'out of stock' in product_context.lower():
-                        in_stock = False
+                # Determine stock status by anchoring to the SKU ID in the Apollo data,
+                # then searching for buttonState within 500 chars after it.
+                # This is more reliable than looking near the URL (which bleeds across products).
+                in_stock = True
+                if sku_id:
+                    sku_anchor = f'"skuId":"{sku_id}"'
+                    sku_pos = script_content.find(sku_anchor)
+                    if sku_pos >= 0:
+                        sku_context = script_content[sku_pos:sku_pos + 500]
+                        btn_match = re.search(r'"buttonState":"([^"]+)"', sku_context)
+                        if btn_match:
+                            in_stock = btn_match.group(1) in ("ADD_TO_CART", "PRE_ORDER", "CHECK_STORES")
 
                 if name and len(name) > 5:
                     products.append(Product(
@@ -789,12 +882,14 @@ class BestBuyScraper(BaseScraper):
                         break
                     parent = parent.find_parent()
 
-            # Check stock
+            # Check stock — "add to cart" takes priority over "unavailable"
             in_stock = True
             parent = link.find_parent()
             for _ in range(5):
                 if parent:
                     text = parent.get_text().lower()
+                    if any(phrase in text for phrase in ["add to cart", "add to bag", "pre-order"]):
+                        break  # purchasable — in_stock stays True
                     if any(phrase in text for phrase in ["sold out", "out of stock", "unavailable"]):
                         in_stock = False
                         break
@@ -818,6 +913,13 @@ class BestBuyScraper(BaseScraper):
         """Check if a product card indicates in-stock status"""
         card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
 
+        # "Add to Cart" takes priority — a card can show "Unavailable" for in-store/delivery
+        # while still being purchasable online, so check purchasable state first.
+        if "add to cart" in card_text or "add to bag" in card_text:
+            return True
+        if "pre-order" in card_text:
+            return True
+
         # Check for disabled add to cart button
         add_btn = card.select_one(".add-to-cart-button")
         if add_btn and "btn-disabled" in add_btn.get("class", []):
@@ -835,17 +937,6 @@ class BestBuyScraper(BaseScraper):
             if phrase in card_text:
                 return False
 
-        in_stock_phrases = [
-            "add to cart",
-            "add to bag",
-            "available",
-            "in stock",
-        ]
-
-        for phrase in in_stock_phrases:
-            if phrase in card_text:
-                return True
-
         return True
 
     def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:

scrapers/gamestop.py

@@ -12,7 +12,7 @@ from bs4 import BeautifulSoup
 
 from .base import BaseScraper, Product
 from src.browser import get_browser
-from config import CAPTCHA_WAIT_TIMEOUT
+from config import CAPTCHA_WAIT_TIMEOUT, GAMESTOP_HEADLESS
 
 # Add parent directory to path for tools import
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -20,6 +20,20 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 logger = logging.getLogger(__name__)
 
 
+def _extract_img_url(img_tag) -> Optional[str]:
+    """Extract real image URL from an img tag, skipping lazy-load placeholders"""
+    if not img_tag:
+        return None
+    for attr in ("src", "data-src", "data-lazy", "data-lazy-src", "data-srcset", "srcset"):
+        val = img_tag.get(attr, "")
+        if not val:
+            continue
+        url = val.split()[0].rstrip(",")
+        if url and not url.startswith("data:") and len(url) > 20:
+            return url
+    return None
+
+
 def warmup_gamestop():
     """
     Warmup function to solve Cloudflare CAPTCHA manually.
@@ -38,7 +52,7 @@ def warmup_gamestop():
     logger.info("Starting GameStop warmup with undetected-chromedriver...")
 
     # Use stealth browser instead of Playwright
-    browser = StealthBrowser(headless=False, session_name="gamestop")
+    browser = StealthBrowser(headless=GAMESTOP_HEADLESS, session_name="gamestop")
 
     try:
         browser.start()
@@ -117,7 +131,7 @@ class GameStopScraper(BaseScraper):
         # Check if we need to create a new browser
         if self._stealth_browser is None:
             logger.info("Creating new GameStop stealth browser...")
-            self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
+            self._stealth_browser = StealthBrowser(headless=GAMESTOP_HEADLESS, session_name="gamestop")
             self._stealth_browser.start()
             self._restart_attempts = 0
             return self._stealth_browser
@@ -194,6 +208,14 @@ class GameStopScraper(BaseScraper):
                     browser.driver.get(page_url)
                     time.sleep(5)  # Wait for page load
 
+                    # Scroll to trigger lazy-loaded images
+                    browser.driver.execute_script("window.scrollTo(0, document.body.scrollHeight / 2)")
+                    time.sleep(1)
+                    browser.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+                    time.sleep(1)
+                    browser.driver.execute_script("window.scrollTo(0, 0)")
+                    time.sleep(1)
+
                     html = browser.driver.page_source
                     break  # Success, exit retry loop
 
@@ -223,19 +245,15 @@ class GameStopScraper(BaseScraper):
                 )
 
                 if is_cloudflare:
-                    logger.warning("Cloudflare challenge detected! Waiting for manual solve...")
-                    start_time = time.time()
-                    while time.time() - start_time < CAPTCHA_WAIT_TIMEOUT:
-                        time.sleep(5)
-                        html = browser.driver.page_source
-                        html_lower = html.lower()
-                        title = browser.driver.title.lower()
-                        if "just a moment" not in title and "cf-challenge" not in html_lower:
-                            logger.info("Cloudflare challenge solved!")
-                            break
-                    else:
+                    logger.warning("Cloudflare challenge detected!")
+                    solved = browser.resolve_captcha_interactively(page_url, CAPTCHA_WAIT_TIMEOUT)
+                    if not solved:
                         logger.error("Cloudflare challenge not solved in time")
                         return all_products
+                    # Re-navigate in headless mode with fresh cookies
+                    browser.driver.get(page_url)
+                    time.sleep(5)
+                    html = browser.driver.page_source
 
                 # Save debug screenshot
                 try:
@@ -371,10 +389,7 @@ class GameStopScraper(BaseScraper):
             in_stock = self._check_card_stock_status(card)
 
             # Get image
-            img = card.select_one("img")
-            image_url = None
-            if img:
-                image_url = img.get("src") or img.get("data-src") or img.get("data-lazy")
+            image_url = _extract_img_url(card.select_one("img"))
             if image_url and not image_url.startswith("http"):
                 image_url = f"{self.base_url}{image_url}"
 
@@ -437,10 +452,9 @@ class GameStopScraper(BaseScraper):
             parent = link.find_parent()
             for _ in range(5):
                 if parent:
-                    img = parent.select_one("img")
-                    if img:
-                        image_url = img.get("src") or img.get("data-src")
-                        if image_url and not image_url.startswith("http"):
+                    image_url = _extract_img_url(parent.select_one("img"))
+                    if image_url:
+                        if not image_url.startswith("http"):
                             image_url = f"{self.base_url}{image_url}"
                         break
                     parent = parent.find_parent()
@@ -474,28 +488,26 @@ class GameStopScraper(BaseScraper):
         """Check if a product card indicates in-stock status"""
         card_text = card.get_text().lower() if hasattr(card, "get_text") else str(card).lower()
 
-        out_of_stock_phrases = [
+        out_of_stock_phrases = {
             "sold out",
             "out of stock",
             "not available",
             "unavailable",
             "currently unavailable",
-        ]
+        }
 
-        for phrase in out_of_stock_phrases:
-            if phrase in card_text:
+        if any(phrase in card_text for phrase in out_of_stock_phrases):
             return False
 
-        in_stock_phrases = [
+        in_stock_phrases = {
             "add to cart",
             "add to bag",
             "available",
             "buy now",
             "in stock",
-        ]
+        }
 
-        for phrase in in_stock_phrases:
-            if phrase in card_text:
+        if any(phrase in card_text for phrase in in_stock_phrases):
             return True
 
         # Default: assume in stock if listed

scrapers/target.py

@@ -14,6 +14,50 @@ from src.browser import get_browser
 logger = logging.getLogger(__name__)
 
 
+def _find_image_in_json(data, depth=0) -> Optional[str]:
+    """Recursively search a JSON dict for a Target image URL"""
+    if depth > 6:
+        return None
+    if isinstance(data, str):
+        if 'scene7.com' in data and not data.endswith('/'):
+            return data
+        return None
+    if isinstance(data, dict):
+        # Check known Target image keys first
+        for key in ('primary_image_url', 'base_url', 'url', 'src'):
+            val = data.get(key)
+            if isinstance(val, str) and 'scene7.com' in val:
+                return val
+        # Recurse, prioritizing enrichment/images paths
+        for key in ('enrichment', 'images', 'image'):
+            if key in data:
+                result = _find_image_in_json(data[key], depth + 1)
+                if result:
+                    return result
+        for val in data.values():
+            result = _find_image_in_json(val, depth + 1)
+            if result:
+                return result
+    if isinstance(data, list) and data:
+        return _find_image_in_json(data[0], depth + 1)
+    return None
+
+
+def _extract_img_url(img_tag) -> Optional[str]:
+    """Extract real image URL from an img tag, skipping lazy-load placeholders"""
+    if not img_tag:
+        return None
+    for attr in ("src", "data-src", "data-lazy-src", "data-srcset", "srcset"):
+        val = img_tag.get(attr, "")
+        if not val:
+            continue
+        # srcset may contain multiple URLs - take the first
+        url = val.split()[0].rstrip(",")
+        if url and not url.startswith("data:") and len(url) > 20:
+            return url
+    return None
+
+
 class TargetScraper(BaseScraper):
     """Scraper for Target.com"""
 
@@ -178,9 +222,8 @@ class TargetScraper(BaseScraper):
             if fulfillment:
                 in_stock = fulfillment.get("is_out_of_stock_in_all_store_locations", True) is False
 
-            # Get image
-            images = data.get("images", [])
-            image_url = images[0].get("base_url") if images else None
+            # Get image - search all known Target JSON paths
+            image_url = _find_image_in_json(data)
 
             return Product(
                 name=name,
@@ -247,11 +290,16 @@ class TargetScraper(BaseScraper):
                         break
                     parent = parent.find_parent()
 
-            # Try to find image
-            image_url = None
-            img = link.select_one("img")
-            if img:
-                image_url = img.get("src") or img.get("data-src")
+            # Try to find image - check inside link first, then walk up to parents
+            image_url = _extract_img_url(link.select_one("img"))
+            if not image_url:
+                parent = link.find_parent()
+                for _ in range(4):
+                    if parent:
+                        image_url = _extract_img_url(parent.select_one("img"))
+                        if image_url:
+                            break
+                        parent = parent.find_parent()
 
             # Check stock (assume in stock unless we see otherwise)
             in_stock = True
@@ -307,8 +355,7 @@ class TargetScraper(BaseScraper):
             in_stock = not out_of_stock
 
             # Get image
-            img = card.select_one("img")
-            image_url = img.get("src") if img else None
+            image_url = _extract_img_url(card.select_one("img"))
 
             # Extract product ID from URL
             product_id = ""

src/scraper_state.py

@@ -68,16 +68,20 @@ class ScraperStateManager:
         return DEFAULT_STATE.copy()
 
     def _save_state(self):
-        """Save state to file"""
+        """Save state to file using atomic write to prevent corruption"""
         with self._lock:
             try:
-                with open(STATE_FILE, 'w') as f:
+                tmp_file = STATE_FILE.with_suffix('.json.tmp')
+                with open(tmp_file, 'w') as f:
                     json.dump(self.state, f, indent=2, default=str)
+                os.replace(tmp_file, STATE_FILE)
             except Exception as e:
                 print(f"Error saving scraper state: {e}")
 
     def get_state(self) -> dict:
-        """Get current state"""
+        """Get current state, reloading from file to pick up changes from other processes"""
+        self.state = self._load_state()
+
         # Check if monitor is still running
         if self.state["monitor_pid"]:
             try:

tools/stealth_browser.py

@@ -117,6 +117,10 @@ class StealthBrowser:
         options.add_argument("--no-sandbox")
         options.add_argument("--disable-infobars")
 
+        # Headless mode (must be set via options in uc >= 3.5)
+        if self.headless:
+            options.add_argument("--headless=new")
+
         # Window size (realistic resolution)
         options.add_argument("--window-size=1920,1080")
 
@@ -130,10 +134,6 @@ class StealthBrowser:
             if proxy_str:
                 options.add_argument(f"--proxy-server={proxy_str}")
 
-        # Headless mode (note: more detectable)
-        if self.headless:
-            options.add_argument("--headless=new")
-
         return options
 
     def _format_proxy(self, proxy: Dict) -> Optional[str]:
@@ -164,6 +164,7 @@ class StealthBrowser:
             # This prevents "ChromeDriver only supports Chrome version X" errors
             self.driver = uc.Chrome(
                 options=options,
+                headless=self.headless,
                 use_subprocess=True,
                 version_main=146,  # Match user's Chrome version
             )
@@ -207,7 +208,13 @@ class StealthBrowser:
             return True
         except Exception as e:
             error_msg = str(e).lower()
-            if "invalid session id" in error_msg or "session deleted" in error_msg or "no such session" in error_msg:
+            dead_session_errors = [
+                "invalid session id", "session deleted", "no such session",
+                "newconnectionerror", "connection refused", "winerror 10061",
+                "failed to establish a new connection", "max retries exceeded",
+                "browser has closed",
+            ]
+            if any(err in error_msg for err in dead_session_errors):
                 logger.warning(f"Browser session invalid: {e}")
                 return False
             # Other errors might be temporary
@@ -430,6 +437,64 @@ class StealthBrowser:
 
         return False
 
+    def resolve_captcha_interactively(self, url: str, timeout: int = 120) -> bool:
+        """
+        When running headless and a CAPTCHA is detected, temporarily pop a visible browser
+        window for the user to solve it, then return to headless mode with the updated cookies.
+        """
+        if not self.headless:
+            # Already visible - just wait in place
+            return self.wait_for_captcha_solve(timeout)
+
+        logger.info("CAPTCHA detected in headless mode - switching to visible browser for manual solve...")
+        print("\n" + "=" * 60)
+        print("CAPTCHA DETECTED - Opening visible browser window")
+        print("Please solve the challenge, then it will return to headless.")
+        print("=" * 60 + "\n")
+
+        # Save current cookies before restarting
+        self._save_cookies()
+
+        # Stop headless driver
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception:
+                pass
+            self.driver = None
+            self._setup_complete = False
+
+        # Restart visible
+        self.headless = False
+        solved = False
+        try:
+            self.start()
+            self.driver.get(url)
+            time.sleep(3)
+            solved = self.wait_for_captcha_solve(timeout)
+            if solved:
+                self._save_cookies()
+                logger.info("CAPTCHA solved - saving session and returning to headless mode")
+                print("\n>>> CAPTCHA solved! Returning to headless mode...\n")
+        except Exception as e:
+            logger.error(f"Error during interactive CAPTCHA solve: {e}")
+        finally:
+            # Always return to headless
+            if self.driver:
+                try:
+                    self.driver.quit()
+                except Exception:
+                    pass
+                self.driver = None
+                self._setup_complete = False
+            self.headless = True
+            try:
+                self.start()
+            except Exception as e:
+                logger.error(f"Failed to restart headless browser after CAPTCHA solve: {e}")
+
+        return solved
+
     def wait_for_captcha_solve(self, timeout: int = 120):
         """
         Wait for user to solve CAPTCHA manually.