mmcghen/pokemon-stock-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanups for scrapers

Browse Source
This commit is contained in:
Michael McGhen
2026-04-10 10:04:20 -04:00
parent 0d5cc08bc0
commit 052a762182
15 changed files with 1411 additions and 567 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chrome-extension/background.js
+347 -152
View File
@@ -18,9 +18,11 @@ const DEFAULT_CONFIG = {
let knownProducts = {};
let config = DEFAULT_CONFIG;
let persistentTabId = null; // Keep tab open for faster refreshes
let checkLoopRunning = false; // Prevent multiple loops
let isChecking = false; // Prevent overlapping checks
let lastNotificationTime = 0; // Rate limiting for Discord
let botBackoffUntil = 0; // Timestamp (ms) until which checks are paused after bot detection
let consecutiveBotDetections = 0; // Track consecutive failures for exponential backoff
let captchaTabId = null; // Tab currently showing a captcha — kept open for user to solve
// API monitoring - track SKUs seen from backend API calls
let knownSkus = new Set();
@@ -34,6 +36,7 @@ let apiStats = {
// Initialize
chrome.runtime.onInstalled.addListener(() => {
console.log("Pokemon Stock Monitor installed");
chrome.storage.local.remove(["persistentTabId", "captchaTabId"]);
loadConfig().then(() => {
loadProducts();
loadKnownSkus();
@@ -42,51 +45,60 @@ chrome.runtime.onInstalled.addListener(() => {
});
// Load on startup
chrome.runtime.onStartup.addListener(() => {
loadConfig().then(() => {
loadProducts();
loadKnownSkus();
startCheckLoop();
});
chrome.runtime.onStartup.addListener(async () => {
await loadConfig();
await loadPersistentTabId(); // must complete before any check runs
loadProducts();
loadKnownSkus();
startCheckLoop();
});
// Start the check loop (uses setTimeout to bypass Chrome's 30-sec alarm minimum)
const ALARM_NAME = "stockCheck";
// Start the check loop using chrome.alarms so it survives service worker restarts
// Adds ±15s jitter to the first delay to avoid predictable bot-like cadence
function startCheckLoop() {
if (checkLoopRunning) {
console.log("Check loop already running");
return;
}
checkLoopRunning = true;
console.log(`Starting check loop: every ${config.checkIntervalSeconds} seconds`);
scheduleNextCheck();
const intervalMinutes = Math.max(1, config.checkIntervalSeconds / 60);
const jitterMinutes = (Math.random() - 0.5) * 0.5; // ±15 seconds
const delayMinutes = Math.max(0.1, intervalMinutes + jitterMinutes);
chrome.alarms.get(ALARM_NAME, (existing) => {
if (!existing) {
chrome.alarms.create(ALARM_NAME, { delayInMinutes: delayMinutes, periodInMinutes: intervalMinutes });
console.log(`Check alarm created: every ${intervalMinutes} min (first in ~${delayMinutes.toFixed(2)} min)`);
}
});
}
function scheduleNextCheck() {
if (!config.enabled || !checkLoopRunning) {
checkLoopRunning = false;
console.log("Check loop stopped");
return;
}
function stopCheckLoop() {
chrome.alarms.clear(ALARM_NAME);
console.log("Check alarm cleared");
}
const intervalMs = Math.max(60, config.checkIntervalSeconds) * 1000; // Min 60 seconds to avoid bot detection
setTimeout(async () => {
// Handle alarms
chrome.alarms.onAlarm.addListener(async (alarm) => {
if (alarm.name === ALARM_NAME) {
if (!isChecking && config.enabled) {
// Skip check if we're still in bot-detection backoff window
if (Date.now() < botBackoffUntil) {
const remainingSec = Math.round((botBackoffUntil - Date.now()) / 1000);
console.log(`[Backoff] Skipping check — bot backoff active for ${remainingSec}s more`);
return;
}
isChecking = true;
try {
// Service worker may have been killed and restarted since last check —
// restore state from storage before doing anything with tabs or config.
await loadConfig();
await loadPersistentTabId();
await loadBackoffState();
await loadCaptchaTabState();
await runStockCheck();
} finally {
isChecking = false;
}
}
scheduleNextCheck();
}, intervalMs);
}
function stopCheckLoop() {
checkLoopRunning = false;
console.log("Check loop stopped");
}
}
});
// Load config from storage
async function loadConfig() {
@@ -100,7 +112,7 @@ async function loadConfig() {
// Save config to storage
async function saveConfig() {
await chrome.storage.local.set({ config });
// Restart loop with new interval
// Recreate alarm with updated interval
stopCheckLoop();
if (config.enabled) {
startCheckLoop();
@@ -121,6 +133,114 @@ async function saveProducts() {
await chrome.storage.local.set({ knownProducts });
}
// Load/save persistent tab ID from storage so it survives service worker restarts
async function loadPersistentTabId() {
const stored = await chrome.storage.local.get("persistentTabId");
if (stored.persistentTabId) {
try {
await chrome.tabs.get(stored.persistentTabId);
persistentTabId = stored.persistentTabId;
console.log(`Restored persistent tab ${persistentTabId}`);
} catch (e) {
persistentTabId = null;
await chrome.storage.local.remove("persistentTabId");
}
}
}
async function savePersistentTabId(tabId) {
persistentTabId = tabId;
await chrome.storage.local.set({ persistentTabId: tabId });
}
async function clearPersistentTab() {
const idToRemove = persistentTabId;
// Null out state immediately so a concurrent call doesn't double-remove
persistentTabId = null;
await chrome.storage.local.remove("persistentTabId");
if (idToRemove) {
try {
await chrome.tabs.remove(idToRemove);
console.log(`Closed persistent tab ${idToRemove}`);
} catch (e) {
// Tab was already closed externally — that's fine
console.log(`Tab ${idToRemove} already gone: ${e.message}`);
}
}
}
// Backoff state survives service worker restarts via storage
async function loadBackoffState() {
const stored = await chrome.storage.local.get(["botBackoffUntil", "consecutiveBotDetections"]);
botBackoffUntil = stored.botBackoffUntil || 0;
consecutiveBotDetections = stored.consecutiveBotDetections || 0;
}
async function saveBackoffState() {
await chrome.storage.local.set({ botBackoffUntil, consecutiveBotDetections });
}
async function applyBotBackoff() {
consecutiveBotDetections++;
// Exponential backoff: 2^n minutes, capped at 30 minutes, with ±30s jitter
const backoffMinutes = Math.min(30, Math.pow(2, consecutiveBotDetections));
const jitterMs = (Math.random() - 0.5) * 60000; // ±30 seconds
botBackoffUntil = Date.now() + backoffMinutes * 60000 + jitterMs;
await saveBackoffState();
console.log(`[Backoff] Bot detected (${consecutiveBotDetections}x) — pausing checks for ~${backoffMinutes} min`);
}
function resetBotBackoff() {
if (consecutiveBotDetections > 0) {
console.log("[Backoff] Successful check — resetting bot detection counter");
consecutiveBotDetections = 0;
botBackoffUntil = 0;
chrome.storage.local.remove(["botBackoffUntil", "consecutiveBotDetections"]);
}
}
// --- Captcha tab tracking ---
// Persists the tab ID of the open captcha page across service worker restarts.
async function loadCaptchaTabState() {
const stored = await chrome.storage.local.get("captchaTabId");
if (stored.captchaTabId) {
try {
await chrome.tabs.get(stored.captchaTabId);
captchaTabId = stored.captchaTabId;
console.log(`[Captcha] Restored captcha tab ${captchaTabId}`);
} catch (e) {
captchaTabId = null;
await chrome.storage.local.remove("captchaTabId");
}
}
}
async function saveCaptchaTabId(tabId) {
captchaTabId = tabId;
if (tabId) {
await chrome.storage.local.set({ captchaTabId: tabId });
} else {
await chrome.storage.local.remove("captchaTabId");
}
}
// Returns 'solved' | 'still_captcha' | 'gone'
async function checkCaptchaTabStatus() {
if (!captchaTabId) return 'gone';
try {
const tab = await chrome.tabs.get(captchaTabId);
// If still loading or on a known captcha URL, not solved yet
if (!tab.url || tab.status === 'loading' || isCaptchaUrl(tab.url)) {
return 'still_captcha';
}
// Tab is back on pokemoncenter — user likely solved it
return 'solved';
} catch (e) {
return 'gone'; // Tab was closed
}
}
// Load known SKUs from storage (for API monitoring)
async function loadKnownSkus() {
const stored = await chrome.storage.local.get("knownSkus");
@@ -219,20 +339,60 @@ async function runStockCheck() {
return;
}
// If we have a captcha tab open, check whether the user has solved it
// before opening any new tabs or sending more warnings.
if (captchaTabId) {
const status = await checkCaptchaTabStatus();
if (status === 'still_captcha') {
console.log(`[Captcha] Tab ${captchaTabId} still showing captcha — waiting for user`);
await applyBotBackoff(); // Re-arm backoff silently; no Discord ping
return;
}
if (status === 'solved') {
console.log(`[Captcha] Tab ${captchaTabId} looks resolved — resuming normal checks`);
// Reclaim the solved tab as the persistent tab so we don't open a new one
await savePersistentTabId(captchaTabId);
await saveCaptchaTabId(null);
resetBotBackoff();
} else { // 'gone' — user closed the tab themselves
console.log('[Captcha] Captcha tab was closed — resuming normal checks');
await saveCaptchaTabId(null);
resetBotBackoff();
}
}
console.log("Running stock check...");
for (const url of config.urls) {
try {
const products = await fetchAndParseProducts(url);
// Alert if 0 products found - likely bot protection
// Alert if 0 products found — likely captcha or bot protection
if (products.length === 0) {
console.warn("WARNING: 0 products found - possible bot protection!");
await sendWarningNotification("Bot Protection Detected",
"Found 0 products - you may need to manually verify you're human on PokemonCenter.com");
console.warn("WARNING: 0 products found - possible bot protection or captcha!");
botProtectionDetected = true;
const isFirstDetection = consecutiveBotDetections === 0;
await applyBotBackoff();
if (isFirstDetection) {
// Only ping Discord on the first detection — not on every retry
const backoffMin = Math.round((botBackoffUntil - Date.now()) / 60000);
await sendWarningNotification("Bot Protection Detected",
`Found 0 products — captcha may be active. Pausing checks for ~${backoffMin} min. Check the open PokemonCenter tab and solve the captcha if prompted.`);
}
// Keep the tab open so the user can solve the captcha.
// If fetchAndParseProducts already saved it as captchaTabId (URL-detected),
// use that; otherwise promote the current persistentTabId so we track it.
if (!captchaTabId && persistentTabId) {
await saveCaptchaTabId(persistentTabId);
}
persistentTabId = null;
await chrome.storage.local.remove("persistentTabId");
continue;
}
// Successful product load — reset any backoff state
resetBotBackoff();
const { newProducts, restockedProducts } = processProducts(products);
// Send notifications with rate limiting
@@ -275,105 +435,135 @@ async function runStockCheck() {
}
}
// Detect captcha/challenge pages by URL patterns (all lowercase comparisons)
function isCaptchaUrl(url) {
if (!url) return false;
const lower = url.toLowerCase();
// Known bot-protection URL signatures
if (lower.includes('_incapsula_resource')) return true;
if (lower.includes('/cdn-cgi/challenge')) return true;
if (lower.includes('imperva')) return true;
if (lower.includes('incapsula')) return true;
if (lower.includes('distil')) return true;
// "challenge" or "verify" only flag if we've left the pokemoncenter domain entirely
if (!lower.includes('pokemoncenter.com')) return true;
return false;
}
// Fetch and parse products from a URL using content script
async function fetchAndParseProducts(url) {
return new Promise(async (resolve, reject) => {
try {
let tabId;
let isNewTab = false;
// Reuse existing tab or create new one
if (persistentTabId) {
try {
const tab = await chrome.tabs.get(persistentTabId);
tabId = tab.id;
console.log(`Reusing tab ${tabId}, refreshing...`);
await chrome.tabs.update(tabId, { url });
} catch (e) {
// Tab was closed, create new one
persistentTabId = null;
}
let tabId;
try {
// Reuse existing tab or create a new one
if (persistentTabId) {
try {
await chrome.tabs.get(persistentTabId);
tabId = persistentTabId;
console.log(`Reusing tab ${tabId}, navigating...`);
await chrome.tabs.update(tabId, { url });
} catch (e) {
// Tab is gone — clear it and fall through to create a new one
await clearPersistentTab();
tabId = null;
}
if (!persistentTabId) {
const tab = await chrome.tabs.create({ url, active: false });
tabId = tab.id;
persistentTabId = tabId;
isNewTab = true;
console.log(`Created new tab ${tabId}`);
}
// Wait for tab to finish loading
const waitForLoad = () => {
return new Promise((res) => {
const listener = (tid, changeInfo) => {
if (tid === tabId && changeInfo.status === "complete") {
chrome.tabs.onUpdated.removeListener(listener);
res();
}
};
chrome.tabs.onUpdated.addListener(listener);
setTimeout(() => {
chrome.tabs.onUpdated.removeListener(listener);
res();
}, 20000);
});
};
await waitForLoad();
console.log(`Tab ${tabId} loaded`);
// Smart wait: poll for products instead of fixed wait
let products = [];
const startTime = Date.now();
const maxWait = 8000; // Max 8 seconds
const pollInterval = 500; // Check every 500ms
while (Date.now() - startTime < maxWait) {
try {
// Inject content script if needed
try {
await chrome.scripting.executeScript({
target: { tabId },
files: ["content.js"]
});
} catch (e) { /* Already injected */ }
await new Promise(r => setTimeout(r, pollInterval));
const response = await chrome.tabs.sendMessage(tabId, { type: "extractProducts" });
products = response || [];
if (products.length > 0) {
console.log(`Found ${products.length} products after ${Date.now() - startTime}ms`);
break;
}
} catch (e) {
// Content script not ready yet
}
}
if (products.length === 0) {
console.log(`No products found after ${maxWait}ms`);
}
// Apply keyword filter
if (config.keywords && config.keywords.length > 0) {
products = products.filter(p => {
const nameLower = p.name.toLowerCase();
return config.keywords.some(kw => nameLower.includes(kw.toLowerCase()));
});
console.log(`After keyword filter: ${products.length} products`);
}
// Don't close tab - keep for reuse
resolve(products);
} catch (error) {
persistentTabId = null; // Reset on error
reject(error);
}
});
if (!tabId) {
const tab = await chrome.tabs.create({ url, active: false });
tabId = tab.id;
await savePersistentTabId(tabId);
console.log(`Created new tab ${tabId}`);
}
// Wait for tab to finish loading
await new Promise((res) => {
const listener = (tid, changeInfo) => {
if (tid === tabId && changeInfo.status === "complete") {
chrome.tabs.onUpdated.removeListener(listener);
res();
}
};
chrome.tabs.onUpdated.addListener(listener);
setTimeout(() => {
chrome.tabs.onUpdated.removeListener(listener);
res();
}, 20000);
});
// Check if the tab landed on a captcha/challenge page
try {
const tab = await chrome.tabs.get(tabId);
if (isCaptchaUrl(tab.url)) {
console.warn(`[Captcha] Tab ${tabId} landed on challenge page: ${tab.url}`);
// Keep tab open for user; track it so we can poll for resolution
await saveCaptchaTabId(tabId);
persistentTabId = null;
await chrome.storage.local.remove("persistentTabId");
return []; // Caller will handle backoff + single ping
}
} catch (e) { /* tab may have been closed */ }
console.log(`Tab ${tabId} loaded`);
// Poll for products — PokemonCenter is a React SPA so status:"complete" fires
// on the HTML shell before products are rendered via API calls.
// Inject content.js once, then poll via messaging.
let products = [];
const startTime = Date.now();
const maxWait = 25000;
const pollInterval = 1000; // Reduced polling frequency
let injected = false;
while (Date.now() - startTime < maxWait) {
try {
if (!injected) {
try {
await chrome.scripting.executeScript({ target: { tabId }, files: ["content.js"] });
injected = true;
} catch (e) {
injected = true; // Already injected — treat as done
}
}
await new Promise(r => setTimeout(r, pollInterval));
const response = await chrome.tabs.sendMessage(tabId, { type: "extractProducts" });
products = response || [];
if (products.length > 0) {
console.log(`Found ${products.length} products after ${Date.now() - startTime}ms`);
break;
}
} catch (e) {
// Content script not ready yet
}
}
if (products.length === 0) {
console.log(`No products found after ${maxWait}ms — assuming captcha/bot block`);
// Keep the tab open and track it so runStockCheck can poll for resolution
await saveCaptchaTabId(tabId);
persistentTabId = null;
await chrome.storage.local.remove("persistentTabId");
}
// Apply keyword filter
if (config.keywords && config.keywords.length > 0) {
products = products.filter(p =>
config.keywords.some(kw => p.name.toLowerCase().includes(kw.toLowerCase()))
);
console.log(`After keyword filter: ${products.length} products`);
}
// Keep tab open for reuse next check
return products;
} catch (error) {
// On any error, close the tab so we don't leave an untracked orphan
await clearPersistentTab();
tabId = null;
throw error;
}
}
// Note: Product parsing is now done by the content script (content.js)
@@ -548,6 +738,7 @@ async function sendDiscordNotification(product, alertType) {
// Track pending events to sync (restocks, new drops)
let pendingEvents = [];
let botProtectionDetected = false;
// Sync data to local dashboard
async function syncToDashboard() {
@@ -566,7 +757,8 @@ async function syncToDashboard() {
skus: Array.from(knownSkus),
products: productsToSync,
apiStats: apiStats,
events: pendingEvents // Include pending events (restocks, new drops)
events: pendingEvents,
bot_protection_detected: botProtectionDetected
};
try {
@@ -579,8 +771,9 @@ async function syncToDashboard() {
if (response.ok) {
const result = await response.json();
console.log(`[Dashboard Sync] Success - ${result.total_skus} SKUs, ${result.total_products} products`);
// Clear pending events after successful sync
// Clear pending events and flags after successful sync
pendingEvents = [];
botProtectionDetected = false;
return { success: true, ...result };
} else {
console.warn(`[Dashboard Sync] Failed: ${response.status}`);
@@ -600,12 +793,6 @@ function extractSkuFromUrl(url) {
return match ? match[1] : null;
}
// Auto-sync to dashboard periodically (every 5 minutes)
setInterval(() => {
if (config.syncToDashboard) {
syncToDashboard();
}
}, 5 * 60 * 1000);
// Listen for messages from popup and content scripts
@@ -648,16 +835,24 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
});
// Run initial check after a short delay
setTimeout(() => {
loadConfig().then(() => {
loadProducts();
loadKnownSkus().then(() => {
console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center");
if (config.enabled) {
console.log(`Check interval: ${config.checkIntervalSeconds} seconds`);
runStockCheck();
startCheckLoop();
}
});
});
setTimeout(async () => {
await loadConfig();
await loadPersistentTabId(); // must complete before any check runs
await loadBackoffState();
await loadCaptchaTabState();
loadProducts();
await loadKnownSkus();
console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center");
if (config.enabled) {
console.log(`Check interval: ${config.checkIntervalSeconds} seconds`);
startCheckLoop();
// Skip initial check if still in backoff window
if (Date.now() < botBackoffUntil) {
const remainingSec = Math.round((botBackoffUntil - Date.now()) / 1000);
console.log(`[Backoff] Startup — skipping initial check, backoff active for ${remainingSec}s more`);
} else if (!isChecking) {
isChecking = true;
runStockCheck().finally(() => { isChecking = false; });
}
}
}, 3000);
chrome-extension/content.js
+8 -4
View File
@@ -18,10 +18,14 @@
if (event.data?.source !== 'pokemon-api-interceptor') return;
// Forward API data to background script
chrome.runtime.sendMessage({
type: 'apiData',
data: event.data.data
}).catch(() => {});
try {
chrome.runtime.sendMessage({
type: 'apiData',
data: event.data.data
}).catch(() => {});
} catch (e) {
// Extension context invalidated (tab closing or extension reloaded)
}
});
// Inject the interceptor