feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
+206
View File
@@ -0,0 +1,206 @@
// API Interceptor - Captures Pokemon Center API responses
// Injected into the page context to intercept fetch/XHR
(function() {
'use strict';
// Avoid double injection
if (window.__pokemonMonitorInjected) return;
window.__pokemonMonitorInjected = true;
const API_PATTERNS = [
'/tpci-ecommweb-api/product/',
'/site/resourceapi/category/',
'/tpci-ecommweb-api/review/',
'/graphql',
];
function isApiUrl(url) {
return API_PATTERNS.some(pattern => url.includes(pattern));
}
function extractSkusFromData(data, skus = new Set()) {
if (!data) return skus;
if (typeof data === 'object') {
// Look for SKU-like fields
for (const key of ['sku', 'skuCode', 'productId', 'id', 'code']) {
if (data[key] && typeof data[key] === 'string') {
const value = data[key];
// Pokemon Center SKUs: 699-17113, 191-85953, etc.
if (/^\d{1,3}-\d{4,6}$/.test(value) || /^\d{5,}$/.test(value)) {
skus.add(value);
}
}
}
// Recurse into nested objects/arrays
for (const value of Object.values(data)) {
if (typeof value === 'object') {
extractSkusFromData(value, skus);
}
}
}
if (Array.isArray(data)) {
for (const item of data) {
extractSkusFromData(item, skus);
}
}
return skus;
}
function sendToExtension(type, data) {
window.postMessage({
source: 'pokemon-api-interceptor',
type: type,
data: data
}, '*');
}
// Intercept fetch
const originalFetch = window.fetch;
window.fetch = async function(...args) {
const response = await originalFetch.apply(this, args);
try {
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
if (isApiUrl(url) && response.ok) {
// Clone response so we can read it without consuming
const clone = response.clone();
const contentType = clone.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
clone.json().then(data => {
const skus = extractSkusFromData(data);
sendToExtension('api-response', {
url: url,
timestamp: Date.now(),
skuCount: skus.size,
skus: Array.from(skus),
preview: JSON.stringify(data).slice(0, 500)
});
}).catch(() => {});
}
}
} catch (e) {
// Silently ignore errors
}
return response;
};
// Intercept XMLHttpRequest
const originalXhrOpen = XMLHttpRequest.prototype.open;
const originalXhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(method, url, ...rest) {
this._monitorUrl = url;
return originalXhrOpen.apply(this, [method, url, ...rest]);
};
XMLHttpRequest.prototype.send = function(...args) {
const xhr = this;
const url = xhr._monitorUrl || '';
if (isApiUrl(url)) {
xhr.addEventListener('load', function() {
try {
if (xhr.status === 200) {
const contentType = xhr.getResponseHeader('content-type') || '';
if (contentType.includes('application/json')) {
const data = JSON.parse(xhr.responseText);
const skus = extractSkusFromData(data);
sendToExtension('api-response', {
url: url,
timestamp: Date.now(),
skuCount: skus.size,
skus: Array.from(skus),
preview: JSON.stringify(data).slice(0, 500)
});
}
}
} catch (e) {
// Silently ignore
}
});
}
return originalXhrSend.apply(this, args);
};
console.log('[Pokemon Monitor] API interceptor active');
// Also scan embedded page data for SKUs (handles server-side rendered pages)
function scanPageForSkus() {
const skus = new Set();
// Method 1: Extract from product URLs in the page
const productLinks = document.querySelectorAll('a[href*="/product/"]');
productLinks.forEach(link => {
const match = link.href.match(/\/product\/(\d{1,3}-\d{4,6})/);
if (match) {
skus.add(match[1]);
}
});
// Method 2: Look for __NEXT_DATA__ (Next.js embedded data)
const nextDataScript = document.getElementById('__NEXT_DATA__');
if (nextDataScript) {
try {
const data = JSON.parse(nextDataScript.textContent);
extractSkusFromData(data, skus);
} catch (e) {}
}
// Method 3: Look for any script tags with JSON containing product data
document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]').forEach(script => {
try {
const data = JSON.parse(script.textContent);
extractSkusFromData(data, skus);
} catch (e) {}
});
// Method 4: Look for data attributes on product elements
document.querySelectorAll('[data-sku], [data-product-id], [data-product-sku]').forEach(el => {
const sku = el.dataset.sku || el.dataset.productId || el.dataset.productSku;
if (sku && /^\d{1,3}-\d{4,6}$/.test(sku)) {
skus.add(sku);
}
});
if (skus.size > 0) {
console.log(`[Pokemon Monitor] Found ${skus.size} SKUs in page`);
sendToExtension('api-response', {
url: window.location.href,
timestamp: Date.now(),
skuCount: skus.size,
skus: Array.from(skus),
source: 'page-scan'
});
}
}
// Scan page after it loads
if (document.readyState === 'complete') {
setTimeout(scanPageForSkus, 1000);
} else {
window.addEventListener('load', () => setTimeout(scanPageForSkus, 1000));
}
// Re-scan when page content changes (for infinite scroll, etc.)
let scanTimeout;
const observer = new MutationObserver(() => {
clearTimeout(scanTimeout);
scanTimeout = setTimeout(scanPageForSkus, 2000);
});
// Start observing after initial load
setTimeout(() => {
observer.observe(document.body, { childList: true, subtree: true });
}, 3000);
})();
+231 -4
View File
@@ -9,7 +9,9 @@ const DEFAULT_CONFIG = {
],
keywords: [], // Empty = all products, or ["chaos rising", "booster"] etc
notifyNewProducts: true,
notifyRestocks: true
notifyRestocks: true,
dashboardUrl: "http://localhost:5000", // Local dashboard URL
syncToDashboard: true // Enable syncing to local dashboard
};
// Store known products
@@ -20,11 +22,21 @@ let checkLoopRunning = false; // Prevent multiple loops
let isChecking = false; // Prevent overlapping checks
let lastNotificationTime = 0; // Rate limiting for Discord
// API monitoring - track SKUs seen from backend API calls
let knownSkus = new Set();
let apiStats = {
lastApiResponse: null,
totalSkusTracked: 0,
newSkusDetected: 0,
apiResponseCount: 0
};
// Initialize
chrome.runtime.onInstalled.addListener(() => {
console.log("Pokemon Stock Monitor installed");
loadConfig().then(() => {
loadProducts();
loadKnownSkus();
startCheckLoop();
});
});
@@ -33,6 +45,7 @@ chrome.runtime.onInstalled.addListener(() => {
chrome.runtime.onStartup.addListener(() => {
loadConfig().then(() => {
loadProducts();
loadKnownSkus();
startCheckLoop();
});
});
@@ -108,6 +121,97 @@ async function saveProducts() {
await chrome.storage.local.set({ knownProducts });
}
// Load known SKUs from storage (for API monitoring)
async function loadKnownSkus() {
const stored = await chrome.storage.local.get("knownSkus");
if (stored.knownSkus) {
knownSkus = new Set(stored.knownSkus);
}
console.log(`[API Monitor] Loaded ${knownSkus.size} known SKUs`);
}
// Save known SKUs to storage
async function saveKnownSkus() {
await chrome.storage.local.set({ knownSkus: Array.from(knownSkus) });
}
// Handle API data from content script interceptor
async function handleApiData(data) {
apiStats.apiResponseCount++;
apiStats.lastApiResponse = new Date().toISOString();
if (!data.skus || data.skus.length === 0) return;
const newSkus = [];
for (const sku of data.skus) {
if (!knownSkus.has(sku)) {
knownSkus.add(sku);
newSkus.push(sku);
apiStats.newSkusDetected++;
}
}
apiStats.totalSkusTracked = knownSkus.size;
if (newSkus.length > 0) {
console.log(`[API Monitor] NEW SKUs detected: ${newSkus.join(', ')}`);
// Send Discord alert for new SKUs
if (config.discordWebhook && config.notifyNewProducts) {
for (const sku of newSkus.slice(0, 3)) { // Limit to 3 to avoid spam
await sendSkuAlert(sku, data.url);
await new Promise(r => setTimeout(r, 1000));
}
}
await saveKnownSkus();
}
}
// Send alert for new SKU detected via API
async function sendSkuAlert(sku, sourceUrl) {
const productUrl = `https://www.pokemoncenter.com/product/${sku}`;
const embed = {
title: "NEW SKU DETECTED (API)",
description: `**SKU: ${sku}**\n\nDetected in backend API before public listing!`,
url: productUrl,
color: 0xFF00FF, // Magenta for API detections
fields: [
{ name: "SKU", value: sku, inline: true },
{ name: "Source", value: "API Intercept", inline: true },
{ name: "Link", value: `[VIEW PRODUCT](${productUrl})`, inline: false }
],
footer: { text: "Pokemon Monitor - API Detection" },
timestamp: new Date().toISOString()
};
try {
const response = await fetch(config.discordWebhook, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
content: "@everyone NEW SKU FROM API!",
embeds: [embed]
})
});
if (response.ok) {
console.log(`[API Monitor] Discord alert sent for SKU: ${sku}`);
chrome.notifications.create({
type: "basic",
iconUrl: "icon128.png",
title: "NEW SKU DETECTED!",
message: `SKU: ${sku} - Check Discord!`
});
}
} catch (error) {
console.error("[API Monitor] Error sending alert:", error);
}
}
// Main stock check function
async function runStockCheck() {
if (!config.enabled) {
@@ -287,11 +391,55 @@ function processProducts(products) {
lastSeen: now,
lastInStock: product.inStock ? now : null
};
// Queue new_drop event for dashboard sync
pendingEvents.push({
type: 'new_drop',
url: product.url,
name: product.name,
price: product.price,
inStock: product.inStock,
timestamp: now
});
} else {
// Existing product - check for restock
if (product.inStock && !existing.inStock) {
restockedProducts.push(product);
existing.lastInStock = now;
// Queue restock event for dashboard sync
pendingEvents.push({
type: 'restock',
url: product.url,
name: product.name,
price: product.price,
previousPrice: existing.price,
timestamp: now
});
}
// Track when items go OUT of stock (for selling rate calculation)
if (!product.inStock && existing.inStock) {
pendingEvents.push({
type: 'out_of_stock',
url: product.url,
name: product.name,
price: product.price,
lastInStock: existing.lastInStock, // When it came in stock
timestamp: now // When it sold out
});
}
// Track price changes
if (product.price && existing.price && product.price !== existing.price) {
pendingEvents.push({
type: 'price_change',
url: product.url,
name: product.name,
oldPrice: existing.price,
newPrice: product.price,
timestamp: now
});
}
// Update
@@ -393,7 +541,69 @@ async function sendDiscordNotification(product, alertType) {
}
}
// Listen for messages from popup
// Track pending events to sync (restocks, new drops)
let pendingEvents = [];
// Sync data to local dashboard
async function syncToDashboard() {
if (!config.syncToDashboard || !config.dashboardUrl) {
return { success: false, reason: 'sync disabled' };
}
// Build products array with all tracking data
const productsToSync = Object.values(knownProducts).map(p => ({
...p,
site: 'pokemoncenter',
sku: p.productId || extractSkuFromUrl(p.url)
}));
const syncData = {
skus: Array.from(knownSkus),
products: productsToSync,
apiStats: apiStats,
events: pendingEvents // Include pending events (restocks, new drops)
};
try {
const response = await fetch(`${config.dashboardUrl}/api/extension/sync`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(syncData)
});
if (response.ok) {
const result = await response.json();
console.log(`[Dashboard Sync] Success - ${result.total_skus} SKUs, ${result.total_products} products`);
// Clear pending events after successful sync
pendingEvents = [];
return { success: true, ...result };
} else {
console.warn(`[Dashboard Sync] Failed: ${response.status}`);
return { success: false, status: response.status };
}
} catch (error) {
// Dashboard might not be running - silently fail
console.debug(`[Dashboard Sync] Dashboard not available: ${error.message}`);
return { success: false, error: error.message };
}
}
// Helper to extract SKU from Pokemon Center URL
function extractSkuFromUrl(url) {
if (!url) return null;
const match = url.match(/\/product\/([^\/\?]+)/);
return match ? match[1] : null;
}
// Auto-sync to dashboard periodically (every 5 minutes)
setInterval(() => {
if (config.syncToDashboard) {
syncToDashboard();
}
}, 5 * 60 * 1000);
// Listen for messages from popup and content scripts
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
if (message.type === "getConfig") {
sendResponse(config);
@@ -407,12 +617,27 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
} else if (message.type === "getStats") {
sendResponse({
totalProducts: Object.keys(knownProducts).length,
enabled: config.enabled
enabled: config.enabled,
apiStats: apiStats,
knownSkuCount: knownSkus.size
});
} else if (message.type === "clearProducts") {
knownProducts = {};
saveProducts();
sendResponse({ success: true });
} else if (message.type === "clearSkus") {
knownSkus = new Set();
apiStats.totalSkusTracked = 0;
saveKnownSkus();
sendResponse({ success: true });
} else if (message.type === "apiData") {
// Handle API data from content script interceptor
handleApiData(message.data);
sendResponse({ success: true });
} else if (message.type === "syncToDashboard") {
// Manual sync to local dashboard
syncToDashboard().then(result => sendResponse(result));
return true; // Keep channel open for async response
}
return true;
});
@@ -420,7 +645,9 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
// Run initial check after a short delay
setTimeout(() => {
loadConfig().then(() => {
loadProducts().then(() => {
loadProducts();
loadKnownSkus().then(() => {
console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center");
if (config.enabled) {
console.log(`Check interval: ${config.checkIntervalSeconds} seconds`);
runStockCheck();
+24 -1
View File
@@ -1,9 +1,32 @@
// Pokemon Stock Monitor - Content Script
// Runs on PokemonCenter pages to extract product data
// Runs on PokemonCenter pages to extract product data and intercept APIs
(function() {
console.log("[Pokemon Monitor] Content script loaded on:", window.location.href);
// Inject the API interceptor into the page context
function injectApiInterceptor() {
const script = document.createElement('script');
script.src = chrome.runtime.getURL('api-interceptor.js');
script.onload = () => script.remove();
(document.head || document.documentElement).appendChild(script);
}
// Listen for messages from the injected interceptor
window.addEventListener('message', (event) => {
if (event.source !== window) return;
if (event.data?.source !== 'pokemon-api-interceptor') return;
// Forward API data to background script
chrome.runtime.sendMessage({
type: 'apiData',
data: event.data.data
}).catch(() => {});
});
// Inject the interceptor
injectApiInterceptor();
// Listen for messages from background script
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
if (message.type === "extractProducts") {
+9 -3
View File
@@ -1,8 +1,8 @@
{
"manifest_version": 3,
"name": "Pokemon Stock Monitor",
"version": "1.0.1",
"description": "Monitors PokemonCenter for restocks and new drops, sends Discord notifications",
"version": "1.1.0",
"description": "Monitors PokemonCenter for restocks and new drops via API interception, sends Discord notifications",
"permissions": [
"alarms",
"storage",
@@ -24,7 +24,13 @@
{
"matches": ["https://www.pokemoncenter.com/*"],
"js": ["content.js"],
"run_at": "document_idle"
"run_at": "document_start"
}
],
"web_accessible_resources": [
{
"resources": ["api-interceptor.js"],
"matches": ["https://www.pokemoncenter.com/*"]
}
]
}
+14
View File
@@ -242,10 +242,24 @@
<span class="slider"></span>
</label>
</div>
<div class="toggle-row">
<span>Sync to Dashboard</span>
<label class="toggle" for="syncDashboard">
<input type="checkbox" id="syncDashboard" checked>
<span class="slider"></span>
</label>
</div>
</div>
<div class="section">
<label>Dashboard URL</label>
<input type="text" id="dashboardUrl" placeholder="http://localhost:5000">
<small>Local dashboard for viewing data</small>
</div>
<button class="btn-primary" id="saveBtn">Save Settings</button>
<button class="btn-secondary" id="checkNowBtn">Check Now</button>
<button class="btn-secondary" id="syncBtn">Sync to Dashboard</button>
<button class="btn-danger" id="clearBtn">Clear Product History</button>
<div class="saved-msg" id="savedMsg">Settings saved!</div>
+15 -1
View File
@@ -13,6 +13,8 @@ document.addEventListener("DOMContentLoaded", async () => {
document.getElementById("enabled").checked = config.enabled !== false;
document.getElementById("notifyNew").checked = config.notifyNewProducts !== false;
document.getElementById("notifyRestock").checked = config.notifyRestocks !== false;
document.getElementById("syncDashboard").checked = config.syncToDashboard !== false;
document.getElementById("dashboardUrl").value = config.dashboardUrl || "http://localhost:5000";
// Update stats
document.getElementById("productCount").textContent = stats.totalProducts || 0;
@@ -30,7 +32,9 @@ document.addEventListener("DOMContentLoaded", async () => {
checkIntervalSeconds: parseInt(document.getElementById("interval").value) || 15,
enabled: document.getElementById("enabled").checked,
notifyNewProducts: document.getElementById("notifyNew").checked,
notifyRestocks: document.getElementById("notifyRestock").checked
notifyRestocks: document.getElementById("notifyRestock").checked,
syncToDashboard: document.getElementById("syncDashboard").checked,
dashboardUrl: document.getElementById("dashboardUrl").value.trim() || "http://localhost:5000"
};
await chrome.runtime.sendMessage({ type: "saveConfig", config: newConfig });
@@ -52,6 +56,16 @@ document.addEventListener("DOMContentLoaded", async () => {
showSaved("History cleared!");
}
});
// Sync to Dashboard button
document.getElementById("syncBtn").addEventListener("click", async () => {
const result = await chrome.runtime.sendMessage({ type: "syncToDashboard" });
if (result.success) {
showSaved(`Synced! ${result.total_skus || 0} SKUs, ${result.total_products || 0} products`);
} else {
showSaved("Sync failed - is dashboard running?");
}
});
});
function updateStatus(enabled) {