feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -9,7 +9,9 @@ const DEFAULT_CONFIG = {
|
||||
],
|
||||
keywords: [], // Empty = all products, or ["chaos rising", "booster"] etc
|
||||
notifyNewProducts: true,
|
||||
notifyRestocks: true
|
||||
notifyRestocks: true,
|
||||
dashboardUrl: "http://localhost:5000", // Local dashboard URL
|
||||
syncToDashboard: true // Enable syncing to local dashboard
|
||||
};
|
||||
|
||||
// Store known products
|
||||
@@ -20,11 +22,21 @@ let checkLoopRunning = false; // Prevent multiple loops
|
||||
let isChecking = false; // Prevent overlapping checks
|
||||
let lastNotificationTime = 0; // Rate limiting for Discord
|
||||
|
||||
// API monitoring - track SKUs seen from backend API calls
|
||||
let knownSkus = new Set();
|
||||
let apiStats = {
|
||||
lastApiResponse: null,
|
||||
totalSkusTracked: 0,
|
||||
newSkusDetected: 0,
|
||||
apiResponseCount: 0
|
||||
};
|
||||
|
||||
// Initialize
|
||||
chrome.runtime.onInstalled.addListener(() => {
|
||||
console.log("Pokemon Stock Monitor installed");
|
||||
loadConfig().then(() => {
|
||||
loadProducts();
|
||||
loadKnownSkus();
|
||||
startCheckLoop();
|
||||
});
|
||||
});
|
||||
@@ -33,6 +45,7 @@ chrome.runtime.onInstalled.addListener(() => {
|
||||
chrome.runtime.onStartup.addListener(() => {
|
||||
loadConfig().then(() => {
|
||||
loadProducts();
|
||||
loadKnownSkus();
|
||||
startCheckLoop();
|
||||
});
|
||||
});
|
||||
@@ -108,6 +121,97 @@ async function saveProducts() {
|
||||
await chrome.storage.local.set({ knownProducts });
|
||||
}
|
||||
|
||||
// Load known SKUs from storage (for API monitoring)
|
||||
async function loadKnownSkus() {
|
||||
const stored = await chrome.storage.local.get("knownSkus");
|
||||
if (stored.knownSkus) {
|
||||
knownSkus = new Set(stored.knownSkus);
|
||||
}
|
||||
console.log(`[API Monitor] Loaded ${knownSkus.size} known SKUs`);
|
||||
}
|
||||
|
||||
// Save known SKUs to storage
|
||||
async function saveKnownSkus() {
|
||||
await chrome.storage.local.set({ knownSkus: Array.from(knownSkus) });
|
||||
}
|
||||
|
||||
// Handle API data from content script interceptor
|
||||
async function handleApiData(data) {
|
||||
apiStats.apiResponseCount++;
|
||||
apiStats.lastApiResponse = new Date().toISOString();
|
||||
|
||||
if (!data.skus || data.skus.length === 0) return;
|
||||
|
||||
const newSkus = [];
|
||||
|
||||
for (const sku of data.skus) {
|
||||
if (!knownSkus.has(sku)) {
|
||||
knownSkus.add(sku);
|
||||
newSkus.push(sku);
|
||||
apiStats.newSkusDetected++;
|
||||
}
|
||||
}
|
||||
|
||||
apiStats.totalSkusTracked = knownSkus.size;
|
||||
|
||||
if (newSkus.length > 0) {
|
||||
console.log(`[API Monitor] NEW SKUs detected: ${newSkus.join(', ')}`);
|
||||
|
||||
// Send Discord alert for new SKUs
|
||||
if (config.discordWebhook && config.notifyNewProducts) {
|
||||
for (const sku of newSkus.slice(0, 3)) { // Limit to 3 to avoid spam
|
||||
await sendSkuAlert(sku, data.url);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
await saveKnownSkus();
|
||||
}
|
||||
}
|
||||
|
||||
// Send alert for new SKU detected via API
|
||||
async function sendSkuAlert(sku, sourceUrl) {
|
||||
const productUrl = `https://www.pokemoncenter.com/product/${sku}`;
|
||||
|
||||
const embed = {
|
||||
title: "NEW SKU DETECTED (API)",
|
||||
description: `**SKU: ${sku}**\n\nDetected in backend API before public listing!`,
|
||||
url: productUrl,
|
||||
color: 0xFF00FF, // Magenta for API detections
|
||||
fields: [
|
||||
{ name: "SKU", value: sku, inline: true },
|
||||
{ name: "Source", value: "API Intercept", inline: true },
|
||||
{ name: "Link", value: `[VIEW PRODUCT](${productUrl})`, inline: false }
|
||||
],
|
||||
footer: { text: "Pokemon Monitor - API Detection" },
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(config.discordWebhook, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
content: "@everyone NEW SKU FROM API!",
|
||||
embeds: [embed]
|
||||
})
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
console.log(`[API Monitor] Discord alert sent for SKU: ${sku}`);
|
||||
|
||||
chrome.notifications.create({
|
||||
type: "basic",
|
||||
iconUrl: "icon128.png",
|
||||
title: "NEW SKU DETECTED!",
|
||||
message: `SKU: ${sku} - Check Discord!`
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[API Monitor] Error sending alert:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Main stock check function
|
||||
async function runStockCheck() {
|
||||
if (!config.enabled) {
|
||||
@@ -287,11 +391,55 @@ function processProducts(products) {
|
||||
lastSeen: now,
|
||||
lastInStock: product.inStock ? now : null
|
||||
};
|
||||
|
||||
// Queue new_drop event for dashboard sync
|
||||
pendingEvents.push({
|
||||
type: 'new_drop',
|
||||
url: product.url,
|
||||
name: product.name,
|
||||
price: product.price,
|
||||
inStock: product.inStock,
|
||||
timestamp: now
|
||||
});
|
||||
} else {
|
||||
// Existing product - check for restock
|
||||
if (product.inStock && !existing.inStock) {
|
||||
restockedProducts.push(product);
|
||||
existing.lastInStock = now;
|
||||
|
||||
// Queue restock event for dashboard sync
|
||||
pendingEvents.push({
|
||||
type: 'restock',
|
||||
url: product.url,
|
||||
name: product.name,
|
||||
price: product.price,
|
||||
previousPrice: existing.price,
|
||||
timestamp: now
|
||||
});
|
||||
}
|
||||
|
||||
// Track when items go OUT of stock (for selling rate calculation)
|
||||
if (!product.inStock && existing.inStock) {
|
||||
pendingEvents.push({
|
||||
type: 'out_of_stock',
|
||||
url: product.url,
|
||||
name: product.name,
|
||||
price: product.price,
|
||||
lastInStock: existing.lastInStock, // When it came in stock
|
||||
timestamp: now // When it sold out
|
||||
});
|
||||
}
|
||||
|
||||
// Track price changes
|
||||
if (product.price && existing.price && product.price !== existing.price) {
|
||||
pendingEvents.push({
|
||||
type: 'price_change',
|
||||
url: product.url,
|
||||
name: product.name,
|
||||
oldPrice: existing.price,
|
||||
newPrice: product.price,
|
||||
timestamp: now
|
||||
});
|
||||
}
|
||||
|
||||
// Update
|
||||
@@ -393,7 +541,69 @@ async function sendDiscordNotification(product, alertType) {
|
||||
}
|
||||
}
|
||||
|
||||
// Listen for messages from popup
|
||||
// Track pending events to sync (restocks, new drops)
|
||||
let pendingEvents = [];
|
||||
|
||||
// Sync data to local dashboard
|
||||
async function syncToDashboard() {
|
||||
if (!config.syncToDashboard || !config.dashboardUrl) {
|
||||
return { success: false, reason: 'sync disabled' };
|
||||
}
|
||||
|
||||
// Build products array with all tracking data
|
||||
const productsToSync = Object.values(knownProducts).map(p => ({
|
||||
...p,
|
||||
site: 'pokemoncenter',
|
||||
sku: p.productId || extractSkuFromUrl(p.url)
|
||||
}));
|
||||
|
||||
const syncData = {
|
||||
skus: Array.from(knownSkus),
|
||||
products: productsToSync,
|
||||
apiStats: apiStats,
|
||||
events: pendingEvents // Include pending events (restocks, new drops)
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(`${config.dashboardUrl}/api/extension/sync`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(syncData)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const result = await response.json();
|
||||
console.log(`[Dashboard Sync] Success - ${result.total_skus} SKUs, ${result.total_products} products`);
|
||||
// Clear pending events after successful sync
|
||||
pendingEvents = [];
|
||||
return { success: true, ...result };
|
||||
} else {
|
||||
console.warn(`[Dashboard Sync] Failed: ${response.status}`);
|
||||
return { success: false, status: response.status };
|
||||
}
|
||||
} catch (error) {
|
||||
// Dashboard might not be running - silently fail
|
||||
console.debug(`[Dashboard Sync] Dashboard not available: ${error.message}`);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to extract SKU from Pokemon Center URL
|
||||
function extractSkuFromUrl(url) {
|
||||
if (!url) return null;
|
||||
const match = url.match(/\/product\/([^\/\?]+)/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
// Auto-sync to dashboard periodically (every 5 minutes)
|
||||
setInterval(() => {
|
||||
if (config.syncToDashboard) {
|
||||
syncToDashboard();
|
||||
}
|
||||
}, 5 * 60 * 1000);
|
||||
|
||||
|
||||
// Listen for messages from popup and content scripts
|
||||
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
||||
if (message.type === "getConfig") {
|
||||
sendResponse(config);
|
||||
@@ -407,12 +617,27 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
||||
} else if (message.type === "getStats") {
|
||||
sendResponse({
|
||||
totalProducts: Object.keys(knownProducts).length,
|
||||
enabled: config.enabled
|
||||
enabled: config.enabled,
|
||||
apiStats: apiStats,
|
||||
knownSkuCount: knownSkus.size
|
||||
});
|
||||
} else if (message.type === "clearProducts") {
|
||||
knownProducts = {};
|
||||
saveProducts();
|
||||
sendResponse({ success: true });
|
||||
} else if (message.type === "clearSkus") {
|
||||
knownSkus = new Set();
|
||||
apiStats.totalSkusTracked = 0;
|
||||
saveKnownSkus();
|
||||
sendResponse({ success: true });
|
||||
} else if (message.type === "apiData") {
|
||||
// Handle API data from content script interceptor
|
||||
handleApiData(message.data);
|
||||
sendResponse({ success: true });
|
||||
} else if (message.type === "syncToDashboard") {
|
||||
// Manual sync to local dashboard
|
||||
syncToDashboard().then(result => sendResponse(result));
|
||||
return true; // Keep channel open for async response
|
||||
}
|
||||
return true;
|
||||
});
|
||||
@@ -420,7 +645,9 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
||||
// Run initial check after a short delay
|
||||
setTimeout(() => {
|
||||
loadConfig().then(() => {
|
||||
loadProducts().then(() => {
|
||||
loadProducts();
|
||||
loadKnownSkus().then(() => {
|
||||
console.log("[API Monitor] Ready - will intercept API calls when you browse Pokemon Center");
|
||||
if (config.enabled) {
|
||||
console.log(`Check interval: ${config.checkIntervalSeconds} seconds`);
|
||||
runStockCheck();
|
||||
|
||||
Reference in New Issue
Block a user