feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
// API Interceptor - Captures Pokemon Center API responses
|
||||
// Injected into the page context to intercept fetch/XHR
|
||||
|
||||
(function() {
|
||||
'use strict';
|
||||
|
||||
// Avoid double injection
|
||||
if (window.__pokemonMonitorInjected) return;
|
||||
window.__pokemonMonitorInjected = true;
|
||||
|
||||
const API_PATTERNS = [
|
||||
'/tpci-ecommweb-api/product/',
|
||||
'/site/resourceapi/category/',
|
||||
'/tpci-ecommweb-api/review/',
|
||||
'/graphql',
|
||||
];
|
||||
|
||||
function isApiUrl(url) {
|
||||
return API_PATTERNS.some(pattern => url.includes(pattern));
|
||||
}
|
||||
|
||||
function extractSkusFromData(data, skus = new Set()) {
|
||||
if (!data) return skus;
|
||||
|
||||
if (typeof data === 'object') {
|
||||
// Look for SKU-like fields
|
||||
for (const key of ['sku', 'skuCode', 'productId', 'id', 'code']) {
|
||||
if (data[key] && typeof data[key] === 'string') {
|
||||
const value = data[key];
|
||||
// Pokemon Center SKUs: 699-17113, 191-85953, etc.
|
||||
if (/^\d{1,3}-\d{4,6}$/.test(value) || /^\d{5,}$/.test(value)) {
|
||||
skus.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recurse into nested objects/arrays
|
||||
for (const value of Object.values(data)) {
|
||||
if (typeof value === 'object') {
|
||||
extractSkusFromData(value, skus);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
for (const item of data) {
|
||||
extractSkusFromData(item, skus);
|
||||
}
|
||||
}
|
||||
|
||||
return skus;
|
||||
}
|
||||
|
||||
function sendToExtension(type, data) {
|
||||
window.postMessage({
|
||||
source: 'pokemon-api-interceptor',
|
||||
type: type,
|
||||
data: data
|
||||
}, '*');
|
||||
}
|
||||
|
||||
// Intercept fetch
|
||||
const originalFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
const response = await originalFetch.apply(this, args);
|
||||
|
||||
try {
|
||||
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
|
||||
|
||||
if (isApiUrl(url) && response.ok) {
|
||||
// Clone response so we can read it without consuming
|
||||
const clone = response.clone();
|
||||
const contentType = clone.headers.get('content-type') || '';
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
clone.json().then(data => {
|
||||
const skus = extractSkusFromData(data);
|
||||
|
||||
sendToExtension('api-response', {
|
||||
url: url,
|
||||
timestamp: Date.now(),
|
||||
skuCount: skus.size,
|
||||
skus: Array.from(skus),
|
||||
preview: JSON.stringify(data).slice(0, 500)
|
||||
});
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Silently ignore errors
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
// Intercept XMLHttpRequest
|
||||
const originalXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const originalXhrSend = XMLHttpRequest.prototype.send;
|
||||
|
||||
XMLHttpRequest.prototype.open = function(method, url, ...rest) {
|
||||
this._monitorUrl = url;
|
||||
return originalXhrOpen.apply(this, [method, url, ...rest]);
|
||||
};
|
||||
|
||||
XMLHttpRequest.prototype.send = function(...args) {
|
||||
const xhr = this;
|
||||
const url = xhr._monitorUrl || '';
|
||||
|
||||
if (isApiUrl(url)) {
|
||||
xhr.addEventListener('load', function() {
|
||||
try {
|
||||
if (xhr.status === 200) {
|
||||
const contentType = xhr.getResponseHeader('content-type') || '';
|
||||
if (contentType.includes('application/json')) {
|
||||
const data = JSON.parse(xhr.responseText);
|
||||
const skus = extractSkusFromData(data);
|
||||
|
||||
sendToExtension('api-response', {
|
||||
url: url,
|
||||
timestamp: Date.now(),
|
||||
skuCount: skus.size,
|
||||
skus: Array.from(skus),
|
||||
preview: JSON.stringify(data).slice(0, 500)
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Silently ignore
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return originalXhrSend.apply(this, args);
|
||||
};
|
||||
|
||||
console.log('[Pokemon Monitor] API interceptor active');
|
||||
|
||||
// Also scan embedded page data for SKUs (handles server-side rendered pages)
|
||||
function scanPageForSkus() {
|
||||
const skus = new Set();
|
||||
|
||||
// Method 1: Extract from product URLs in the page
|
||||
const productLinks = document.querySelectorAll('a[href*="/product/"]');
|
||||
productLinks.forEach(link => {
|
||||
const match = link.href.match(/\/product\/(\d{1,3}-\d{4,6})/);
|
||||
if (match) {
|
||||
skus.add(match[1]);
|
||||
}
|
||||
});
|
||||
|
||||
// Method 2: Look for __NEXT_DATA__ (Next.js embedded data)
|
||||
const nextDataScript = document.getElementById('__NEXT_DATA__');
|
||||
if (nextDataScript) {
|
||||
try {
|
||||
const data = JSON.parse(nextDataScript.textContent);
|
||||
extractSkusFromData(data, skus);
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
// Method 3: Look for any script tags with JSON containing product data
|
||||
document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]').forEach(script => {
|
||||
try {
|
||||
const data = JSON.parse(script.textContent);
|
||||
extractSkusFromData(data, skus);
|
||||
} catch (e) {}
|
||||
});
|
||||
|
||||
// Method 4: Look for data attributes on product elements
|
||||
document.querySelectorAll('[data-sku], [data-product-id], [data-product-sku]').forEach(el => {
|
||||
const sku = el.dataset.sku || el.dataset.productId || el.dataset.productSku;
|
||||
if (sku && /^\d{1,3}-\d{4,6}$/.test(sku)) {
|
||||
skus.add(sku);
|
||||
}
|
||||
});
|
||||
|
||||
if (skus.size > 0) {
|
||||
console.log(`[Pokemon Monitor] Found ${skus.size} SKUs in page`);
|
||||
sendToExtension('api-response', {
|
||||
url: window.location.href,
|
||||
timestamp: Date.now(),
|
||||
skuCount: skus.size,
|
||||
skus: Array.from(skus),
|
||||
source: 'page-scan'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Scan page after it loads
|
||||
if (document.readyState === 'complete') {
|
||||
setTimeout(scanPageForSkus, 1000);
|
||||
} else {
|
||||
window.addEventListener('load', () => setTimeout(scanPageForSkus, 1000));
|
||||
}
|
||||
|
||||
// Re-scan when page content changes (for infinite scroll, etc.)
|
||||
let scanTimeout;
|
||||
const observer = new MutationObserver(() => {
|
||||
clearTimeout(scanTimeout);
|
||||
scanTimeout = setTimeout(scanPageForSkus, 2000);
|
||||
});
|
||||
|
||||
// Start observing after initial load
|
||||
setTimeout(() => {
|
||||
observer.observe(document.body, { childList: true, subtree: true });
|
||||
}, 3000);
|
||||
})();
|
||||
Reference in New Issue
Block a user