8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
207 lines
6.0 KiB
JavaScript
207 lines
6.0 KiB
JavaScript
// API Interceptor - Captures Pokemon Center API responses
|
|
// Injected into the page context to intercept fetch/XHR
|
|
|
|
(function() {
|
|
'use strict';
|
|
|
|
// Avoid double injection
|
|
if (window.__pokemonMonitorInjected) return;
|
|
window.__pokemonMonitorInjected = true;
|
|
|
|
const API_PATTERNS = [
|
|
'/tpci-ecommweb-api/product/',
|
|
'/site/resourceapi/category/',
|
|
'/tpci-ecommweb-api/review/',
|
|
'/graphql',
|
|
];
|
|
|
|
function isApiUrl(url) {
|
|
return API_PATTERNS.some(pattern => url.includes(pattern));
|
|
}
|
|
|
|
function extractSkusFromData(data, skus = new Set()) {
|
|
if (!data) return skus;
|
|
|
|
if (typeof data === 'object') {
|
|
// Look for SKU-like fields
|
|
for (const key of ['sku', 'skuCode', 'productId', 'id', 'code']) {
|
|
if (data[key] && typeof data[key] === 'string') {
|
|
const value = data[key];
|
|
// Pokemon Center SKUs: 699-17113, 191-85953, etc.
|
|
if (/^\d{1,3}-\d{4,6}$/.test(value) || /^\d{5,}$/.test(value)) {
|
|
skus.add(value);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Recurse into nested objects/arrays
|
|
for (const value of Object.values(data)) {
|
|
if (typeof value === 'object') {
|
|
extractSkusFromData(value, skus);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Array.isArray(data)) {
|
|
for (const item of data) {
|
|
extractSkusFromData(item, skus);
|
|
}
|
|
}
|
|
|
|
return skus;
|
|
}
|
|
|
|
function sendToExtension(type, data) {
|
|
window.postMessage({
|
|
source: 'pokemon-api-interceptor',
|
|
type: type,
|
|
data: data
|
|
}, '*');
|
|
}
|
|
|
|
// Intercept fetch
|
|
const originalFetch = window.fetch;
|
|
window.fetch = async function(...args) {
|
|
const response = await originalFetch.apply(this, args);
|
|
|
|
try {
|
|
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
|
|
|
|
if (isApiUrl(url) && response.ok) {
|
|
// Clone response so we can read it without consuming
|
|
const clone = response.clone();
|
|
const contentType = clone.headers.get('content-type') || '';
|
|
|
|
if (contentType.includes('application/json')) {
|
|
clone.json().then(data => {
|
|
const skus = extractSkusFromData(data);
|
|
|
|
sendToExtension('api-response', {
|
|
url: url,
|
|
timestamp: Date.now(),
|
|
skuCount: skus.size,
|
|
skus: Array.from(skus),
|
|
preview: JSON.stringify(data).slice(0, 500)
|
|
});
|
|
}).catch(() => {});
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Silently ignore errors
|
|
}
|
|
|
|
return response;
|
|
};
|
|
|
|
// Intercept XMLHttpRequest
|
|
const originalXhrOpen = XMLHttpRequest.prototype.open;
|
|
const originalXhrSend = XMLHttpRequest.prototype.send;
|
|
|
|
XMLHttpRequest.prototype.open = function(method, url, ...rest) {
|
|
this._monitorUrl = url;
|
|
return originalXhrOpen.apply(this, [method, url, ...rest]);
|
|
};
|
|
|
|
XMLHttpRequest.prototype.send = function(...args) {
|
|
const xhr = this;
|
|
const url = xhr._monitorUrl || '';
|
|
|
|
if (isApiUrl(url)) {
|
|
xhr.addEventListener('load', function() {
|
|
try {
|
|
if (xhr.status === 200) {
|
|
const contentType = xhr.getResponseHeader('content-type') || '';
|
|
if (contentType.includes('application/json')) {
|
|
const data = JSON.parse(xhr.responseText);
|
|
const skus = extractSkusFromData(data);
|
|
|
|
sendToExtension('api-response', {
|
|
url: url,
|
|
timestamp: Date.now(),
|
|
skuCount: skus.size,
|
|
skus: Array.from(skus),
|
|
preview: JSON.stringify(data).slice(0, 500)
|
|
});
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Silently ignore
|
|
}
|
|
});
|
|
}
|
|
|
|
return originalXhrSend.apply(this, args);
|
|
};
|
|
|
|
console.log('[Pokemon Monitor] API interceptor active');
|
|
|
|
// Also scan embedded page data for SKUs (handles server-side rendered pages)
|
|
function scanPageForSkus() {
|
|
const skus = new Set();
|
|
|
|
// Method 1: Extract from product URLs in the page
|
|
const productLinks = document.querySelectorAll('a[href*="/product/"]');
|
|
productLinks.forEach(link => {
|
|
const match = link.href.match(/\/product\/(\d{1,3}-\d{4,6})/);
|
|
if (match) {
|
|
skus.add(match[1]);
|
|
}
|
|
});
|
|
|
|
// Method 2: Look for __NEXT_DATA__ (Next.js embedded data)
|
|
const nextDataScript = document.getElementById('__NEXT_DATA__');
|
|
if (nextDataScript) {
|
|
try {
|
|
const data = JSON.parse(nextDataScript.textContent);
|
|
extractSkusFromData(data, skus);
|
|
} catch (e) {}
|
|
}
|
|
|
|
// Method 3: Look for any script tags with JSON containing product data
|
|
document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]').forEach(script => {
|
|
try {
|
|
const data = JSON.parse(script.textContent);
|
|
extractSkusFromData(data, skus);
|
|
} catch (e) {}
|
|
});
|
|
|
|
// Method 4: Look for data attributes on product elements
|
|
document.querySelectorAll('[data-sku], [data-product-id], [data-product-sku]').forEach(el => {
|
|
const sku = el.dataset.sku || el.dataset.productId || el.dataset.productSku;
|
|
if (sku && /^\d{1,3}-\d{4,6}$/.test(sku)) {
|
|
skus.add(sku);
|
|
}
|
|
});
|
|
|
|
if (skus.size > 0) {
|
|
console.log(`[Pokemon Monitor] Found ${skus.size} SKUs in page`);
|
|
sendToExtension('api-response', {
|
|
url: window.location.href,
|
|
timestamp: Date.now(),
|
|
skuCount: skus.size,
|
|
skus: Array.from(skus),
|
|
source: 'page-scan'
|
|
});
|
|
}
|
|
}
|
|
|
|
// Scan page after it loads
|
|
if (document.readyState === 'complete') {
|
|
setTimeout(scanPageForSkus, 1000);
|
|
} else {
|
|
window.addEventListener('load', () => setTimeout(scanPageForSkus, 1000));
|
|
}
|
|
|
|
// Re-scan when page content changes (for infinite scroll, etc.)
|
|
let scanTimeout;
|
|
const observer = new MutationObserver(() => {
|
|
clearTimeout(scanTimeout);
|
|
scanTimeout = setTimeout(scanPageForSkus, 2000);
|
|
});
|
|
|
|
// Start observing after initial load
|
|
setTimeout(() => {
|
|
observer.observe(document.body, { childList: true, subtree: true });
|
|
}, 3000);
|
|
})();
|