feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,370 @@
|
||||
"""
|
||||
Pokemon Center Backend Monitor
|
||||
|
||||
Detects NEW products added to the API before they're publicly announced.
|
||||
This is how accounts like @pokepullzhq detect drops early.
|
||||
|
||||
Strategy:
|
||||
1. Maintain a list of all known product SKUs
|
||||
2. Periodically check the API for current products
|
||||
3. Compare: Any new SKUs = potential silent drop
|
||||
4. Alert immediately on new detections
|
||||
|
||||
Usage:
|
||||
1. First run warmup_session.py to get valid cookies
|
||||
2. Then run: python backend_monitor.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import pickle
|
||||
import logging
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Set
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File paths
|
||||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
SESSION_DIR = DATA_DIR / "sessions"
|
||||
COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
|
||||
KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
|
||||
DETECTIONS_LOG = DATA_DIR / "detections.json"
|
||||
|
||||
# API configuration
|
||||
API_BASE = "https://www.pokemoncenter.com"
|
||||
|
||||
# Categories to monitor for new products
|
||||
MONITOR_CATEGORIES = [
|
||||
"new-releases",
|
||||
"tcg-cards",
|
||||
# Add more as needed
|
||||
]
|
||||
|
||||
# Headers required for API calls
|
||||
HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/",
|
||||
}
|
||||
|
||||
# Check interval (seconds)
|
||||
CHECK_INTERVAL = 60
|
||||
|
||||
|
||||
class BackendMonitor:
|
||||
"""Monitors Pokemon Center API for new product drops"""
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(HEADERS)
|
||||
self.known_skus: Set[str] = set()
|
||||
self.cookies_valid = False
|
||||
self.detections: List[Dict] = []
|
||||
|
||||
def load_cookies(self) -> bool:
|
||||
"""Load cookies from warmup session"""
|
||||
if not COOKIE_FILE.exists():
|
||||
logger.error(f"No cookie file found at {COOKIE_FILE}")
|
||||
logger.info("Run warmup_session.py first!")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(COOKIE_FILE, 'rb') as f:
|
||||
cookies = pickle.load(f)
|
||||
|
||||
for cookie in cookies:
|
||||
self.session.cookies.set(
|
||||
cookie['name'],
|
||||
cookie['value'],
|
||||
domain=cookie.get('domain', '.pokemoncenter.com'),
|
||||
path=cookie.get('path', '/')
|
||||
)
|
||||
|
||||
logger.info(f"Loaded {len(cookies)} cookies")
|
||||
self.cookies_valid = True
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cookies: {e}")
|
||||
return False
|
||||
|
||||
def load_known_skus(self):
|
||||
"""Load previously seen SKUs"""
|
||||
if KNOWN_SKUS_FILE.exists():
|
||||
with open(KNOWN_SKUS_FILE, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.known_skus = set(data.get('skus', []))
|
||||
logger.info(f"Loaded {len(self.known_skus)} known SKUs")
|
||||
else:
|
||||
logger.info("No known SKUs file - will create on first run")
|
||||
self.known_skus = set()
|
||||
|
||||
def save_known_skus(self):
|
||||
"""Save known SKUs to file"""
|
||||
with open(KNOWN_SKUS_FILE, 'w') as f:
|
||||
json.dump({
|
||||
'skus': list(self.known_skus),
|
||||
'last_updated': datetime.now().isoformat(),
|
||||
'count': len(self.known_skus)
|
||||
}, f, indent=2)
|
||||
|
||||
def log_detection(self, sku: str, product_info: Dict):
|
||||
"""Log a new product detection"""
|
||||
detection = {
|
||||
'sku': sku,
|
||||
'detected_at': datetime.now().isoformat(),
|
||||
'product_info': product_info
|
||||
}
|
||||
self.detections.append(detection)
|
||||
|
||||
# Append to detections log file
|
||||
detections = []
|
||||
if DETECTIONS_LOG.exists():
|
||||
with open(DETECTIONS_LOG, 'r') as f:
|
||||
detections = json.load(f)
|
||||
|
||||
detections.append(detection)
|
||||
|
||||
with open(DETECTIONS_LOG, 'w') as f:
|
||||
json.dump(detections, f, indent=2)
|
||||
|
||||
def get_category_products(self, category: str) -> Optional[Dict]:
|
||||
"""Fetch products from a category endpoint"""
|
||||
url = f"{API_BASE}/site/resourceapi/category/{category}"
|
||||
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 403:
|
||||
logger.warning("API blocked - cookies may have expired")
|
||||
self.cookies_valid = False
|
||||
return None
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
|
||||
logger.warning(f"Unexpected status {response.status_code} for {category}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching {category}: {e}")
|
||||
return None
|
||||
|
||||
def get_product_details(self, sku: str) -> Optional[Dict]:
|
||||
"""Get full details for a specific product"""
|
||||
url = f"{API_BASE}/tpci-ecommweb-api/product/{sku}"
|
||||
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching product {sku}: {e}")
|
||||
return None
|
||||
|
||||
def extract_skus_from_response(self, data: Dict) -> Set[str]:
|
||||
"""Extract product SKUs from API response"""
|
||||
skus = set()
|
||||
|
||||
# The response structure varies - try multiple approaches
|
||||
# This will need adjustment based on actual API response
|
||||
|
||||
def find_skus(obj, depth=0):
|
||||
"""Recursively find SKU-like values"""
|
||||
if depth > 10: # Prevent infinite recursion
|
||||
return
|
||||
|
||||
if isinstance(obj, dict):
|
||||
# Look for SKU fields
|
||||
for key in ['sku', 'skuCode', 'productId', 'id', 'code']:
|
||||
if key in obj:
|
||||
value = obj[key]
|
||||
if isinstance(value, str) and self._looks_like_sku(value):
|
||||
skus.add(value)
|
||||
|
||||
# Look in nested objects
|
||||
for value in obj.values():
|
||||
find_skus(value, depth + 1)
|
||||
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
find_skus(item, depth + 1)
|
||||
|
||||
find_skus(data)
|
||||
return skus
|
||||
|
||||
def _looks_like_sku(self, value: str) -> bool:
|
||||
"""Check if a string looks like a Pokemon Center SKU"""
|
||||
# SKUs we've seen: 699-17113, 191-85953, 10-10191-109
|
||||
if not value:
|
||||
return False
|
||||
|
||||
# Must contain digits and possibly hyphens
|
||||
has_digit = any(c.isdigit() for c in value)
|
||||
reasonable_length = 5 <= len(value) <= 20
|
||||
|
||||
return has_digit and reasonable_length
|
||||
|
||||
def check_for_new_products(self) -> List[Dict]:
|
||||
"""Main check - look for new SKUs across all categories"""
|
||||
if not self.cookies_valid:
|
||||
if not self.load_cookies():
|
||||
return []
|
||||
|
||||
new_products = []
|
||||
current_skus = set()
|
||||
|
||||
for category in MONITOR_CATEGORIES:
|
||||
logger.debug(f"Checking category: {category}")
|
||||
|
||||
data = self.get_category_products(category)
|
||||
if data:
|
||||
skus = self.extract_skus_from_response(data)
|
||||
current_skus.update(skus)
|
||||
logger.debug(f" Found {len(skus)} SKUs in {category}")
|
||||
|
||||
if not current_skus:
|
||||
logger.warning("No SKUs found - API may not be returning data")
|
||||
return []
|
||||
|
||||
# Find new SKUs
|
||||
new_skus = current_skus - self.known_skus
|
||||
|
||||
if new_skus:
|
||||
logger.info(f"!!! DETECTED {len(new_skus)} NEW SKU(s) !!!")
|
||||
|
||||
for sku in new_skus:
|
||||
# Get full product details
|
||||
details = self.get_product_details(sku)
|
||||
|
||||
product_info = {
|
||||
'sku': sku,
|
||||
'details': details,
|
||||
'detected_at': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Try to extract name from details
|
||||
name = "Unknown Product"
|
||||
if details:
|
||||
# Look for name in various places
|
||||
name = (
|
||||
details.get('name') or
|
||||
details.get('displayName') or
|
||||
details.get('definition', {}).get('display-name') or
|
||||
sku
|
||||
)
|
||||
|
||||
logger.info(f" NEW: {sku} - {name}")
|
||||
self.log_detection(sku, product_info)
|
||||
new_products.append(product_info)
|
||||
|
||||
# Add to known SKUs
|
||||
self.known_skus.add(sku)
|
||||
|
||||
# Save updated known SKUs
|
||||
self.save_known_skus()
|
||||
|
||||
else:
|
||||
logger.info(f"Check complete - {len(current_skus)} products, no new drops")
|
||||
|
||||
return new_products
|
||||
|
||||
def send_discord_alert(self, product: Dict):
|
||||
"""Send Discord notification for new product"""
|
||||
# Import from your existing discord_notifier
|
||||
try:
|
||||
from src.discord_notifier import send_notification
|
||||
# You'd format and send the alert here
|
||||
logger.info(f"Discord alert sent for {product['sku']}")
|
||||
except ImportError:
|
||||
logger.warning("Discord notifier not available")
|
||||
|
||||
def run(self):
|
||||
"""Main monitoring loop"""
|
||||
print("=" * 60)
|
||||
print("Pokemon Center Backend Monitor")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("This monitors for NEW products added to the API.")
|
||||
print("New SKUs = potential silent drops before announcement!")
|
||||
print()
|
||||
print(f"Check interval: {CHECK_INTERVAL} seconds")
|
||||
print(f"Monitoring categories: {', '.join(MONITOR_CATEGORIES)}")
|
||||
print()
|
||||
print("Press Ctrl+C to stop")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Load known SKUs
|
||||
self.load_known_skus()
|
||||
|
||||
# Load cookies
|
||||
if not self.load_cookies():
|
||||
print("\nERROR: No valid cookies!")
|
||||
print("Run: python warmup_session.py")
|
||||
return
|
||||
|
||||
# First check - populate known SKUs if empty
|
||||
if not self.known_skus:
|
||||
logger.info("First run - building initial SKU database...")
|
||||
self.check_for_new_products()
|
||||
logger.info(f"Baseline established with {len(self.known_skus)} products")
|
||||
print()
|
||||
|
||||
# Main loop
|
||||
check_count = 0
|
||||
while True:
|
||||
try:
|
||||
check_count += 1
|
||||
logger.info(f"--- Check #{check_count} ---")
|
||||
|
||||
new_products = self.check_for_new_products()
|
||||
|
||||
if new_products:
|
||||
print()
|
||||
print("!" * 60)
|
||||
print("!!! NEW PRODUCT DETECTED !!!")
|
||||
print("!" * 60)
|
||||
for p in new_products:
|
||||
print(f" SKU: {p['sku']}")
|
||||
# Send Discord alert
|
||||
self.send_discord_alert(p)
|
||||
print("!" * 60)
|
||||
print()
|
||||
|
||||
# Wait for next check
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopping monitor...")
|
||||
self.save_known_skus()
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in main loop: {e}")
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
|
||||
def main():
|
||||
monitor = BackendMonitor()
|
||||
monitor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user