mmcghen
8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
371 lines
12 KiB
Python
371 lines
12 KiB
Python
"""
|
|
Pokemon Center Backend Monitor
|
|
|
|
Detects NEW products added to the API before they're publicly announced.
|
|
This is how accounts like @pokepullzhq detect drops early.
|
|
|
|
Strategy:
|
|
1. Maintain a list of all known product SKUs
|
|
2. Periodically check the API for current products
|
|
3. Compare: Any new SKUs = potential silent drop
|
|
4. Alert immediately on new detections
|
|
|
|
Usage:
|
|
1. First run warmup_session.py to get valid cookies
|
|
2. Then run: python backend_monitor.py
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
import pickle
|
|
import logging
|
|
import requests
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from typing import Optional, Dict, List, Set
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# File paths
|
|
DATA_DIR = Path(__file__).parent.parent / "data"
|
|
SESSION_DIR = DATA_DIR / "sessions"
|
|
COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
|
|
KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
|
|
DETECTIONS_LOG = DATA_DIR / "detections.json"
|
|
|
|
# API configuration
|
|
API_BASE = "https://www.pokemoncenter.com"
|
|
|
|
# Categories to monitor for new products
|
|
MONITOR_CATEGORIES = [
|
|
"new-releases",
|
|
"tcg-cards",
|
|
# Add more as needed
|
|
]
|
|
|
|
# Headers required for API calls
|
|
HEADERS = {
|
|
"Accept": "application/json",
|
|
"Accept-Version": "1",
|
|
"Content-Type": "application/json",
|
|
"X-Store-Locale": "en-us",
|
|
"X-Store-Scope": "pokemon",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
|
"Referer": "https://www.pokemoncenter.com/",
|
|
}
|
|
|
|
# Check interval (seconds)
|
|
CHECK_INTERVAL = 60
|
|
|
|
|
|
class BackendMonitor:
|
|
"""Monitors Pokemon Center API for new product drops"""
|
|
|
|
def __init__(self):
|
|
self.session = requests.Session()
|
|
self.session.headers.update(HEADERS)
|
|
self.known_skus: Set[str] = set()
|
|
self.cookies_valid = False
|
|
self.detections: List[Dict] = []
|
|
|
|
def load_cookies(self) -> bool:
|
|
"""Load cookies from warmup session"""
|
|
if not COOKIE_FILE.exists():
|
|
logger.error(f"No cookie file found at {COOKIE_FILE}")
|
|
logger.info("Run warmup_session.py first!")
|
|
return False
|
|
|
|
try:
|
|
with open(COOKIE_FILE, 'rb') as f:
|
|
cookies = pickle.load(f)
|
|
|
|
for cookie in cookies:
|
|
self.session.cookies.set(
|
|
cookie['name'],
|
|
cookie['value'],
|
|
domain=cookie.get('domain', '.pokemoncenter.com'),
|
|
path=cookie.get('path', '/')
|
|
)
|
|
|
|
logger.info(f"Loaded {len(cookies)} cookies")
|
|
self.cookies_valid = True
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to load cookies: {e}")
|
|
return False
|
|
|
|
def load_known_skus(self):
|
|
"""Load previously seen SKUs"""
|
|
if KNOWN_SKUS_FILE.exists():
|
|
with open(KNOWN_SKUS_FILE, 'r') as f:
|
|
data = json.load(f)
|
|
self.known_skus = set(data.get('skus', []))
|
|
logger.info(f"Loaded {len(self.known_skus)} known SKUs")
|
|
else:
|
|
logger.info("No known SKUs file - will create on first run")
|
|
self.known_skus = set()
|
|
|
|
def save_known_skus(self):
|
|
"""Save known SKUs to file"""
|
|
with open(KNOWN_SKUS_FILE, 'w') as f:
|
|
json.dump({
|
|
'skus': list(self.known_skus),
|
|
'last_updated': datetime.now().isoformat(),
|
|
'count': len(self.known_skus)
|
|
}, f, indent=2)
|
|
|
|
def log_detection(self, sku: str, product_info: Dict):
|
|
"""Log a new product detection"""
|
|
detection = {
|
|
'sku': sku,
|
|
'detected_at': datetime.now().isoformat(),
|
|
'product_info': product_info
|
|
}
|
|
self.detections.append(detection)
|
|
|
|
# Append to detections log file
|
|
detections = []
|
|
if DETECTIONS_LOG.exists():
|
|
with open(DETECTIONS_LOG, 'r') as f:
|
|
detections = json.load(f)
|
|
|
|
detections.append(detection)
|
|
|
|
with open(DETECTIONS_LOG, 'w') as f:
|
|
json.dump(detections, f, indent=2)
|
|
|
|
def get_category_products(self, category: str) -> Optional[Dict]:
|
|
"""Fetch products from a category endpoint"""
|
|
url = f"{API_BASE}/site/resourceapi/category/{category}"
|
|
|
|
try:
|
|
response = self.session.get(url, timeout=15)
|
|
|
|
if response.status_code == 403:
|
|
logger.warning("API blocked - cookies may have expired")
|
|
self.cookies_valid = False
|
|
return None
|
|
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
|
|
logger.warning(f"Unexpected status {response.status_code} for {category}")
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching {category}: {e}")
|
|
return None
|
|
|
|
def get_product_details(self, sku: str) -> Optional[Dict]:
|
|
"""Get full details for a specific product"""
|
|
url = f"{API_BASE}/tpci-ecommweb-api/product/{sku}"
|
|
|
|
try:
|
|
response = self.session.get(url, timeout=15)
|
|
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching product {sku}: {e}")
|
|
return None
|
|
|
|
def extract_skus_from_response(self, data: Dict) -> Set[str]:
|
|
"""Extract product SKUs from API response"""
|
|
skus = set()
|
|
|
|
# The response structure varies - try multiple approaches
|
|
# This will need adjustment based on actual API response
|
|
|
|
def find_skus(obj, depth=0):
|
|
"""Recursively find SKU-like values"""
|
|
if depth > 10: # Prevent infinite recursion
|
|
return
|
|
|
|
if isinstance(obj, dict):
|
|
# Look for SKU fields
|
|
for key in ['sku', 'skuCode', 'productId', 'id', 'code']:
|
|
if key in obj:
|
|
value = obj[key]
|
|
if isinstance(value, str) and self._looks_like_sku(value):
|
|
skus.add(value)
|
|
|
|
# Look in nested objects
|
|
for value in obj.values():
|
|
find_skus(value, depth + 1)
|
|
|
|
elif isinstance(obj, list):
|
|
for item in obj:
|
|
find_skus(item, depth + 1)
|
|
|
|
find_skus(data)
|
|
return skus
|
|
|
|
def _looks_like_sku(self, value: str) -> bool:
|
|
"""Check if a string looks like a Pokemon Center SKU"""
|
|
# SKUs we've seen: 699-17113, 191-85953, 10-10191-109
|
|
if not value:
|
|
return False
|
|
|
|
# Must contain digits and possibly hyphens
|
|
has_digit = any(c.isdigit() for c in value)
|
|
reasonable_length = 5 <= len(value) <= 20
|
|
|
|
return has_digit and reasonable_length
|
|
|
|
def check_for_new_products(self) -> List[Dict]:
|
|
"""Main check - look for new SKUs across all categories"""
|
|
if not self.cookies_valid:
|
|
if not self.load_cookies():
|
|
return []
|
|
|
|
new_products = []
|
|
current_skus = set()
|
|
|
|
for category in MONITOR_CATEGORIES:
|
|
logger.debug(f"Checking category: {category}")
|
|
|
|
data = self.get_category_products(category)
|
|
if data:
|
|
skus = self.extract_skus_from_response(data)
|
|
current_skus.update(skus)
|
|
logger.debug(f" Found {len(skus)} SKUs in {category}")
|
|
|
|
if not current_skus:
|
|
logger.warning("No SKUs found - API may not be returning data")
|
|
return []
|
|
|
|
# Find new SKUs
|
|
new_skus = current_skus - self.known_skus
|
|
|
|
if new_skus:
|
|
logger.info(f"!!! DETECTED {len(new_skus)} NEW SKU(s) !!!")
|
|
|
|
for sku in new_skus:
|
|
# Get full product details
|
|
details = self.get_product_details(sku)
|
|
|
|
product_info = {
|
|
'sku': sku,
|
|
'details': details,
|
|
'detected_at': datetime.now().isoformat()
|
|
}
|
|
|
|
# Try to extract name from details
|
|
name = "Unknown Product"
|
|
if details:
|
|
# Look for name in various places
|
|
name = (
|
|
details.get('name') or
|
|
details.get('displayName') or
|
|
details.get('definition', {}).get('display-name') or
|
|
sku
|
|
)
|
|
|
|
logger.info(f" NEW: {sku} - {name}")
|
|
self.log_detection(sku, product_info)
|
|
new_products.append(product_info)
|
|
|
|
# Add to known SKUs
|
|
self.known_skus.add(sku)
|
|
|
|
# Save updated known SKUs
|
|
self.save_known_skus()
|
|
|
|
else:
|
|
logger.info(f"Check complete - {len(current_skus)} products, no new drops")
|
|
|
|
return new_products
|
|
|
|
def send_discord_alert(self, product: Dict):
|
|
"""Send Discord notification for new product"""
|
|
# Import from your existing discord_notifier
|
|
try:
|
|
from src.discord_notifier import send_notification
|
|
# You'd format and send the alert here
|
|
logger.info(f"Discord alert sent for {product['sku']}")
|
|
except ImportError:
|
|
logger.warning("Discord notifier not available")
|
|
|
|
def run(self):
|
|
"""Main monitoring loop"""
|
|
print("=" * 60)
|
|
print("Pokemon Center Backend Monitor")
|
|
print("=" * 60)
|
|
print()
|
|
print("This monitors for NEW products added to the API.")
|
|
print("New SKUs = potential silent drops before announcement!")
|
|
print()
|
|
print(f"Check interval: {CHECK_INTERVAL} seconds")
|
|
print(f"Monitoring categories: {', '.join(MONITOR_CATEGORIES)}")
|
|
print()
|
|
print("Press Ctrl+C to stop")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
# Load known SKUs
|
|
self.load_known_skus()
|
|
|
|
# Load cookies
|
|
if not self.load_cookies():
|
|
print("\nERROR: No valid cookies!")
|
|
print("Run: python warmup_session.py")
|
|
return
|
|
|
|
# First check - populate known SKUs if empty
|
|
if not self.known_skus:
|
|
logger.info("First run - building initial SKU database...")
|
|
self.check_for_new_products()
|
|
logger.info(f"Baseline established with {len(self.known_skus)} products")
|
|
print()
|
|
|
|
# Main loop
|
|
check_count = 0
|
|
while True:
|
|
try:
|
|
check_count += 1
|
|
logger.info(f"--- Check #{check_count} ---")
|
|
|
|
new_products = self.check_for_new_products()
|
|
|
|
if new_products:
|
|
print()
|
|
print("!" * 60)
|
|
print("!!! NEW PRODUCT DETECTED !!!")
|
|
print("!" * 60)
|
|
for p in new_products:
|
|
print(f" SKU: {p['sku']}")
|
|
# Send Discord alert
|
|
self.send_discord_alert(p)
|
|
print("!" * 60)
|
|
print()
|
|
|
|
# Wait for next check
|
|
time.sleep(CHECK_INTERVAL)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\nStopping monitor...")
|
|
self.save_known_skus()
|
|
break
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in main loop: {e}")
|
|
time.sleep(CHECK_INTERVAL)
|
|
|
|
|
|
def main():
|
|
monitor = BackendMonitor()
|
|
monitor.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|