feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Pokemon Center API Monitor
|
||||
|
||||
Uses cookies from a browser session to make API calls directly.
|
||||
Much lighter than full browser scraping once cookies are established.
|
||||
|
||||
Workflow:
|
||||
1. Run warmup_session.py first to get valid cookies
|
||||
2. This script uses those cookies to call APIs directly
|
||||
3. Falls back to browser if cookies expire
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import pickle
|
||||
import logging
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Any
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Session/cookie storage
|
||||
SESSION_DIR = Path(__file__).parent / "sessions"
|
||||
COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
|
||||
|
||||
# API endpoints discovered from HAR analysis
|
||||
API_BASE = "https://www.pokemoncenter.com"
|
||||
ENDPOINTS = {
|
||||
"product": "/tpci-ecommweb-api/product/{sku}",
|
||||
"status": "/tpci-ecommweb-api/product/status/{encoded_id}",
|
||||
"category": "/site/resourceapi/category/{category}",
|
||||
"reviews": "/tpci-ecommweb-api/review/get-product-scores",
|
||||
}
|
||||
|
||||
# Required headers from HAR capture
|
||||
BASE_HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/category/tcg-cards",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
}
|
||||
|
||||
# Known product SKUs for TCG (build this list over time)
|
||||
KNOWN_TCG_SKUS_FILE = Path(__file__).parent / "known_tcg_skus.json"
|
||||
|
||||
|
||||
class PokemonCenterAPI:
|
||||
"""Direct API client for Pokemon Center using session cookies"""
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(BASE_HEADERS)
|
||||
self.cookies_loaded = False
|
||||
self.last_cookie_refresh = None
|
||||
|
||||
def load_cookies(self) -> bool:
|
||||
"""Load cookies from the saved session file"""
|
||||
if not COOKIE_FILE.exists():
|
||||
logger.warning(f"No cookie file found at {COOKIE_FILE}")
|
||||
logger.info("Run warmup_session.py first to create a valid session")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(COOKIE_FILE, 'rb') as f:
|
||||
cookies = pickle.load(f)
|
||||
|
||||
# Add cookies to session
|
||||
for cookie in cookies:
|
||||
self.session.cookies.set(
|
||||
cookie['name'],
|
||||
cookie['value'],
|
||||
domain=cookie.get('domain', '.pokemoncenter.com'),
|
||||
path=cookie.get('path', '/')
|
||||
)
|
||||
|
||||
logger.info(f"Loaded {len(cookies)} cookies from session file")
|
||||
self.cookies_loaded = True
|
||||
self.last_cookie_refresh = datetime.now()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cookies: {e}")
|
||||
return False
|
||||
|
||||
def _make_request(self, method: str, url: str, **kwargs) -> Optional[requests.Response]:
|
||||
"""Make a request with error handling"""
|
||||
try:
|
||||
if method.upper() == "GET":
|
||||
response = self.session.get(url, timeout=15, **kwargs)
|
||||
else:
|
||||
response = self.session.post(url, timeout=15, **kwargs)
|
||||
|
||||
# Check for blocking
|
||||
if response.status_code == 403:
|
||||
if "captcha" in response.text.lower() or "blocked" in response.text.lower():
|
||||
logger.warning("Request blocked - cookies may have expired")
|
||||
self.cookies_loaded = False
|
||||
return None
|
||||
|
||||
return response
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Request failed: {e}")
|
||||
return None
|
||||
|
||||
def get_product(self, sku: str) -> Optional[Dict]:
|
||||
"""Get product details by SKU"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['product'].format(sku=sku)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_product_status(self, encoded_id: str) -> Optional[Dict]:
|
||||
"""Get product availability status"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['status'].format(encoded_id=encoded_id)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_category(self, category: str = "new-releases") -> Optional[Dict]:
|
||||
"""Get category listing (potential goldmine for new drops!)"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['category'].format(category=category)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_review_scores(self, sku_list: List[str]) -> Optional[Dict]:
|
||||
"""Get review scores for multiple SKUs"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
skus = ",".join(sku_list)
|
||||
url = f"{API_BASE}{ENDPOINTS['reviews']}?skuList={skus}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
|
||||
def load_known_skus() -> List[str]:
|
||||
"""Load list of known TCG SKUs"""
|
||||
if KNOWN_TCG_SKUS_FILE.exists():
|
||||
with open(KNOWN_TCG_SKUS_FILE, 'r') as f:
|
||||
return json.load(f)
|
||||
return []
|
||||
|
||||
|
||||
def save_known_skus(skus: List[str]):
|
||||
"""Save list of known TCG SKUs"""
|
||||
with open(KNOWN_TCG_SKUS_FILE, 'w') as f:
|
||||
json.dump(skus, f, indent=2)
|
||||
|
||||
|
||||
def test_api():
|
||||
"""Test the API client"""
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Monitor Test")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
api = PokemonCenterAPI()
|
||||
|
||||
if not api.load_cookies():
|
||||
print("\nNo valid cookies found!")
|
||||
print("Please run: python warmup_session.py")
|
||||
print("Then try again.")
|
||||
return
|
||||
|
||||
print("\n[1] Testing category endpoint (new-releases)...")
|
||||
category_data = api.get_category("new-releases")
|
||||
if category_data:
|
||||
print(" SUCCESS! Category data retrieved.")
|
||||
print(f" Keys: {list(category_data.keys())[:5]}")
|
||||
|
||||
# Save for analysis
|
||||
with open("category_response.json", "w") as f:
|
||||
json.dump(category_data, f, indent=2)
|
||||
print(" Saved to category_response.json")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print("\n[2] Testing product endpoint...")
|
||||
# Try a known SKU from our HAR capture
|
||||
product_data = api.get_product("699-17113")
|
||||
if product_data:
|
||||
print(" SUCCESS! Product data retrieved.")
|
||||
print(f" Keys: {list(product_data.keys())[:5]}")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print("\n[3] Testing review scores endpoint...")
|
||||
reviews = api.get_review_scores(["699-17113", "191-85953"])
|
||||
if reviews:
|
||||
print(" SUCCESS! Review scores retrieved.")
|
||||
print(f" Data: {reviews}")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
if category_data or product_data:
|
||||
print("API access working! Can monitor without full browser scraping.")
|
||||
print("Cookies will expire eventually - re-run warmup when needed.")
|
||||
else:
|
||||
print("API blocked. Need fresh cookies from browser session.")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_api()
|
||||
Reference in New Issue
Block a user