feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Utility and experimental scripts
|
||||
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
HAR File Analyzer for Pokemon Center API Discovery
|
||||
|
||||
Analyzes a HAR (HTTP Archive) file exported from Chrome DevTools
|
||||
to find API endpoints used by Pokemon Center.
|
||||
|
||||
Usage:
|
||||
1. Export HAR from Chrome DevTools Network tab
|
||||
2. Save as pokemon_network.har in this folder
|
||||
3. Run: python analyze_har.py
|
||||
|
||||
Or specify a different file:
|
||||
python analyze_har.py my_capture.har
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from collections import defaultdict
|
||||
|
||||
# Patterns that indicate interesting API endpoints
|
||||
INTERESTING_PATTERNS = [
|
||||
r'/api/',
|
||||
r'/graphql',
|
||||
r'/v\d+/',
|
||||
r'product',
|
||||
r'catalog',
|
||||
r'search',
|
||||
r'inventory',
|
||||
r'stock',
|
||||
r'\.json$',
|
||||
r'algolia',
|
||||
r'contentful',
|
||||
r'commercetools',
|
||||
r'demandware',
|
||||
r'sfcc',
|
||||
]
|
||||
|
||||
# Skip these resource types
|
||||
SKIP_TYPES = [
|
||||
'image', 'stylesheet', 'font', 'script', 'media',
|
||||
'.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
|
||||
'.css', '.woff', '.woff2', '.ttf',
|
||||
'.js', # Usually not APIs, but we'll catch fetch calls
|
||||
]
|
||||
|
||||
def should_skip(url, mime_type=''):
|
||||
"""Check if we should skip this request"""
|
||||
url_lower = url.lower()
|
||||
mime_lower = mime_type.lower()
|
||||
|
||||
for skip in SKIP_TYPES:
|
||||
if skip in url_lower or skip in mime_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_interesting(url):
|
||||
"""Check if URL matches patterns we care about"""
|
||||
url_lower = url.lower()
|
||||
for pattern in INTERESTING_PATTERNS:
|
||||
if re.search(pattern, url_lower):
|
||||
return True
|
||||
return False
|
||||
|
||||
def extract_json_preview(content, max_length=500):
|
||||
"""Try to extract and preview JSON content"""
|
||||
if not content:
|
||||
return None
|
||||
|
||||
try:
|
||||
# HAR stores content as text or base64
|
||||
text = content.get('text', '')
|
||||
if not text:
|
||||
return None
|
||||
|
||||
# Try to parse as JSON
|
||||
data = json.loads(text)
|
||||
|
||||
# Return a preview
|
||||
preview = json.dumps(data, indent=2)
|
||||
if len(preview) > max_length:
|
||||
return preview[:max_length] + "\n... (truncated)"
|
||||
return preview
|
||||
except:
|
||||
return None
|
||||
|
||||
def analyze_har(har_path):
|
||||
"""Analyze a HAR file for API endpoints"""
|
||||
|
||||
print(f"\nLoading {har_path}...")
|
||||
|
||||
with open(har_path, 'r', encoding='utf-8') as f:
|
||||
har_data = json.load(f)
|
||||
|
||||
entries = har_data.get('log', {}).get('entries', [])
|
||||
print(f"Found {len(entries)} network requests")
|
||||
|
||||
# Categorize requests
|
||||
api_calls = []
|
||||
json_responses = []
|
||||
graphql_calls = []
|
||||
third_party_apis = []
|
||||
|
||||
domains = defaultdict(int)
|
||||
|
||||
for entry in entries:
|
||||
request = entry.get('request', {})
|
||||
response = entry.get('response', {})
|
||||
|
||||
url = request.get('url', '')
|
||||
method = request.get('method', 'GET')
|
||||
status = response.get('status', 0)
|
||||
mime_type = response.get('content', {}).get('mimeType', '')
|
||||
|
||||
# Track domains
|
||||
parsed = urlparse(url)
|
||||
domains[parsed.netloc] += 1
|
||||
|
||||
# Skip resources we don't care about
|
||||
if should_skip(url, mime_type):
|
||||
continue
|
||||
|
||||
# Check for JSON responses
|
||||
if 'json' in mime_type.lower():
|
||||
content_preview = extract_json_preview(response.get('content', {}))
|
||||
json_responses.append({
|
||||
'url': url,
|
||||
'method': method,
|
||||
'status': status,
|
||||
'preview': content_preview,
|
||||
'headers': {h['name']: h['value'] for h in request.get('headers', [])},
|
||||
'response_headers': {h['name']: h['value'] for h in response.get('headers', [])}
|
||||
})
|
||||
|
||||
# Check for GraphQL
|
||||
if 'graphql' in url.lower():
|
||||
post_data = request.get('postData', {})
|
||||
graphql_calls.append({
|
||||
'url': url,
|
||||
'method': method,
|
||||
'body': post_data.get('text', ''),
|
||||
'status': status
|
||||
})
|
||||
|
||||
# Check for interesting patterns
|
||||
if is_interesting(url):
|
||||
api_calls.append({
|
||||
'url': url,
|
||||
'method': method,
|
||||
'status': status,
|
||||
'mime': mime_type
|
||||
})
|
||||
|
||||
# Check for third-party APIs
|
||||
if any(tp in url.lower() for tp in ['algolia', 'contentful', 'commercetools', 'sfcc']):
|
||||
third_party_apis.append({
|
||||
'url': url,
|
||||
'method': method,
|
||||
'status': status
|
||||
})
|
||||
|
||||
# Print analysis
|
||||
print("\n" + "=" * 70)
|
||||
print("HAR ANALYSIS RESULTS")
|
||||
print("=" * 70)
|
||||
|
||||
# Domains summary
|
||||
print("\n[DOMAINS CONTACTED]")
|
||||
print("-" * 40)
|
||||
for domain, count in sorted(domains.items(), key=lambda x: -x[1])[:15]:
|
||||
print(f" {count:4d} requests {domain}")
|
||||
|
||||
# GraphQL calls
|
||||
if graphql_calls:
|
||||
print("\n[GRAPHQL ENDPOINTS]")
|
||||
print("-" * 40)
|
||||
for call in graphql_calls:
|
||||
print(f"\n [{call['method']}] {call['url']}")
|
||||
print(f" Status: {call['status']}")
|
||||
if call['body']:
|
||||
try:
|
||||
body = json.loads(call['body'])
|
||||
if 'query' in body:
|
||||
query_preview = body['query'][:200].replace('\n', ' ')
|
||||
print(f" Query: {query_preview}...")
|
||||
if 'operationName' in body:
|
||||
print(f" Operation: {body['operationName']}")
|
||||
except:
|
||||
print(f" Body: {call['body'][:200]}...")
|
||||
|
||||
# JSON responses (potential APIs)
|
||||
if json_responses:
|
||||
print("\n[JSON RESPONSES - Potential APIs]")
|
||||
print("-" * 40)
|
||||
|
||||
# Filter to most interesting ones
|
||||
product_related = [r for r in json_responses if 'product' in r['url'].lower()]
|
||||
other_json = [r for r in json_responses if 'product' not in r['url'].lower()][:10]
|
||||
|
||||
for resp in product_related + other_json:
|
||||
print(f"\n [{resp['method']}] {resp['url'][:100]}")
|
||||
print(f" Status: {resp['status']}")
|
||||
|
||||
if resp['preview']:
|
||||
print(f" Preview:")
|
||||
for line in resp['preview'].split('\n')[:10]:
|
||||
print(f" {line}")
|
||||
|
||||
# Third-party APIs
|
||||
if third_party_apis:
|
||||
print("\n[THIRD-PARTY APIs]")
|
||||
print("-" * 40)
|
||||
for api in third_party_apis:
|
||||
print(f" [{api['method']}] {api['url'][:100]}")
|
||||
|
||||
# Interesting endpoints
|
||||
if api_calls:
|
||||
print("\n[OTHER INTERESTING ENDPOINTS]")
|
||||
print("-" * 40)
|
||||
seen = set()
|
||||
for call in api_calls:
|
||||
# Dedupe by base URL
|
||||
parsed = urlparse(call['url'])
|
||||
base = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
|
||||
if base in seen:
|
||||
continue
|
||||
seen.add(base)
|
||||
|
||||
print(f" [{call['method']}] {call['url'][:100]}")
|
||||
print(f" Status: {call['status']}, Type: {call['mime']}")
|
||||
|
||||
# Save detailed results
|
||||
output_file = Path(har_path).stem + "_analysis.json"
|
||||
output_path = Path(har_path).parent / output_file
|
||||
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump({
|
||||
'summary': {
|
||||
'total_requests': len(entries),
|
||||
'json_responses': len(json_responses),
|
||||
'graphql_calls': len(graphql_calls),
|
||||
'third_party_apis': len(third_party_apis),
|
||||
'interesting_endpoints': len(api_calls)
|
||||
},
|
||||
'domains': dict(domains),
|
||||
'json_responses': json_responses,
|
||||
'graphql_calls': graphql_calls,
|
||||
'third_party_apis': third_party_apis,
|
||||
'api_calls': api_calls
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"\n[SAVED] Detailed results saved to: {output_path}")
|
||||
|
||||
# Recommendations
|
||||
print("\n" + "=" * 70)
|
||||
print("RECOMMENDATIONS")
|
||||
print("=" * 70)
|
||||
|
||||
if graphql_calls:
|
||||
print("""
|
||||
[OK] GraphQL endpoint found! This is likely the main data source.
|
||||
Next steps:
|
||||
1. Examine the query structure in the analysis JSON
|
||||
2. Test if the endpoint works without cookies/auth
|
||||
3. Build a monitor that polls this endpoint
|
||||
""")
|
||||
|
||||
if any('algolia' in api['url'].lower() for api in third_party_apis):
|
||||
print("""
|
||||
[OK] Algolia search detected! This is often used for product search.
|
||||
Next steps:
|
||||
1. Find the Algolia App ID and Search API Key (usually in page source)
|
||||
2. Query Algolia directly - very fast and low detection risk
|
||||
3. Search for "apiKey" or "applicationId" in the page source
|
||||
""")
|
||||
|
||||
if json_responses and not graphql_calls:
|
||||
print("""
|
||||
[INFO] JSON responses found but no GraphQL. Check the analysis JSON for:
|
||||
- Responses containing product arrays
|
||||
- URLs with /api/ or /v1/, /v2/ patterns
|
||||
- Look for pagination parameters (page, limit, offset)
|
||||
""")
|
||||
|
||||
if not json_responses and not graphql_calls:
|
||||
print("""
|
||||
[WARNING] No obvious API endpoints found. Possible reasons:
|
||||
- Server-side rendering (data embedded in HTML)
|
||||
- API calls blocked by bot protection
|
||||
- Need to scroll/interact more to trigger lazy loading
|
||||
|
||||
Try:
|
||||
- Scrolling more on the page before exporting HAR
|
||||
- Clicking on product filters
|
||||
- Looking at individual product pages
|
||||
""")
|
||||
|
||||
def main():
|
||||
# Default HAR file location
|
||||
default_har = Path(__file__).parent / "pokemon_network.har"
|
||||
|
||||
# Check command line args
|
||||
if len(sys.argv) > 1:
|
||||
har_path = Path(sys.argv[1])
|
||||
else:
|
||||
har_path = default_har
|
||||
|
||||
if not har_path.exists():
|
||||
print("=" * 70)
|
||||
print("HAR File Analyzer")
|
||||
print("=" * 70)
|
||||
print(f"""
|
||||
No HAR file found at: {har_path}
|
||||
|
||||
To capture a HAR file:
|
||||
1. Open Chrome and go to: https://www.pokemoncenter.com/category/tcg-cards
|
||||
2. Press F12 to open DevTools
|
||||
3. Go to the Network tab
|
||||
4. Check "Preserve log" checkbox
|
||||
5. Refresh the page (F5)
|
||||
6. Scroll down to load more products
|
||||
7. Right-click in Network panel → "Save all as HAR with content"
|
||||
8. Save as: {default_har}
|
||||
9. Run this script again
|
||||
|
||||
Or specify a HAR file:
|
||||
python analyze_har.py path/to/your/file.har
|
||||
""")
|
||||
return
|
||||
|
||||
analyze_har(har_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,267 @@
|
||||
"""
|
||||
API Discovery Tool for Pokemon Center
|
||||
Attempts to find and test API endpoints that could be used instead of browser scraping.
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# Common API patterns to try
|
||||
BASE_URL = "https://www.pokemoncenter.com"
|
||||
|
||||
# Headers to mimic a real browser
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Referer": "https://www.pokemoncenter.com/",
|
||||
"Origin": "https://www.pokemoncenter.com",
|
||||
}
|
||||
|
||||
# Common API endpoint patterns for e-commerce sites
|
||||
API_PATTERNS = [
|
||||
# REST API patterns
|
||||
"/api/products",
|
||||
"/api/v1/products",
|
||||
"/api/v2/products",
|
||||
"/api/catalog/products",
|
||||
"/api/search",
|
||||
"/api/inventory",
|
||||
|
||||
# GraphQL
|
||||
"/graphql",
|
||||
"/api/graphql",
|
||||
|
||||
# Common e-commerce platforms
|
||||
"/rest/V1/products", # Magento
|
||||
"/_api/products", # Wix
|
||||
"/cdn/shop/products.json", # Shopify pattern
|
||||
"/products.json", # Shopify
|
||||
|
||||
# Search APIs
|
||||
"/api/search/products",
|
||||
"/search/suggest",
|
||||
"/api/autocomplete",
|
||||
|
||||
# Algolia (very common for e-commerce search)
|
||||
# Note: Algolia requires app ID and API key from the page
|
||||
]
|
||||
|
||||
# Pokemon TCG specific search terms
|
||||
SEARCH_TERMS = ["pokemon", "tcg", "cards", "booster", "etb"]
|
||||
|
||||
|
||||
def test_endpoint(url: str, method: str = "GET", data: dict = None) -> dict:
|
||||
"""Test an API endpoint"""
|
||||
try:
|
||||
if method == "GET":
|
||||
response = requests.get(url, headers=HEADERS, timeout=10)
|
||||
else:
|
||||
response = requests.post(url, headers=HEADERS, json=data, timeout=10)
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"status": response.status_code,
|
||||
"content_type": response.headers.get("content-type", ""),
|
||||
"size": len(response.content),
|
||||
"sample": response.text[:500] if response.status_code == 200 else None
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"url": url,
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
def discover_apis():
|
||||
"""Attempt to discover API endpoints"""
|
||||
print("=" * 60)
|
||||
print("Pokemon Center API Discovery")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
results = []
|
||||
|
||||
# Test common patterns
|
||||
print("Testing common API patterns...")
|
||||
for pattern in API_PATTERNS:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
results.append(result)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
print(f" Content-Type: {result['content_type']}")
|
||||
print(f" Size: {result['size']} bytes")
|
||||
elif result["status"] != "error" and result["status"] < 500:
|
||||
print(f" [{result['status']}] {url}")
|
||||
|
||||
# Test search with query params
|
||||
print()
|
||||
print("Testing search endpoints...")
|
||||
search_patterns = [
|
||||
"/api/search?q=pokemon",
|
||||
"/api/products?search=tcg",
|
||||
"/api/catalog?category=tcg-cards",
|
||||
"/search?q=pokemon+tcg",
|
||||
]
|
||||
|
||||
for pattern in search_patterns:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
results.append(result)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
|
||||
# Look for Algolia configuration
|
||||
print()
|
||||
print("Checking for Algolia search...")
|
||||
# Algolia is often exposed in page source
|
||||
try:
|
||||
response = requests.get(BASE_URL, headers=HEADERS, timeout=15)
|
||||
if "algolia" in response.text.lower():
|
||||
print(" [!] Algolia detected in page source")
|
||||
# Try to extract app ID and search key
|
||||
import re
|
||||
app_id = re.search(r'["\']?algolia[_-]?app[_-]?id["\']?\s*[:=]\s*["\']([A-Z0-9]+)["\']', response.text, re.I)
|
||||
api_key = re.search(r'["\']?algolia[_-]?(?:search[_-]?)?(?:api[_-]?)?key["\']?\s*[:=]\s*["\']([a-f0-9]+)["\']', response.text, re.I)
|
||||
|
||||
if app_id:
|
||||
print(f" App ID: {app_id.group(1)}")
|
||||
if api_key:
|
||||
print(f" Search Key: {api_key.group(1)}")
|
||||
|
||||
# Check for other API clues
|
||||
if "graphql" in response.text.lower():
|
||||
print(" [!] GraphQL detected in page source")
|
||||
|
||||
if "__NEXT_DATA__" in response.text:
|
||||
print(" [!] Next.js detected - may have API routes at /api/*")
|
||||
|
||||
if "window.__INITIAL_STATE__" in response.text or "window.__PRELOADED_STATE__" in response.text:
|
||||
print(" [!] Pre-rendered state detected - data may be in page source")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error checking main page: {e}")
|
||||
|
||||
# Summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Summary")
|
||||
print("=" * 60)
|
||||
|
||||
working = [r for r in results if r.get("status") == 200]
|
||||
if working:
|
||||
print(f"Found {len(working)} potentially working endpoints:")
|
||||
for r in working:
|
||||
print(f" - {r['url']}")
|
||||
else:
|
||||
print("No direct API endpoints found.")
|
||||
print()
|
||||
print("Alternative approaches to consider:")
|
||||
print(" 1. Monitor sitemap.xml for new products")
|
||||
print(" 2. Use Google Shopping API or similar aggregators")
|
||||
print(" 3. Check if they have an RSS feed")
|
||||
print(" 4. Use a service like Distill.io for change detection")
|
||||
print(" 5. Proxy rotation with residential IPs")
|
||||
print(" 6. Lower check frequency + add human-like delays")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def check_sitemap():
|
||||
"""Check sitemap for product URLs"""
|
||||
print()
|
||||
print("Checking sitemap...")
|
||||
|
||||
sitemap_urls = [
|
||||
"/sitemap.xml",
|
||||
"/sitemap_index.xml",
|
||||
"/sitemaps/sitemap.xml",
|
||||
"/robots.txt", # Often contains sitemap location
|
||||
]
|
||||
|
||||
for pattern in sitemap_urls:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
if "sitemap" in result.get("sample", "").lower():
|
||||
print(f" Contains sitemap references")
|
||||
if "product" in result.get("sample", "").lower():
|
||||
print(f" Contains product references")
|
||||
|
||||
|
||||
def check_rss():
|
||||
"""Check for RSS feeds"""
|
||||
print()
|
||||
print("Checking for RSS/Atom feeds...")
|
||||
|
||||
feed_urls = [
|
||||
"/feed",
|
||||
"/rss",
|
||||
"/feed.xml",
|
||||
"/rss.xml",
|
||||
"/atom.xml",
|
||||
"/blog/feed",
|
||||
"/news/feed",
|
||||
]
|
||||
|
||||
for pattern in feed_urls:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
|
||||
if result["status"] == 200 and ("xml" in result.get("content_type", "") or "rss" in result.get("content_type", "")):
|
||||
print(f" [OK] {url}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
discover_apis()
|
||||
check_sitemap()
|
||||
check_rss()
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Next Steps")
|
||||
print("=" * 60)
|
||||
print("""
|
||||
To avoid bot detection, consider these strategies:
|
||||
|
||||
1. API-BASED MONITORING (if endpoints found):
|
||||
- Call API endpoints directly with requests
|
||||
- Much faster and less detectable than browser
|
||||
- Can check more frequently
|
||||
|
||||
2. SITEMAP MONITORING:
|
||||
- Parse sitemap.xml periodically
|
||||
- Detect new product URLs without visiting pages
|
||||
- Very low detection risk
|
||||
|
||||
3. HASH-BASED CHANGE DETECTION:
|
||||
- Fetch page, hash content
|
||||
- Only alert when hash changes
|
||||
- Reduces unnecessary processing
|
||||
|
||||
4. RESIDENTIAL PROXY ROTATION:
|
||||
- Use services like Bright Data, Oxylabs
|
||||
- Rotate IPs to avoid blocks
|
||||
- More expensive but reliable
|
||||
|
||||
5. HUMAN-LIKE BEHAVIOR:
|
||||
- Random delays between 60-180 seconds
|
||||
- Vary user agent strings
|
||||
- Add mouse movements and scrolling
|
||||
- Use real browser cookies
|
||||
|
||||
6. THIRD-PARTY ALERTS:
|
||||
- Discord servers that track Pokemon Center
|
||||
- Stock alert services (NowInStock, etc.)
|
||||
- Browser extensions like Distill.io
|
||||
|
||||
Run this script to see what APIs are available:
|
||||
python api_discovery.py
|
||||
""")
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Pokemon Center API Monitor
|
||||
|
||||
Uses cookies from a browser session to make API calls directly.
|
||||
Much lighter than full browser scraping once cookies are established.
|
||||
|
||||
Workflow:
|
||||
1. Run warmup_session.py first to get valid cookies
|
||||
2. This script uses those cookies to call APIs directly
|
||||
3. Falls back to browser if cookies expire
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import pickle
|
||||
import logging
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Any
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Session/cookie storage
|
||||
SESSION_DIR = Path(__file__).parent / "sessions"
|
||||
COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
|
||||
|
||||
# API endpoints discovered from HAR analysis
|
||||
API_BASE = "https://www.pokemoncenter.com"
|
||||
ENDPOINTS = {
|
||||
"product": "/tpci-ecommweb-api/product/{sku}",
|
||||
"status": "/tpci-ecommweb-api/product/status/{encoded_id}",
|
||||
"category": "/site/resourceapi/category/{category}",
|
||||
"reviews": "/tpci-ecommweb-api/review/get-product-scores",
|
||||
}
|
||||
|
||||
# Required headers from HAR capture
|
||||
BASE_HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/category/tcg-cards",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
}
|
||||
|
||||
# Known product SKUs for TCG (build this list over time)
|
||||
KNOWN_TCG_SKUS_FILE = Path(__file__).parent / "known_tcg_skus.json"
|
||||
|
||||
|
||||
class PokemonCenterAPI:
|
||||
"""Direct API client for Pokemon Center using session cookies"""
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(BASE_HEADERS)
|
||||
self.cookies_loaded = False
|
||||
self.last_cookie_refresh = None
|
||||
|
||||
def load_cookies(self) -> bool:
|
||||
"""Load cookies from the saved session file"""
|
||||
if not COOKIE_FILE.exists():
|
||||
logger.warning(f"No cookie file found at {COOKIE_FILE}")
|
||||
logger.info("Run warmup_session.py first to create a valid session")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(COOKIE_FILE, 'rb') as f:
|
||||
cookies = pickle.load(f)
|
||||
|
||||
# Add cookies to session
|
||||
for cookie in cookies:
|
||||
self.session.cookies.set(
|
||||
cookie['name'],
|
||||
cookie['value'],
|
||||
domain=cookie.get('domain', '.pokemoncenter.com'),
|
||||
path=cookie.get('path', '/')
|
||||
)
|
||||
|
||||
logger.info(f"Loaded {len(cookies)} cookies from session file")
|
||||
self.cookies_loaded = True
|
||||
self.last_cookie_refresh = datetime.now()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cookies: {e}")
|
||||
return False
|
||||
|
||||
def _make_request(self, method: str, url: str, **kwargs) -> Optional[requests.Response]:
|
||||
"""Make a request with error handling"""
|
||||
try:
|
||||
if method.upper() == "GET":
|
||||
response = self.session.get(url, timeout=15, **kwargs)
|
||||
else:
|
||||
response = self.session.post(url, timeout=15, **kwargs)
|
||||
|
||||
# Check for blocking
|
||||
if response.status_code == 403:
|
||||
if "captcha" in response.text.lower() or "blocked" in response.text.lower():
|
||||
logger.warning("Request blocked - cookies may have expired")
|
||||
self.cookies_loaded = False
|
||||
return None
|
||||
|
||||
return response
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Request failed: {e}")
|
||||
return None
|
||||
|
||||
def get_product(self, sku: str) -> Optional[Dict]:
|
||||
"""Get product details by SKU"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['product'].format(sku=sku)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_product_status(self, encoded_id: str) -> Optional[Dict]:
|
||||
"""Get product availability status"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['status'].format(encoded_id=encoded_id)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_category(self, category: str = "new-releases") -> Optional[Dict]:
|
||||
"""Get category listing (potential goldmine for new drops!)"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
url = f"{API_BASE}{ENDPOINTS['category'].format(category=category)}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
def get_review_scores(self, sku_list: List[str]) -> Optional[Dict]:
|
||||
"""Get review scores for multiple SKUs"""
|
||||
if not self.cookies_loaded:
|
||||
if not self.load_cookies():
|
||||
return None
|
||||
|
||||
skus = ",".join(sku_list)
|
||||
url = f"{API_BASE}{ENDPOINTS['reviews']}?skuList={skus}"
|
||||
response = self._make_request("GET", url)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
|
||||
|
||||
def load_known_skus() -> List[str]:
|
||||
"""Load list of known TCG SKUs"""
|
||||
if KNOWN_TCG_SKUS_FILE.exists():
|
||||
with open(KNOWN_TCG_SKUS_FILE, 'r') as f:
|
||||
return json.load(f)
|
||||
return []
|
||||
|
||||
|
||||
def save_known_skus(skus: List[str]):
|
||||
"""Save list of known TCG SKUs"""
|
||||
with open(KNOWN_TCG_SKUS_FILE, 'w') as f:
|
||||
json.dump(skus, f, indent=2)
|
||||
|
||||
|
||||
def test_api():
|
||||
"""Test the API client"""
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Monitor Test")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
api = PokemonCenterAPI()
|
||||
|
||||
if not api.load_cookies():
|
||||
print("\nNo valid cookies found!")
|
||||
print("Please run: python warmup_session.py")
|
||||
print("Then try again.")
|
||||
return
|
||||
|
||||
print("\n[1] Testing category endpoint (new-releases)...")
|
||||
category_data = api.get_category("new-releases")
|
||||
if category_data:
|
||||
print(" SUCCESS! Category data retrieved.")
|
||||
print(f" Keys: {list(category_data.keys())[:5]}")
|
||||
|
||||
# Save for analysis
|
||||
with open("category_response.json", "w") as f:
|
||||
json.dump(category_data, f, indent=2)
|
||||
print(" Saved to category_response.json")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print("\n[2] Testing product endpoint...")
|
||||
# Try a known SKU from our HAR capture
|
||||
product_data = api.get_product("699-17113")
|
||||
if product_data:
|
||||
print(" SUCCESS! Product data retrieved.")
|
||||
print(f" Keys: {list(product_data.keys())[:5]}")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print("\n[3] Testing review scores endpoint...")
|
||||
reviews = api.get_review_scores(["699-17113", "191-85953"])
|
||||
if reviews:
|
||||
print(" SUCCESS! Review scores retrieved.")
|
||||
print(f" Data: {reviews}")
|
||||
else:
|
||||
print(" FAILED - cookies may have expired")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
if category_data or product_data:
|
||||
print("API access working! Can monitor without full browser scraping.")
|
||||
print("Cookies will expire eventually - re-run warmup when needed.")
|
||||
else:
|
||||
print("API blocked. Need fresh cookies from browser session.")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_api()
|
||||
@@ -0,0 +1,370 @@
|
||||
"""
|
||||
Pokemon Center Backend Monitor
|
||||
|
||||
Detects NEW products added to the API before they're publicly announced.
|
||||
This is how accounts like @pokepullzhq detect drops early.
|
||||
|
||||
Strategy:
|
||||
1. Maintain a list of all known product SKUs
|
||||
2. Periodically check the API for current products
|
||||
3. Compare: Any new SKUs = potential silent drop
|
||||
4. Alert immediately on new detections
|
||||
|
||||
Usage:
|
||||
1. First run warmup_session.py to get valid cookies
|
||||
2. Then run: python backend_monitor.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import pickle
|
||||
import logging
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, List, Set
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File paths
|
||||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
SESSION_DIR = DATA_DIR / "sessions"
|
||||
COOKIE_FILE = SESSION_DIR / "pokemoncenter_cookies.pkl"
|
||||
KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
|
||||
DETECTIONS_LOG = DATA_DIR / "detections.json"
|
||||
|
||||
# API configuration
|
||||
API_BASE = "https://www.pokemoncenter.com"
|
||||
|
||||
# Categories to monitor for new products
|
||||
MONITOR_CATEGORIES = [
|
||||
"new-releases",
|
||||
"tcg-cards",
|
||||
# Add more as needed
|
||||
]
|
||||
|
||||
# Headers required for API calls
|
||||
HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/",
|
||||
}
|
||||
|
||||
# Check interval (seconds)
|
||||
CHECK_INTERVAL = 60
|
||||
|
||||
|
||||
class BackendMonitor:
|
||||
"""Monitors Pokemon Center API for new product drops"""
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(HEADERS)
|
||||
self.known_skus: Set[str] = set()
|
||||
self.cookies_valid = False
|
||||
self.detections: List[Dict] = []
|
||||
|
||||
def load_cookies(self) -> bool:
|
||||
"""Load cookies from warmup session"""
|
||||
if not COOKIE_FILE.exists():
|
||||
logger.error(f"No cookie file found at {COOKIE_FILE}")
|
||||
logger.info("Run warmup_session.py first!")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(COOKIE_FILE, 'rb') as f:
|
||||
cookies = pickle.load(f)
|
||||
|
||||
for cookie in cookies:
|
||||
self.session.cookies.set(
|
||||
cookie['name'],
|
||||
cookie['value'],
|
||||
domain=cookie.get('domain', '.pokemoncenter.com'),
|
||||
path=cookie.get('path', '/')
|
||||
)
|
||||
|
||||
logger.info(f"Loaded {len(cookies)} cookies")
|
||||
self.cookies_valid = True
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cookies: {e}")
|
||||
return False
|
||||
|
||||
def load_known_skus(self):
|
||||
"""Load previously seen SKUs"""
|
||||
if KNOWN_SKUS_FILE.exists():
|
||||
with open(KNOWN_SKUS_FILE, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.known_skus = set(data.get('skus', []))
|
||||
logger.info(f"Loaded {len(self.known_skus)} known SKUs")
|
||||
else:
|
||||
logger.info("No known SKUs file - will create on first run")
|
||||
self.known_skus = set()
|
||||
|
||||
def save_known_skus(self):
|
||||
"""Save known SKUs to file"""
|
||||
with open(KNOWN_SKUS_FILE, 'w') as f:
|
||||
json.dump({
|
||||
'skus': list(self.known_skus),
|
||||
'last_updated': datetime.now().isoformat(),
|
||||
'count': len(self.known_skus)
|
||||
}, f, indent=2)
|
||||
|
||||
def log_detection(self, sku: str, product_info: Dict):
|
||||
"""Log a new product detection"""
|
||||
detection = {
|
||||
'sku': sku,
|
||||
'detected_at': datetime.now().isoformat(),
|
||||
'product_info': product_info
|
||||
}
|
||||
self.detections.append(detection)
|
||||
|
||||
# Append to detections log file
|
||||
detections = []
|
||||
if DETECTIONS_LOG.exists():
|
||||
with open(DETECTIONS_LOG, 'r') as f:
|
||||
detections = json.load(f)
|
||||
|
||||
detections.append(detection)
|
||||
|
||||
with open(DETECTIONS_LOG, 'w') as f:
|
||||
json.dump(detections, f, indent=2)
|
||||
|
||||
def get_category_products(self, category: str) -> Optional[Dict]:
|
||||
"""Fetch products from a category endpoint"""
|
||||
url = f"{API_BASE}/site/resourceapi/category/{category}"
|
||||
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 403:
|
||||
logger.warning("API blocked - cookies may have expired")
|
||||
self.cookies_valid = False
|
||||
return None
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
|
||||
logger.warning(f"Unexpected status {response.status_code} for {category}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching {category}: {e}")
|
||||
return None
|
||||
|
||||
def get_product_details(self, sku: str) -> Optional[Dict]:
|
||||
"""Get full details for a specific product"""
|
||||
url = f"{API_BASE}/tpci-ecommweb-api/product/{sku}"
|
||||
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching product {sku}: {e}")
|
||||
return None
|
||||
|
||||
def extract_skus_from_response(self, data: Dict) -> Set[str]:
|
||||
"""Extract product SKUs from API response"""
|
||||
skus = set()
|
||||
|
||||
# The response structure varies - try multiple approaches
|
||||
# This will need adjustment based on actual API response
|
||||
|
||||
def find_skus(obj, depth=0):
|
||||
"""Recursively find SKU-like values"""
|
||||
if depth > 10: # Prevent infinite recursion
|
||||
return
|
||||
|
||||
if isinstance(obj, dict):
|
||||
# Look for SKU fields
|
||||
for key in ['sku', 'skuCode', 'productId', 'id', 'code']:
|
||||
if key in obj:
|
||||
value = obj[key]
|
||||
if isinstance(value, str) and self._looks_like_sku(value):
|
||||
skus.add(value)
|
||||
|
||||
# Look in nested objects
|
||||
for value in obj.values():
|
||||
find_skus(value, depth + 1)
|
||||
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
find_skus(item, depth + 1)
|
||||
|
||||
find_skus(data)
|
||||
return skus
|
||||
|
||||
def _looks_like_sku(self, value: str) -> bool:
|
||||
"""Check if a string looks like a Pokemon Center SKU"""
|
||||
# SKUs we've seen: 699-17113, 191-85953, 10-10191-109
|
||||
if not value:
|
||||
return False
|
||||
|
||||
# Must contain digits and possibly hyphens
|
||||
has_digit = any(c.isdigit() for c in value)
|
||||
reasonable_length = 5 <= len(value) <= 20
|
||||
|
||||
return has_digit and reasonable_length
|
||||
|
||||
def check_for_new_products(self) -> List[Dict]:
|
||||
"""Main check - look for new SKUs across all categories"""
|
||||
if not self.cookies_valid:
|
||||
if not self.load_cookies():
|
||||
return []
|
||||
|
||||
new_products = []
|
||||
current_skus = set()
|
||||
|
||||
for category in MONITOR_CATEGORIES:
|
||||
logger.debug(f"Checking category: {category}")
|
||||
|
||||
data = self.get_category_products(category)
|
||||
if data:
|
||||
skus = self.extract_skus_from_response(data)
|
||||
current_skus.update(skus)
|
||||
logger.debug(f" Found {len(skus)} SKUs in {category}")
|
||||
|
||||
if not current_skus:
|
||||
logger.warning("No SKUs found - API may not be returning data")
|
||||
return []
|
||||
|
||||
# Find new SKUs
|
||||
new_skus = current_skus - self.known_skus
|
||||
|
||||
if new_skus:
|
||||
logger.info(f"!!! DETECTED {len(new_skus)} NEW SKU(s) !!!")
|
||||
|
||||
for sku in new_skus:
|
||||
# Get full product details
|
||||
details = self.get_product_details(sku)
|
||||
|
||||
product_info = {
|
||||
'sku': sku,
|
||||
'details': details,
|
||||
'detected_at': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Try to extract name from details
|
||||
name = "Unknown Product"
|
||||
if details:
|
||||
# Look for name in various places
|
||||
name = (
|
||||
details.get('name') or
|
||||
details.get('displayName') or
|
||||
details.get('definition', {}).get('display-name') or
|
||||
sku
|
||||
)
|
||||
|
||||
logger.info(f" NEW: {sku} - {name}")
|
||||
self.log_detection(sku, product_info)
|
||||
new_products.append(product_info)
|
||||
|
||||
# Add to known SKUs
|
||||
self.known_skus.add(sku)
|
||||
|
||||
# Save updated known SKUs
|
||||
self.save_known_skus()
|
||||
|
||||
else:
|
||||
logger.info(f"Check complete - {len(current_skus)} products, no new drops")
|
||||
|
||||
return new_products
|
||||
|
||||
def send_discord_alert(self, product: Dict):
|
||||
"""Send Discord notification for new product"""
|
||||
# Import from your existing discord_notifier
|
||||
try:
|
||||
from src.discord_notifier import send_notification
|
||||
# You'd format and send the alert here
|
||||
logger.info(f"Discord alert sent for {product['sku']}")
|
||||
except ImportError:
|
||||
logger.warning("Discord notifier not available")
|
||||
|
||||
def run(self):
|
||||
"""Main monitoring loop"""
|
||||
print("=" * 60)
|
||||
print("Pokemon Center Backend Monitor")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("This monitors for NEW products added to the API.")
|
||||
print("New SKUs = potential silent drops before announcement!")
|
||||
print()
|
||||
print(f"Check interval: {CHECK_INTERVAL} seconds")
|
||||
print(f"Monitoring categories: {', '.join(MONITOR_CATEGORIES)}")
|
||||
print()
|
||||
print("Press Ctrl+C to stop")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Load known SKUs
|
||||
self.load_known_skus()
|
||||
|
||||
# Load cookies
|
||||
if not self.load_cookies():
|
||||
print("\nERROR: No valid cookies!")
|
||||
print("Run: python warmup_session.py")
|
||||
return
|
||||
|
||||
# First check - populate known SKUs if empty
|
||||
if not self.known_skus:
|
||||
logger.info("First run - building initial SKU database...")
|
||||
self.check_for_new_products()
|
||||
logger.info(f"Baseline established with {len(self.known_skus)} products")
|
||||
print()
|
||||
|
||||
# Main loop
|
||||
check_count = 0
|
||||
while True:
|
||||
try:
|
||||
check_count += 1
|
||||
logger.info(f"--- Check #{check_count} ---")
|
||||
|
||||
new_products = self.check_for_new_products()
|
||||
|
||||
if new_products:
|
||||
print()
|
||||
print("!" * 60)
|
||||
print("!!! NEW PRODUCT DETECTED !!!")
|
||||
print("!" * 60)
|
||||
for p in new_products:
|
||||
print(f" SKU: {p['sku']}")
|
||||
# Send Discord alert
|
||||
self.send_discord_alert(p)
|
||||
print("!" * 60)
|
||||
print()
|
||||
|
||||
# Wait for next check
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopping monitor...")
|
||||
self.save_known_skus()
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in main loop: {e}")
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
|
||||
def main():
|
||||
monitor = BackendMonitor()
|
||||
monitor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
API Call Capture Tool for Pokemon Center
|
||||
|
||||
This script opens Pokemon Center and logs ALL network requests,
|
||||
helping identify backend APIs used for product data.
|
||||
|
||||
Usage:
|
||||
python capture_api_calls.py
|
||||
|
||||
Output:
|
||||
- api_captures.json: All captured API calls
|
||||
- Console output with interesting endpoints
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Capture Tool")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Check for selenium
|
||||
try:
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||
except ImportError:
|
||||
print("ERROR: selenium not installed")
|
||||
print("Run: pip install selenium")
|
||||
exit(1)
|
||||
|
||||
# Storage for captured requests
|
||||
captured_requests = []
|
||||
interesting_patterns = [
|
||||
r'api',
|
||||
r'graphql',
|
||||
r'product',
|
||||
r'catalog',
|
||||
r'search',
|
||||
r'inventory',
|
||||
r'stock',
|
||||
r'algolia',
|
||||
r'contentful',
|
||||
r'commercetools',
|
||||
r'\.json',
|
||||
]
|
||||
|
||||
def is_interesting(url):
|
||||
"""Check if URL matches patterns we care about"""
|
||||
url_lower = url.lower()
|
||||
for pattern in interesting_patterns:
|
||||
if re.search(pattern, url_lower):
|
||||
return True
|
||||
return False
|
||||
|
||||
def setup_browser():
|
||||
"""Setup Chrome with network logging enabled"""
|
||||
options = Options()
|
||||
options.add_argument("--disable-blink-features=AutomationControlled")
|
||||
options.add_argument("--window-size=1920,1080")
|
||||
|
||||
# Enable performance logging to capture network requests
|
||||
options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
|
||||
|
||||
driver = webdriver.Chrome(options=options)
|
||||
return driver
|
||||
|
||||
def extract_network_requests(driver):
|
||||
"""Extract network requests from Chrome performance logs"""
|
||||
logs = driver.get_log('performance')
|
||||
requests = []
|
||||
|
||||
for entry in logs:
|
||||
try:
|
||||
log = json.loads(entry['message'])['message']
|
||||
|
||||
# We want Network.requestWillBeSent and Network.responseReceived
|
||||
if log['method'] == 'Network.requestWillBeSent':
|
||||
request = log['params']['request']
|
||||
requests.append({
|
||||
'type': 'request',
|
||||
'url': request.get('url', ''),
|
||||
'method': request.get('method', ''),
|
||||
'headers': request.get('headers', {}),
|
||||
'postData': request.get('postData', None),
|
||||
'timestamp': entry['timestamp']
|
||||
})
|
||||
|
||||
elif log['method'] == 'Network.responseReceived':
|
||||
response = log['params']['response']
|
||||
requests.append({
|
||||
'type': 'response',
|
||||
'url': response.get('url', ''),
|
||||
'status': response.get('status', 0),
|
||||
'mimeType': response.get('mimeType', ''),
|
||||
'headers': response.get('headers', {}),
|
||||
'timestamp': entry['timestamp']
|
||||
})
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return requests
|
||||
|
||||
def analyze_requests(requests):
|
||||
"""Analyze captured requests for interesting patterns"""
|
||||
api_calls = []
|
||||
seen_urls = set()
|
||||
|
||||
for req in requests:
|
||||
url = req.get('url', '')
|
||||
|
||||
# Skip if already seen or not interesting
|
||||
if url in seen_urls:
|
||||
continue
|
||||
if not is_interesting(url):
|
||||
continue
|
||||
|
||||
# Skip common non-API resources
|
||||
if any(ext in url for ext in ['.png', '.jpg', '.gif', '.css', '.woff', '.svg', '.ico']):
|
||||
continue
|
||||
|
||||
seen_urls.add(url)
|
||||
api_calls.append(req)
|
||||
|
||||
return api_calls
|
||||
|
||||
def main():
|
||||
driver = None
|
||||
|
||||
try:
|
||||
print("Starting Chrome with network logging...")
|
||||
driver = setup_browser()
|
||||
|
||||
print("Navigating to Pokemon Center TCG page...")
|
||||
print("(This may trigger a CAPTCHA - solve it if needed)")
|
||||
print()
|
||||
|
||||
# Navigate to the TCG category
|
||||
driver.get("https://www.pokemoncenter.com/category/tcg-cards")
|
||||
|
||||
print("Waiting for page to load...")
|
||||
time.sleep(10)
|
||||
|
||||
# Scroll down to trigger lazy loading
|
||||
print("Scrolling to load more content...")
|
||||
for i in range(3):
|
||||
driver.execute_script("window.scrollBy(0, 800);")
|
||||
time.sleep(2)
|
||||
|
||||
# Wait a bit more
|
||||
time.sleep(5)
|
||||
|
||||
print()
|
||||
print("Extracting network requests...")
|
||||
requests = extract_network_requests(driver)
|
||||
print(f"Captured {len(requests)} total network events")
|
||||
|
||||
# Analyze for interesting APIs
|
||||
api_calls = analyze_requests(requests)
|
||||
print(f"Found {len(api_calls)} potentially interesting API calls")
|
||||
|
||||
# Save all captures
|
||||
output_file = Path(__file__).parent / "api_captures.json"
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump({
|
||||
'captured_at': datetime.now().isoformat(),
|
||||
'page_url': driver.current_url,
|
||||
'total_requests': len(requests),
|
||||
'interesting_calls': api_calls,
|
||||
'all_requests': requests
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"\nSaved full capture to: {output_file}")
|
||||
|
||||
# Display interesting findings
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("INTERESTING API CALLS FOUND")
|
||||
print("=" * 70)
|
||||
|
||||
if not api_calls:
|
||||
print("No obvious API calls detected.")
|
||||
print("This could mean:")
|
||||
print(" 1. Data is server-rendered (no client API)")
|
||||
print(" 2. API calls use non-standard paths")
|
||||
print(" 3. Bot protection blocked the content")
|
||||
else:
|
||||
for call in api_calls:
|
||||
url = call.get('url', '')
|
||||
method = call.get('method', 'GET')
|
||||
status = call.get('status', '-')
|
||||
mime = call.get('mimeType', '')
|
||||
|
||||
# Truncate long URLs
|
||||
display_url = url[:100] + '...' if len(url) > 100 else url
|
||||
|
||||
print(f"\n[{method}] {display_url}")
|
||||
if status != '-':
|
||||
print(f" Status: {status}, Type: {mime}")
|
||||
|
||||
# Check for key patterns
|
||||
if 'graphql' in url.lower():
|
||||
print(" ⚡ GraphQL endpoint!")
|
||||
if 'algolia' in url.lower():
|
||||
print(" 🔍 Algolia search!")
|
||||
if 'product' in url.lower():
|
||||
print(" 📦 Product data!")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("NEXT STEPS")
|
||||
print("=" * 70)
|
||||
print("""
|
||||
1. Review api_captures.json for the full data
|
||||
2. Look for JSON responses containing product info
|
||||
3. Test promising endpoints with curl/requests
|
||||
4. Note any required headers (auth tokens, API keys)
|
||||
|
||||
To test an endpoint manually:
|
||||
curl -H "User-Agent: Mozilla/5.0..." "https://api.example.com/endpoint"
|
||||
""")
|
||||
|
||||
# Keep browser open for manual inspection
|
||||
print()
|
||||
input("Browser is still open. Press ENTER to close and exit...")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if driver:
|
||||
driver.quit()
|
||||
print("Browser closed.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,415 @@
|
||||
"""
|
||||
Smart Pokemon Center Monitor
|
||||
|
||||
Uses stealth browser with adaptive modes and human-like behavior.
|
||||
Detects new products before they're announced.
|
||||
|
||||
Modes:
|
||||
- STEALTH: Normal monitoring (~1 min intervals, human-like)
|
||||
- ALERT: Fast checking when new product detected (~20 sec)
|
||||
- COOLDOWN: Gradual return to stealth after alert
|
||||
|
||||
Usage:
|
||||
python smart_monitor.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, Set, List, Dict
|
||||
from enum import Enum
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s [%(levelname)s] %(message)s',
|
||||
datefmt='%H:%M:%S'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File paths
|
||||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
KNOWN_SKUS_FILE = DATA_DIR / "known_skus.json"
|
||||
DETECTIONS_FILE = DATA_DIR / "detections.json"
|
||||
|
||||
# URLs to monitor
|
||||
MONITOR_URLS = [
|
||||
"https://www.pokemoncenter.com/category/tcg-cards?sort=newest",
|
||||
"https://www.pokemoncenter.com/category/new-releases",
|
||||
]
|
||||
|
||||
|
||||
class MonitorMode(Enum):
|
||||
STEALTH = "stealth"
|
||||
ALERT = "alert"
|
||||
COOLDOWN = "cooldown"
|
||||
|
||||
|
||||
class SmartMonitor:
|
||||
"""Adaptive monitor with human-like behavior"""
|
||||
|
||||
def __init__(self):
|
||||
self.browser = None
|
||||
self.known_skus: Set[str] = set()
|
||||
self.mode = MonitorMode.STEALTH
|
||||
self.alert_triggered_at: Optional[datetime] = None
|
||||
self.check_count = 0
|
||||
self.last_detection: Optional[Dict] = None
|
||||
|
||||
# Mode timing settings
|
||||
self.timing = {
|
||||
MonitorMode.STEALTH: (45, 90), # 45-90 seconds
|
||||
MonitorMode.ALERT: (15, 30), # 15-30 seconds
|
||||
MonitorMode.COOLDOWN: (60, 120), # 60-120 seconds
|
||||
}
|
||||
|
||||
# Alert mode duration
|
||||
self.alert_duration = timedelta(minutes=30)
|
||||
self.cooldown_duration = timedelta(minutes=15)
|
||||
|
||||
def start_browser(self):
|
||||
"""Start the stealth browser"""
|
||||
if self.browser:
|
||||
return
|
||||
|
||||
logger.info("Starting stealth browser...")
|
||||
|
||||
try:
|
||||
from stealth_browser import StealthBrowser
|
||||
self.browser = StealthBrowser(
|
||||
headless=False,
|
||||
session_name="smart_monitor"
|
||||
)
|
||||
self.browser.start()
|
||||
logger.info("Browser started successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start browser: {e}")
|
||||
raise
|
||||
|
||||
def stop_browser(self):
|
||||
"""Stop the browser"""
|
||||
if self.browser:
|
||||
logger.info("Stopping browser...")
|
||||
self.browser.stop()
|
||||
self.browser = None
|
||||
|
||||
def load_known_skus(self):
|
||||
"""Load previously seen SKUs"""
|
||||
if KNOWN_SKUS_FILE.exists():
|
||||
with open(KNOWN_SKUS_FILE, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.known_skus = set(data.get('skus', []))
|
||||
logger.info(f"Loaded {len(self.known_skus)} known SKUs")
|
||||
else:
|
||||
self.known_skus = set()
|
||||
logger.info("No known SKUs file - starting fresh")
|
||||
|
||||
def save_known_skus(self):
|
||||
"""Save known SKUs"""
|
||||
with open(KNOWN_SKUS_FILE, 'w') as f:
|
||||
json.dump({
|
||||
'skus': list(self.known_skus),
|
||||
'updated': datetime.now().isoformat(),
|
||||
'count': len(self.known_skus)
|
||||
}, f, indent=2)
|
||||
|
||||
def log_detection(self, sku: str, name: str, url: str):
|
||||
"""Log a new product detection"""
|
||||
detection = {
|
||||
'sku': sku,
|
||||
'name': name,
|
||||
'url': url,
|
||||
'detected_at': datetime.now().isoformat(),
|
||||
'mode': self.mode.value
|
||||
}
|
||||
|
||||
self.last_detection = detection
|
||||
|
||||
# Load existing detections
|
||||
detections = []
|
||||
if DETECTIONS_FILE.exists():
|
||||
with open(DETECTIONS_FILE, 'r') as f:
|
||||
detections = json.load(f)
|
||||
|
||||
detections.append(detection)
|
||||
|
||||
with open(DETECTIONS_FILE, 'w') as f:
|
||||
json.dump(detections, f, indent=2)
|
||||
|
||||
logger.info(f"Detection logged: {sku}")
|
||||
|
||||
def get_interval(self) -> float:
|
||||
"""Get randomized check interval based on current mode"""
|
||||
min_sec, max_sec = self.timing[self.mode]
|
||||
|
||||
# Add gaussian jitter for more natural timing
|
||||
base = (min_sec + max_sec) / 2
|
||||
jitter = random.gauss(0, (max_sec - min_sec) / 4)
|
||||
interval = base + jitter
|
||||
|
||||
# Clamp to bounds
|
||||
return max(min_sec * 0.8, min(max_sec * 1.2, interval))
|
||||
|
||||
def maybe_do_human_action(self):
|
||||
"""Occasionally do something human-like"""
|
||||
if not self.browser or not self.browser.driver:
|
||||
return
|
||||
|
||||
action = random.random()
|
||||
|
||||
if action < 0.3:
|
||||
# Scroll randomly
|
||||
self.browser.human_scroll()
|
||||
elif action < 0.4:
|
||||
# Small mouse movement
|
||||
self.browser.human_mouse_move()
|
||||
elif action < 0.45:
|
||||
# Longer pause (human distraction)
|
||||
pause = random.uniform(3, 8)
|
||||
logger.debug(f"Human pause: {pause:.1f}s")
|
||||
time.sleep(pause)
|
||||
|
||||
def maybe_take_break(self) -> bool:
|
||||
"""Occasionally take a longer break"""
|
||||
# 3% chance of a break
|
||||
if random.random() < 0.03:
|
||||
break_time = random.randint(120, 300) # 2-5 minutes
|
||||
logger.info(f"Taking a break for {break_time}s (human-like pause)")
|
||||
time.sleep(break_time)
|
||||
return True
|
||||
return False
|
||||
|
||||
def extract_skus_from_page(self) -> Set[str]:
|
||||
"""Extract product SKUs from the current page"""
|
||||
if not self.browser or not self.browser.driver:
|
||||
return set()
|
||||
|
||||
skus = set()
|
||||
|
||||
try:
|
||||
# Get page source and parse
|
||||
from bs4 import BeautifulSoup
|
||||
html = self.browser.driver.page_source
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# Look for product links - Pokemon Center format: /product/SKU/name
|
||||
import re
|
||||
product_links = soup.select('a[href*="/product/"]')
|
||||
|
||||
for link in product_links:
|
||||
href = link.get('href', '')
|
||||
# Extract SKU from URL like /product/699-17113/product-name
|
||||
match = re.search(r'/product/([0-9]+-?[0-9]+)', href)
|
||||
if match:
|
||||
skus.add(match.group(1))
|
||||
|
||||
# Also look for data attributes
|
||||
for elem in soup.select('[data-sku], [data-product-id]'):
|
||||
sku = elem.get('data-sku') or elem.get('data-product-id')
|
||||
if sku and re.match(r'^[0-9]+-?[0-9]+', sku):
|
||||
skus.add(sku)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting SKUs: {e}")
|
||||
|
||||
return skus
|
||||
|
||||
def check_page(self, url: str) -> Set[str]:
|
||||
"""Load a page and extract SKUs"""
|
||||
if not self.browser:
|
||||
self.start_browser()
|
||||
|
||||
try:
|
||||
# Human-like delay before navigation
|
||||
self.browser.human_delay(0.5, 2.0)
|
||||
|
||||
# Navigate
|
||||
logger.debug(f"Loading: {url}")
|
||||
self.browser.driver.get(url)
|
||||
|
||||
# Wait for page load
|
||||
time.sleep(random.uniform(3, 6))
|
||||
|
||||
# Check for CAPTCHA
|
||||
if self.browser.check_for_captcha():
|
||||
logger.warning("CAPTCHA detected!")
|
||||
print("\n" + "!" * 50)
|
||||
print("CAPTCHA DETECTED - Please solve it in the browser")
|
||||
print("!" * 50 + "\n")
|
||||
self.browser.wait_for_captcha_solve(timeout=120)
|
||||
|
||||
# Human actions
|
||||
self.maybe_do_human_action()
|
||||
|
||||
# Extract SKUs
|
||||
skus = self.extract_skus_from_page()
|
||||
|
||||
return skus
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking page: {e}")
|
||||
return set()
|
||||
|
||||
def update_mode(self):
|
||||
"""Update monitoring mode based on state"""
|
||||
now = datetime.now()
|
||||
|
||||
if self.mode == MonitorMode.ALERT:
|
||||
# Check if alert period is over
|
||||
if self.alert_triggered_at:
|
||||
elapsed = now - self.alert_triggered_at
|
||||
if elapsed > self.alert_duration:
|
||||
logger.info("Alert period over, entering cooldown")
|
||||
self.mode = MonitorMode.COOLDOWN
|
||||
|
||||
elif self.mode == MonitorMode.COOLDOWN:
|
||||
# Check if cooldown is over
|
||||
if self.alert_triggered_at:
|
||||
elapsed = now - self.alert_triggered_at
|
||||
if elapsed > (self.alert_duration + self.cooldown_duration):
|
||||
logger.info("Cooldown over, returning to stealth mode")
|
||||
self.mode = MonitorMode.STEALTH
|
||||
self.alert_triggered_at = None
|
||||
|
||||
def trigger_alert_mode(self):
|
||||
"""Switch to alert mode"""
|
||||
logger.info("!!! ENTERING ALERT MODE - Faster checks !!!")
|
||||
self.mode = MonitorMode.ALERT
|
||||
self.alert_triggered_at = datetime.now()
|
||||
|
||||
def send_discord_alert(self, sku: str, name: str, url: str):
|
||||
"""Send Discord notification"""
|
||||
try:
|
||||
from src.discord_notifier import DiscordNotifier
|
||||
from config import DISCORD_WEBHOOK_URL
|
||||
|
||||
if DISCORD_WEBHOOK_URL and DISCORD_WEBHOOK_URL != "YOUR_WEBHOOK_URL_HERE":
|
||||
notifier = DiscordNotifier(DISCORD_WEBHOOK_URL)
|
||||
# Create a simple product dict
|
||||
product = {
|
||||
'name': name,
|
||||
'url': f"https://www.pokemoncenter.com/product/{sku}",
|
||||
'price': 'Check site',
|
||||
'site': 'pokemoncenter',
|
||||
}
|
||||
notifier.send_stock_alert(product, "NEW BACKEND DETECTION")
|
||||
logger.info("Discord alert sent!")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not send Discord alert: {e}")
|
||||
|
||||
def run_check(self) -> List[str]:
|
||||
"""Run a single check across all monitored URLs"""
|
||||
self.check_count += 1
|
||||
all_skus = set()
|
||||
new_skus = []
|
||||
|
||||
logger.info(f"Check #{self.check_count} | Mode: {self.mode.value.upper()}")
|
||||
|
||||
for url in MONITOR_URLS:
|
||||
skus = self.check_page(url)
|
||||
all_skus.update(skus)
|
||||
|
||||
# Brief pause between pages
|
||||
if url != MONITOR_URLS[-1]:
|
||||
time.sleep(random.uniform(2, 5))
|
||||
|
||||
logger.info(f"Found {len(all_skus)} total SKUs")
|
||||
|
||||
# Find new SKUs
|
||||
if self.known_skus: # Only check if we have a baseline
|
||||
new = all_skus - self.known_skus
|
||||
if new:
|
||||
for sku in new:
|
||||
logger.info(f"!!! NEW SKU DETECTED: {sku}")
|
||||
new_skus.append(sku)
|
||||
|
||||
# Log and alert
|
||||
url = f"https://www.pokemoncenter.com/product/{sku}"
|
||||
self.log_detection(sku, f"New Product {sku}", url)
|
||||
self.send_discord_alert(sku, f"New Product {sku}", url)
|
||||
|
||||
# Trigger alert mode
|
||||
self.trigger_alert_mode()
|
||||
|
||||
# Update known SKUs
|
||||
self.known_skus.update(all_skus)
|
||||
self.save_known_skus()
|
||||
|
||||
return new_skus
|
||||
|
||||
def run(self):
|
||||
"""Main monitoring loop"""
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(" SMART POKEMON CENTER MONITOR")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("Modes:")
|
||||
print(f" STEALTH: {self.timing[MonitorMode.STEALTH]} sec (normal)")
|
||||
print(f" ALERT: {self.timing[MonitorMode.ALERT]} sec (after detection)")
|
||||
print(f" COOLDOWN: {self.timing[MonitorMode.COOLDOWN]} sec (transition)")
|
||||
print()
|
||||
print("Press Ctrl+C to stop")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
try:
|
||||
# Initialize
|
||||
self.load_known_skus()
|
||||
self.start_browser()
|
||||
|
||||
# First check - build baseline if needed
|
||||
if not self.known_skus:
|
||||
logger.info("First run - building SKU baseline...")
|
||||
self.run_check()
|
||||
logger.info(f"Baseline: {len(self.known_skus)} products")
|
||||
print()
|
||||
|
||||
# Main loop
|
||||
while True:
|
||||
# Maybe take a break
|
||||
if self.maybe_take_break():
|
||||
continue
|
||||
|
||||
# Run check
|
||||
new_skus = self.run_check()
|
||||
|
||||
if new_skus:
|
||||
print()
|
||||
print("!" * 60)
|
||||
print("!!! NEW PRODUCT(S) DETECTED !!!")
|
||||
for sku in new_skus:
|
||||
print(f" -> {sku}")
|
||||
print("!" * 60)
|
||||
print()
|
||||
|
||||
# Update mode
|
||||
self.update_mode()
|
||||
|
||||
# Get interval and wait
|
||||
interval = self.get_interval()
|
||||
logger.info(f"Next check in {interval:.0f}s")
|
||||
time.sleep(interval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nStopping monitor...")
|
||||
except Exception as e:
|
||||
logger.error(f"Monitor error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
self.save_known_skus()
|
||||
self.stop_browser()
|
||||
print("Monitor stopped.")
|
||||
|
||||
|
||||
def main():
|
||||
monitor = SmartMonitor()
|
||||
monitor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,539 @@
|
||||
"""
|
||||
Stealth Browser Module
|
||||
Uses undetected-chromedriver to bypass bot detection (Imperva, Cloudflare, etc.)
|
||||
Includes session persistence, proxy rotation, and human-like behavior.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Data directory
|
||||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
|
||||
# Session storage directory
|
||||
SESSION_DIR = DATA_DIR / "sessions"
|
||||
SESSION_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Proxy configuration file
|
||||
PROXY_FILE = DATA_DIR / "proxies.json"
|
||||
|
||||
|
||||
class ProxyRotator:
|
||||
"""Manages proxy rotation for requests"""
|
||||
|
||||
def __init__(self, proxy_file: Path = PROXY_FILE):
|
||||
self.proxies: List[Dict] = []
|
||||
self.current_index = 0
|
||||
self.failed_proxies: Dict[str, datetime] = {}
|
||||
self.cooldown_minutes = 30
|
||||
|
||||
self._load_proxies(proxy_file)
|
||||
|
||||
def _load_proxies(self, proxy_file: Path):
|
||||
"""Load proxies from configuration file"""
|
||||
if proxy_file.exists():
|
||||
try:
|
||||
with open(proxy_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
self.proxies = data.get("proxies", [])
|
||||
logger.info(f"Loaded {len(self.proxies)} proxies")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load proxies: {e}")
|
||||
|
||||
def get_proxy(self) -> Optional[Dict]:
|
||||
"""Get next available proxy"""
|
||||
if not self.proxies:
|
||||
return None
|
||||
|
||||
# Clean up expired cooldowns
|
||||
now = datetime.now()
|
||||
self.failed_proxies = {
|
||||
k: v for k, v in self.failed_proxies.items()
|
||||
if now - v < timedelta(minutes=self.cooldown_minutes)
|
||||
}
|
||||
|
||||
# Find next working proxy
|
||||
attempts = 0
|
||||
while attempts < len(self.proxies):
|
||||
proxy = self.proxies[self.current_index]
|
||||
proxy_key = f"{proxy.get('host')}:{proxy.get('port')}"
|
||||
|
||||
self.current_index = (self.current_index + 1) % len(self.proxies)
|
||||
|
||||
if proxy_key not in self.failed_proxies:
|
||||
return proxy
|
||||
|
||||
attempts += 1
|
||||
|
||||
# All proxies in cooldown, return first one anyway
|
||||
return self.proxies[0] if self.proxies else None
|
||||
|
||||
def mark_failed(self, proxy: Dict):
|
||||
"""Mark a proxy as failed (temporary cooldown)"""
|
||||
if proxy:
|
||||
proxy_key = f"{proxy.get('host')}:{proxy.get('port')}"
|
||||
self.failed_proxies[proxy_key] = datetime.now()
|
||||
logger.warning(f"Proxy {proxy_key} marked as failed")
|
||||
|
||||
|
||||
class StealthBrowser:
|
||||
"""
|
||||
Stealth browser using undetected-chromedriver.
|
||||
Designed to bypass Imperva/Incapsula and similar bot protection.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
headless: bool = False,
|
||||
proxy: Optional[Dict] = None,
|
||||
user_data_dir: Optional[str] = None,
|
||||
session_name: str = "default"
|
||||
):
|
||||
self.headless = headless
|
||||
self.proxy = proxy
|
||||
self.user_data_dir = user_data_dir
|
||||
self.session_name = session_name
|
||||
self.driver = None
|
||||
self._setup_complete = False
|
||||
|
||||
def _get_chrome_options(self):
|
||||
"""Configure Chrome options for stealth"""
|
||||
import undetected_chromedriver as uc
|
||||
|
||||
options = uc.ChromeOptions()
|
||||
|
||||
# Basic stealth settings - compatible with newer Chrome versions
|
||||
options.add_argument("--disable-blink-features=AutomationControlled")
|
||||
options.add_argument("--disable-dev-shm-usage")
|
||||
options.add_argument("--no-sandbox")
|
||||
options.add_argument("--disable-infobars")
|
||||
|
||||
# Window size (realistic resolution)
|
||||
options.add_argument("--window-size=1920,1080")
|
||||
|
||||
# User data directory for session persistence
|
||||
if self.user_data_dir:
|
||||
options.add_argument(f"--user-data-dir={self.user_data_dir}")
|
||||
|
||||
# Proxy configuration
|
||||
if self.proxy:
|
||||
proxy_str = self._format_proxy(self.proxy)
|
||||
if proxy_str:
|
||||
options.add_argument(f"--proxy-server={proxy_str}")
|
||||
|
||||
# Headless mode (note: more detectable)
|
||||
if self.headless:
|
||||
options.add_argument("--headless=new")
|
||||
|
||||
return options
|
||||
|
||||
def _format_proxy(self, proxy: Dict) -> Optional[str]:
|
||||
"""Format proxy dict into Chrome proxy string"""
|
||||
if not proxy:
|
||||
return None
|
||||
|
||||
host = proxy.get("host")
|
||||
port = proxy.get("port")
|
||||
|
||||
if not host or not port:
|
||||
return None
|
||||
|
||||
protocol = proxy.get("protocol", "http")
|
||||
return f"{protocol}://{host}:{port}"
|
||||
|
||||
def start(self):
|
||||
"""Start the browser"""
|
||||
if self.driver:
|
||||
return
|
||||
|
||||
try:
|
||||
import undetected_chromedriver as uc
|
||||
|
||||
options = self._get_chrome_options()
|
||||
|
||||
# Create driver with version_main to match installed Chrome version
|
||||
# This prevents "ChromeDriver only supports Chrome version X" errors
|
||||
self.driver = uc.Chrome(
|
||||
options=options,
|
||||
use_subprocess=True,
|
||||
version_main=146, # Match user's Chrome version
|
||||
)
|
||||
|
||||
# Set realistic viewport
|
||||
self.driver.set_window_size(1920, 1080)
|
||||
|
||||
# Load saved cookies if they exist
|
||||
self._load_cookies()
|
||||
|
||||
self._setup_complete = True
|
||||
logger.info("Stealth browser started successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start stealth browser: {e}")
|
||||
raise
|
||||
|
||||
def stop(self):
|
||||
"""Stop the browser and save session"""
|
||||
if self.driver:
|
||||
try:
|
||||
self._save_cookies()
|
||||
self.driver.quit()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error stopping browser: {e}")
|
||||
finally:
|
||||
self.driver = None
|
||||
self._setup_complete = False
|
||||
|
||||
def _get_cookie_file(self) -> Path:
|
||||
"""Get path to cookie file for this session"""
|
||||
return SESSION_DIR / f"{self.session_name}_cookies.pkl"
|
||||
|
||||
def _save_cookies(self):
|
||||
"""Save cookies to file for session persistence"""
|
||||
if not self.driver:
|
||||
return
|
||||
|
||||
try:
|
||||
cookies = self.driver.get_cookies()
|
||||
cookie_file = self._get_cookie_file()
|
||||
|
||||
with open(cookie_file, 'wb') as f:
|
||||
pickle.dump(cookies, f)
|
||||
|
||||
logger.debug(f"Saved {len(cookies)} cookies to {cookie_file}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save cookies: {e}")
|
||||
|
||||
def _load_cookies(self):
|
||||
"""Load cookies from file"""
|
||||
cookie_file = self._get_cookie_file()
|
||||
|
||||
if not cookie_file.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
with open(cookie_file, 'rb') as f:
|
||||
cookies = pickle.load(f)
|
||||
|
||||
# Need to visit domain first before adding cookies
|
||||
# This will be done when navigating to the actual page
|
||||
|
||||
self._pending_cookies = cookies
|
||||
logger.debug(f"Loaded {len(cookies)} cookies from {cookie_file}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load cookies: {e}")
|
||||
self._pending_cookies = []
|
||||
|
||||
def _apply_pending_cookies(self, domain: str):
|
||||
"""Apply loaded cookies after visiting domain"""
|
||||
if not hasattr(self, '_pending_cookies') or not self._pending_cookies:
|
||||
return
|
||||
|
||||
for cookie in self._pending_cookies:
|
||||
try:
|
||||
# Only add cookies for matching domain
|
||||
if domain in cookie.get('domain', ''):
|
||||
self.driver.add_cookie(cookie)
|
||||
except Exception:
|
||||
pass # Some cookies may fail, that's ok
|
||||
|
||||
self._pending_cookies = []
|
||||
|
||||
def human_delay(self, min_seconds: float = 1.0, max_seconds: float = 3.0):
|
||||
"""Add human-like random delay"""
|
||||
delay = random.uniform(min_seconds, max_seconds)
|
||||
time.sleep(delay)
|
||||
|
||||
def human_scroll(self):
|
||||
"""Scroll like a human would"""
|
||||
if not self.driver:
|
||||
return
|
||||
|
||||
# Random scroll amount
|
||||
scroll_amount = random.randint(200, 600)
|
||||
|
||||
# Smooth scroll
|
||||
self.driver.execute_script(f"""
|
||||
window.scrollBy({{
|
||||
top: {scroll_amount},
|
||||
behavior: 'smooth'
|
||||
}});
|
||||
""")
|
||||
|
||||
self.human_delay(0.5, 1.5)
|
||||
|
||||
def human_mouse_move(self):
|
||||
"""Simulate mouse movement (basic)"""
|
||||
if not self.driver:
|
||||
return
|
||||
|
||||
# Move mouse to random position
|
||||
try:
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
|
||||
actions = ActionChains(self.driver)
|
||||
|
||||
# Random coordinates within viewport
|
||||
x = random.randint(100, 800)
|
||||
y = random.randint(100, 600)
|
||||
|
||||
# Move by offset from current position
|
||||
actions.move_by_offset(x, y).perform()
|
||||
|
||||
# Reset position
|
||||
actions.move_by_offset(-x, -y).perform()
|
||||
except Exception:
|
||||
pass # Mouse movement is optional
|
||||
|
||||
def get_page(self, url: str, wait_time: float = 5.0) -> str:
|
||||
"""
|
||||
Navigate to URL with human-like behavior.
|
||||
|
||||
Args:
|
||||
url: URL to navigate to
|
||||
wait_time: Time to wait for page load
|
||||
|
||||
Returns:
|
||||
Page HTML content
|
||||
"""
|
||||
if not self.driver:
|
||||
self.start()
|
||||
|
||||
try:
|
||||
# Pre-navigation delay
|
||||
self.human_delay(0.5, 1.5)
|
||||
|
||||
# Navigate
|
||||
self.driver.get(url)
|
||||
|
||||
# Apply any pending cookies
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(url).netloc
|
||||
self._apply_pending_cookies(domain)
|
||||
|
||||
# Wait for page load
|
||||
time.sleep(wait_time)
|
||||
|
||||
# Human-like behavior
|
||||
self.human_scroll()
|
||||
self.human_delay(1, 2)
|
||||
self.human_mouse_move()
|
||||
|
||||
# Get page content
|
||||
html = self.driver.page_source
|
||||
|
||||
# Save cookies after successful page load
|
||||
self._save_cookies()
|
||||
|
||||
return html
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting page {url}: {e}")
|
||||
raise
|
||||
|
||||
def check_for_captcha(self) -> bool:
|
||||
"""Check if page has a CAPTCHA challenge blocking content"""
|
||||
if not self.driver:
|
||||
return False
|
||||
|
||||
page_source = self.driver.page_source.lower()
|
||||
|
||||
# First check: Did the page load actual content?
|
||||
# If we see product elements, it's NOT a CAPTCHA page
|
||||
content_loaded_indicators = [
|
||||
'class="product-card',
|
||||
'data-product-id',
|
||||
'data-sku',
|
||||
'/product/',
|
||||
'add to cart',
|
||||
'product-grid',
|
||||
'product-list',
|
||||
]
|
||||
|
||||
for indicator in content_loaded_indicators:
|
||||
if indicator in page_source:
|
||||
# Page has actual content - no CAPTCHA
|
||||
return False
|
||||
|
||||
# Only flag as CAPTCHA if we see blocking indicators AND no content
|
||||
captcha_indicators = [
|
||||
"verify you are human",
|
||||
"press & hold",
|
||||
"press and hold",
|
||||
"checking your browser",
|
||||
"just a moment",
|
||||
"enable javascript and cookies",
|
||||
"access denied",
|
||||
"blocked",
|
||||
"challenge-running",
|
||||
"cf-browser-verification",
|
||||
"ddos-guard",
|
||||
]
|
||||
|
||||
for indicator in captcha_indicators:
|
||||
if indicator in page_source:
|
||||
return True
|
||||
|
||||
# Also check if page is suspiciously empty (might be blocked)
|
||||
if len(page_source) < 5000 and "pokemoncenter" not in page_source:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def wait_for_captcha_solve(self, timeout: int = 120):
|
||||
"""
|
||||
Wait for user to solve CAPTCHA manually.
|
||||
Only works in non-headless mode.
|
||||
"""
|
||||
if self.headless:
|
||||
logger.warning("Cannot solve CAPTCHA in headless mode")
|
||||
return False
|
||||
|
||||
logger.info("CAPTCHA detected! Please solve it manually...")
|
||||
print("\n" + "=" * 50)
|
||||
print("CAPTCHA DETECTED!")
|
||||
print("Please solve the CAPTCHA in the browser window.")
|
||||
print("=" * 50 + "\n")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
if not self.check_for_captcha():
|
||||
logger.info("CAPTCHA solved!")
|
||||
self._save_cookies() # Save session after solving
|
||||
return True
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
logger.warning("CAPTCHA solve timeout")
|
||||
return False
|
||||
|
||||
def screenshot(self, filename: str = "screenshot.png"):
|
||||
"""Take a screenshot for debugging"""
|
||||
if self.driver:
|
||||
try:
|
||||
self.driver.save_screenshot(filename)
|
||||
logger.info(f"Screenshot saved to {filename}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save screenshot: {e}")
|
||||
|
||||
|
||||
class StealthBrowserPool:
|
||||
"""
|
||||
Manages multiple stealth browser instances with proxy rotation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pool_size: int = 1,
|
||||
use_proxies: bool = False,
|
||||
headless: bool = False
|
||||
):
|
||||
self.pool_size = pool_size
|
||||
self.use_proxies = use_proxies
|
||||
self.headless = headless
|
||||
self.browsers: List[StealthBrowser] = []
|
||||
self.proxy_rotator = ProxyRotator() if use_proxies else None
|
||||
self.current_index = 0
|
||||
|
||||
def get_browser(self) -> StealthBrowser:
|
||||
"""Get a browser from the pool"""
|
||||
# Create browser if pool is empty
|
||||
if not self.browsers:
|
||||
proxy = self.proxy_rotator.get_proxy() if self.proxy_rotator else None
|
||||
browser = StealthBrowser(
|
||||
headless=self.headless,
|
||||
proxy=proxy,
|
||||
session_name=f"pool_{self.current_index}"
|
||||
)
|
||||
browser.start()
|
||||
self.browsers.append(browser)
|
||||
return browser
|
||||
|
||||
# Rotate through browsers
|
||||
browser = self.browsers[self.current_index]
|
||||
self.current_index = (self.current_index + 1) % len(self.browsers)
|
||||
return browser
|
||||
|
||||
def shutdown_all(self):
|
||||
"""Shutdown all browsers in pool"""
|
||||
for browser in self.browsers:
|
||||
try:
|
||||
browser.stop()
|
||||
except Exception:
|
||||
pass
|
||||
self.browsers = []
|
||||
|
||||
|
||||
# Global instances
|
||||
_stealth_browser: Optional[StealthBrowser] = None
|
||||
_browser_pool: Optional[StealthBrowserPool] = None
|
||||
|
||||
|
||||
def get_stealth_browser(
|
||||
headless: bool = False,
|
||||
session_name: str = "pokemoncenter"
|
||||
) -> StealthBrowser:
|
||||
"""Get or create the global stealth browser instance"""
|
||||
global _stealth_browser
|
||||
|
||||
if _stealth_browser is None:
|
||||
_stealth_browser = StealthBrowser(
|
||||
headless=headless,
|
||||
session_name=session_name
|
||||
)
|
||||
|
||||
if not _stealth_browser._setup_complete:
|
||||
_stealth_browser.start()
|
||||
|
||||
return _stealth_browser
|
||||
|
||||
|
||||
def shutdown_stealth_browser():
|
||||
"""Shutdown the global stealth browser"""
|
||||
global _stealth_browser
|
||||
|
||||
if _stealth_browser:
|
||||
_stealth_browser.stop()
|
||||
_stealth_browser = None
|
||||
|
||||
|
||||
def create_proxy_config_template():
|
||||
"""Create a template proxies.json file"""
|
||||
template = {
|
||||
"proxies": [
|
||||
{
|
||||
"host": "proxy1.example.com",
|
||||
"port": 8080,
|
||||
"protocol": "http",
|
||||
"username": "user",
|
||||
"password": "pass"
|
||||
},
|
||||
{
|
||||
"host": "proxy2.example.com",
|
||||
"port": 8080,
|
||||
"protocol": "http",
|
||||
"username": "user",
|
||||
"password": "pass"
|
||||
}
|
||||
],
|
||||
"_comment": "Add your residential proxies here. Recommended providers: Bright Data, Oxylabs, Smartproxy"
|
||||
}
|
||||
|
||||
if not PROXY_FILE.exists():
|
||||
with open(PROXY_FILE, 'w') as f:
|
||||
json.dump(template, f, indent=2)
|
||||
logger.info(f"Created proxy template at {PROXY_FILE}")
|
||||
|
||||
|
||||
# Create template on import
|
||||
create_proxy_config_template()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Warmup tool for sites with CAPTCHA/bot protection.
|
||||
Run this before the main monitor to solve CAPTCHAs manually.
|
||||
|
||||
Usage:
|
||||
python tools/warmup.py gamestop
|
||||
python tools/warmup.py all
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import logging
|
||||
from scrapers.gamestop import warmup_gamestop
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def warmup_all():
|
||||
"""Warm up all sites that need it"""
|
||||
results = {}
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info("Starting warmup for GameStop...")
|
||||
logger.info("=" * 50)
|
||||
results["gamestop"] = warmup_gamestop()
|
||||
|
||||
# Add more sites here as needed
|
||||
# results["pokemoncenter"] = warmup_pokemoncenter()
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info("Warmup Results:")
|
||||
for site, success in results.items():
|
||||
status = "OK" if success else "FAILED"
|
||||
logger.info(f" {site}: {status}")
|
||||
logger.info("=" * 50)
|
||||
|
||||
return all(results.values())
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(__doc__)
|
||||
print("\nAvailable sites: gamestop, all")
|
||||
sys.exit(1)
|
||||
|
||||
site = sys.argv[1].lower()
|
||||
|
||||
if site == "gamestop":
|
||||
success = warmup_gamestop()
|
||||
elif site == "all":
|
||||
success = warmup_all()
|
||||
else:
|
||||
print(f"Unknown site: {site}")
|
||||
print("Available sites: gamestop, all")
|
||||
sys.exit(1)
|
||||
|
||||
if success:
|
||||
logger.info("Warmup completed successfully!")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("Warmup failed or timed out")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Session Warmup Tool for Pokemon Center
|
||||
|
||||
Run this BEFORE starting the monitor to:
|
||||
1. Open a stealth browser
|
||||
2. Let you manually browse and solve any CAPTCHAs
|
||||
3. Save cookies/session for the monitor to reuse
|
||||
|
||||
Usage:
|
||||
python warmup_session.py
|
||||
|
||||
After running:
|
||||
1. Browse Pokemon Center normally for a few minutes
|
||||
2. Add items to cart, look at products, etc.
|
||||
3. Solve any CAPTCHAs that appear
|
||||
4. Press Enter in this terminal when done
|
||||
5. The session will be saved and reused by the monitor
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
print("=" * 60)
|
||||
print("Pokemon Center Session Warmup")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Import stealth browser
|
||||
try:
|
||||
from .stealth_browser import StealthBrowser
|
||||
except ImportError as e:
|
||||
print(f"Error importing stealth browser: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("Starting stealth browser...")
|
||||
print("This will open a Chrome window.")
|
||||
print()
|
||||
|
||||
browser = StealthBrowser(
|
||||
headless=False,
|
||||
session_name="pokemoncenter" # Same name used by monitor
|
||||
)
|
||||
|
||||
try:
|
||||
browser.start()
|
||||
print("[OK] Browser started!")
|
||||
print()
|
||||
|
||||
# Navigate to Pokemon Center
|
||||
print("Navigating to Pokemon Center...")
|
||||
browser.driver.get("https://www.pokemoncenter.com")
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("WARMUP INSTRUCTIONS")
|
||||
print("=" * 60)
|
||||
print("""
|
||||
1. If you see a CAPTCHA or "Pardon Our Interruption":
|
||||
- Solve it in the browser window
|
||||
- Wait for the page to load
|
||||
|
||||
2. Browse naturally for 2-3 minutes:
|
||||
- Click on some products
|
||||
- Look at different categories
|
||||
- Add something to cart (you don't have to buy)
|
||||
- This builds a legitimate browsing profile
|
||||
|
||||
3. Navigate to the TCG section:
|
||||
- https://www.pokemoncenter.com/category/tcg-cards
|
||||
|
||||
4. Once you're browsing normally without issues:
|
||||
- Come back here
|
||||
- Press ENTER to save the session
|
||||
""")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
input("Press ENTER when you're done browsing to save the session...")
|
||||
|
||||
print()
|
||||
print("Saving session cookies...")
|
||||
browser._save_cookies()
|
||||
|
||||
# Check how many cookies we got
|
||||
cookies = browser.driver.get_cookies()
|
||||
print(f"[OK] Saved {len(cookies)} cookies")
|
||||
|
||||
# Get current URL for reference
|
||||
current_url = browser.driver.current_url
|
||||
print(f"[OK] Final URL: {current_url}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("SESSION SAVED!")
|
||||
print("=" * 60)
|
||||
print("""
|
||||
Your session has been saved. The monitor will now use these
|
||||
cookies when checking Pokemon Center.
|
||||
|
||||
Tips for best results:
|
||||
- Run the monitor with 3-5 minute check intervals
|
||||
- Keep USE_STEALTH_BROWSER = True in config.py
|
||||
- If you get blocked again, run this warmup again
|
||||
|
||||
Session file: data/sessions/pokemoncenter_cookies.pkl
|
||||
""")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted by user")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
print("Closing browser...")
|
||||
browser.stop()
|
||||
print("Done!")
|
||||
Reference in New Issue
Block a user