feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
API Call Capture Tool for Pokemon Center
|
||||
|
||||
This script opens Pokemon Center and logs ALL network requests,
|
||||
helping identify backend APIs used for product data.
|
||||
|
||||
Usage:
|
||||
python capture_api_calls.py
|
||||
|
||||
Output:
|
||||
- api_captures.json: All captured API calls
|
||||
- Console output with interesting endpoints
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Capture Tool")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Check for selenium
|
||||
try:
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||
except ImportError:
|
||||
print("ERROR: selenium not installed")
|
||||
print("Run: pip install selenium")
|
||||
exit(1)
|
||||
|
||||
# Storage for captured requests
|
||||
captured_requests = []
|
||||
interesting_patterns = [
|
||||
r'api',
|
||||
r'graphql',
|
||||
r'product',
|
||||
r'catalog',
|
||||
r'search',
|
||||
r'inventory',
|
||||
r'stock',
|
||||
r'algolia',
|
||||
r'contentful',
|
||||
r'commercetools',
|
||||
r'\.json',
|
||||
]
|
||||
|
||||
def is_interesting(url):
|
||||
"""Check if URL matches patterns we care about"""
|
||||
url_lower = url.lower()
|
||||
for pattern in interesting_patterns:
|
||||
if re.search(pattern, url_lower):
|
||||
return True
|
||||
return False
|
||||
|
||||
def setup_browser():
|
||||
"""Setup Chrome with network logging enabled"""
|
||||
options = Options()
|
||||
options.add_argument("--disable-blink-features=AutomationControlled")
|
||||
options.add_argument("--window-size=1920,1080")
|
||||
|
||||
# Enable performance logging to capture network requests
|
||||
options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
|
||||
|
||||
driver = webdriver.Chrome(options=options)
|
||||
return driver
|
||||
|
||||
def extract_network_requests(driver):
|
||||
"""Extract network requests from Chrome performance logs"""
|
||||
logs = driver.get_log('performance')
|
||||
requests = []
|
||||
|
||||
for entry in logs:
|
||||
try:
|
||||
log = json.loads(entry['message'])['message']
|
||||
|
||||
# We want Network.requestWillBeSent and Network.responseReceived
|
||||
if log['method'] == 'Network.requestWillBeSent':
|
||||
request = log['params']['request']
|
||||
requests.append({
|
||||
'type': 'request',
|
||||
'url': request.get('url', ''),
|
||||
'method': request.get('method', ''),
|
||||
'headers': request.get('headers', {}),
|
||||
'postData': request.get('postData', None),
|
||||
'timestamp': entry['timestamp']
|
||||
})
|
||||
|
||||
elif log['method'] == 'Network.responseReceived':
|
||||
response = log['params']['response']
|
||||
requests.append({
|
||||
'type': 'response',
|
||||
'url': response.get('url', ''),
|
||||
'status': response.get('status', 0),
|
||||
'mimeType': response.get('mimeType', ''),
|
||||
'headers': response.get('headers', {}),
|
||||
'timestamp': entry['timestamp']
|
||||
})
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return requests
|
||||
|
||||
def analyze_requests(requests):
|
||||
"""Analyze captured requests for interesting patterns"""
|
||||
api_calls = []
|
||||
seen_urls = set()
|
||||
|
||||
for req in requests:
|
||||
url = req.get('url', '')
|
||||
|
||||
# Skip if already seen or not interesting
|
||||
if url in seen_urls:
|
||||
continue
|
||||
if not is_interesting(url):
|
||||
continue
|
||||
|
||||
# Skip common non-API resources
|
||||
if any(ext in url for ext in ['.png', '.jpg', '.gif', '.css', '.woff', '.svg', '.ico']):
|
||||
continue
|
||||
|
||||
seen_urls.add(url)
|
||||
api_calls.append(req)
|
||||
|
||||
return api_calls
|
||||
|
||||
def main():
|
||||
driver = None
|
||||
|
||||
try:
|
||||
print("Starting Chrome with network logging...")
|
||||
driver = setup_browser()
|
||||
|
||||
print("Navigating to Pokemon Center TCG page...")
|
||||
print("(This may trigger a CAPTCHA - solve it if needed)")
|
||||
print()
|
||||
|
||||
# Navigate to the TCG category
|
||||
driver.get("https://www.pokemoncenter.com/category/tcg-cards")
|
||||
|
||||
print("Waiting for page to load...")
|
||||
time.sleep(10)
|
||||
|
||||
# Scroll down to trigger lazy loading
|
||||
print("Scrolling to load more content...")
|
||||
for i in range(3):
|
||||
driver.execute_script("window.scrollBy(0, 800);")
|
||||
time.sleep(2)
|
||||
|
||||
# Wait a bit more
|
||||
time.sleep(5)
|
||||
|
||||
print()
|
||||
print("Extracting network requests...")
|
||||
requests = extract_network_requests(driver)
|
||||
print(f"Captured {len(requests)} total network events")
|
||||
|
||||
# Analyze for interesting APIs
|
||||
api_calls = analyze_requests(requests)
|
||||
print(f"Found {len(api_calls)} potentially interesting API calls")
|
||||
|
||||
# Save all captures
|
||||
output_file = Path(__file__).parent / "api_captures.json"
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump({
|
||||
'captured_at': datetime.now().isoformat(),
|
||||
'page_url': driver.current_url,
|
||||
'total_requests': len(requests),
|
||||
'interesting_calls': api_calls,
|
||||
'all_requests': requests
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"\nSaved full capture to: {output_file}")
|
||||
|
||||
# Display interesting findings
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("INTERESTING API CALLS FOUND")
|
||||
print("=" * 70)
|
||||
|
||||
if not api_calls:
|
||||
print("No obvious API calls detected.")
|
||||
print("This could mean:")
|
||||
print(" 1. Data is server-rendered (no client API)")
|
||||
print(" 2. API calls use non-standard paths")
|
||||
print(" 3. Bot protection blocked the content")
|
||||
else:
|
||||
for call in api_calls:
|
||||
url = call.get('url', '')
|
||||
method = call.get('method', 'GET')
|
||||
status = call.get('status', '-')
|
||||
mime = call.get('mimeType', '')
|
||||
|
||||
# Truncate long URLs
|
||||
display_url = url[:100] + '...' if len(url) > 100 else url
|
||||
|
||||
print(f"\n[{method}] {display_url}")
|
||||
if status != '-':
|
||||
print(f" Status: {status}, Type: {mime}")
|
||||
|
||||
# Check for key patterns
|
||||
if 'graphql' in url.lower():
|
||||
print(" ⚡ GraphQL endpoint!")
|
||||
if 'algolia' in url.lower():
|
||||
print(" 🔍 Algolia search!")
|
||||
if 'product' in url.lower():
|
||||
print(" 📦 Product data!")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("NEXT STEPS")
|
||||
print("=" * 70)
|
||||
print("""
|
||||
1. Review api_captures.json for the full data
|
||||
2. Look for JSON responses containing product info
|
||||
3. Test promising endpoints with curl/requests
|
||||
4. Note any required headers (auth tokens, API keys)
|
||||
|
||||
To test an endpoint manually:
|
||||
curl -H "User-Agent: Mozilla/5.0..." "https://api.example.com/endpoint"
|
||||
""")
|
||||
|
||||
# Keep browser open for manual inspection
|
||||
print()
|
||||
input("Browser is still open. Press ENTER to close and exit...")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if driver:
|
||||
driver.quit()
|
||||
print("Browser closed.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user