feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
"""
|
||||
API Discovery Tool for Pokemon Center
|
||||
Attempts to find and test API endpoints that could be used instead of browser scraping.
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# Common API patterns to try
|
||||
BASE_URL = "https://www.pokemoncenter.com"
|
||||
|
||||
# Headers to mimic a real browser
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Referer": "https://www.pokemoncenter.com/",
|
||||
"Origin": "https://www.pokemoncenter.com",
|
||||
}
|
||||
|
||||
# Common API endpoint patterns for e-commerce sites
|
||||
API_PATTERNS = [
|
||||
# REST API patterns
|
||||
"/api/products",
|
||||
"/api/v1/products",
|
||||
"/api/v2/products",
|
||||
"/api/catalog/products",
|
||||
"/api/search",
|
||||
"/api/inventory",
|
||||
|
||||
# GraphQL
|
||||
"/graphql",
|
||||
"/api/graphql",
|
||||
|
||||
# Common e-commerce platforms
|
||||
"/rest/V1/products", # Magento
|
||||
"/_api/products", # Wix
|
||||
"/cdn/shop/products.json", # Shopify pattern
|
||||
"/products.json", # Shopify
|
||||
|
||||
# Search APIs
|
||||
"/api/search/products",
|
||||
"/search/suggest",
|
||||
"/api/autocomplete",
|
||||
|
||||
# Algolia (very common for e-commerce search)
|
||||
# Note: Algolia requires app ID and API key from the page
|
||||
]
|
||||
|
||||
# Pokemon TCG specific search terms
|
||||
SEARCH_TERMS = ["pokemon", "tcg", "cards", "booster", "etb"]
|
||||
|
||||
|
||||
def test_endpoint(url: str, method: str = "GET", data: dict = None) -> dict:
|
||||
"""Test an API endpoint"""
|
||||
try:
|
||||
if method == "GET":
|
||||
response = requests.get(url, headers=HEADERS, timeout=10)
|
||||
else:
|
||||
response = requests.post(url, headers=HEADERS, json=data, timeout=10)
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"status": response.status_code,
|
||||
"content_type": response.headers.get("content-type", ""),
|
||||
"size": len(response.content),
|
||||
"sample": response.text[:500] if response.status_code == 200 else None
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"url": url,
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
def discover_apis():
|
||||
"""Attempt to discover API endpoints"""
|
||||
print("=" * 60)
|
||||
print("Pokemon Center API Discovery")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
results = []
|
||||
|
||||
# Test common patterns
|
||||
print("Testing common API patterns...")
|
||||
for pattern in API_PATTERNS:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
results.append(result)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
print(f" Content-Type: {result['content_type']}")
|
||||
print(f" Size: {result['size']} bytes")
|
||||
elif result["status"] != "error" and result["status"] < 500:
|
||||
print(f" [{result['status']}] {url}")
|
||||
|
||||
# Test search with query params
|
||||
print()
|
||||
print("Testing search endpoints...")
|
||||
search_patterns = [
|
||||
"/api/search?q=pokemon",
|
||||
"/api/products?search=tcg",
|
||||
"/api/catalog?category=tcg-cards",
|
||||
"/search?q=pokemon+tcg",
|
||||
]
|
||||
|
||||
for pattern in search_patterns:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
results.append(result)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
|
||||
# Look for Algolia configuration
|
||||
print()
|
||||
print("Checking for Algolia search...")
|
||||
# Algolia is often exposed in page source
|
||||
try:
|
||||
response = requests.get(BASE_URL, headers=HEADERS, timeout=15)
|
||||
if "algolia" in response.text.lower():
|
||||
print(" [!] Algolia detected in page source")
|
||||
# Try to extract app ID and search key
|
||||
import re
|
||||
app_id = re.search(r'["\']?algolia[_-]?app[_-]?id["\']?\s*[:=]\s*["\']([A-Z0-9]+)["\']', response.text, re.I)
|
||||
api_key = re.search(r'["\']?algolia[_-]?(?:search[_-]?)?(?:api[_-]?)?key["\']?\s*[:=]\s*["\']([a-f0-9]+)["\']', response.text, re.I)
|
||||
|
||||
if app_id:
|
||||
print(f" App ID: {app_id.group(1)}")
|
||||
if api_key:
|
||||
print(f" Search Key: {api_key.group(1)}")
|
||||
|
||||
# Check for other API clues
|
||||
if "graphql" in response.text.lower():
|
||||
print(" [!] GraphQL detected in page source")
|
||||
|
||||
if "__NEXT_DATA__" in response.text:
|
||||
print(" [!] Next.js detected - may have API routes at /api/*")
|
||||
|
||||
if "window.__INITIAL_STATE__" in response.text or "window.__PRELOADED_STATE__" in response.text:
|
||||
print(" [!] Pre-rendered state detected - data may be in page source")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error checking main page: {e}")
|
||||
|
||||
# Summary
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Summary")
|
||||
print("=" * 60)
|
||||
|
||||
working = [r for r in results if r.get("status") == 200]
|
||||
if working:
|
||||
print(f"Found {len(working)} potentially working endpoints:")
|
||||
for r in working:
|
||||
print(f" - {r['url']}")
|
||||
else:
|
||||
print("No direct API endpoints found.")
|
||||
print()
|
||||
print("Alternative approaches to consider:")
|
||||
print(" 1. Monitor sitemap.xml for new products")
|
||||
print(" 2. Use Google Shopping API or similar aggregators")
|
||||
print(" 3. Check if they have an RSS feed")
|
||||
print(" 4. Use a service like Distill.io for change detection")
|
||||
print(" 5. Proxy rotation with residential IPs")
|
||||
print(" 6. Lower check frequency + add human-like delays")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def check_sitemap():
|
||||
"""Check sitemap for product URLs"""
|
||||
print()
|
||||
print("Checking sitemap...")
|
||||
|
||||
sitemap_urls = [
|
||||
"/sitemap.xml",
|
||||
"/sitemap_index.xml",
|
||||
"/sitemaps/sitemap.xml",
|
||||
"/robots.txt", # Often contains sitemap location
|
||||
]
|
||||
|
||||
for pattern in sitemap_urls:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
|
||||
if result["status"] == 200:
|
||||
print(f" [OK] {url}")
|
||||
if "sitemap" in result.get("sample", "").lower():
|
||||
print(f" Contains sitemap references")
|
||||
if "product" in result.get("sample", "").lower():
|
||||
print(f" Contains product references")
|
||||
|
||||
|
||||
def check_rss():
|
||||
"""Check for RSS feeds"""
|
||||
print()
|
||||
print("Checking for RSS/Atom feeds...")
|
||||
|
||||
feed_urls = [
|
||||
"/feed",
|
||||
"/rss",
|
||||
"/feed.xml",
|
||||
"/rss.xml",
|
||||
"/atom.xml",
|
||||
"/blog/feed",
|
||||
"/news/feed",
|
||||
]
|
||||
|
||||
for pattern in feed_urls:
|
||||
url = urljoin(BASE_URL, pattern)
|
||||
result = test_endpoint(url)
|
||||
|
||||
if result["status"] == 200 and ("xml" in result.get("content_type", "") or "rss" in result.get("content_type", "")):
|
||||
print(f" [OK] {url}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
discover_apis()
|
||||
check_sitemap()
|
||||
check_rss()
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Next Steps")
|
||||
print("=" * 60)
|
||||
print("""
|
||||
To avoid bot detection, consider these strategies:
|
||||
|
||||
1. API-BASED MONITORING (if endpoints found):
|
||||
- Call API endpoints directly with requests
|
||||
- Much faster and less detectable than browser
|
||||
- Can check more frequently
|
||||
|
||||
2. SITEMAP MONITORING:
|
||||
- Parse sitemap.xml periodically
|
||||
- Detect new product URLs without visiting pages
|
||||
- Very low detection risk
|
||||
|
||||
3. HASH-BASED CHANGE DETECTION:
|
||||
- Fetch page, hash content
|
||||
- Only alert when hash changes
|
||||
- Reduces unnecessary processing
|
||||
|
||||
4. RESIDENTIAL PROXY ROTATION:
|
||||
- Use services like Bright Data, Oxylabs
|
||||
- Rotate IPs to avoid blocks
|
||||
- More expensive but reliable
|
||||
|
||||
5. HUMAN-LIKE BEHAVIOR:
|
||||
- Random delays between 60-180 seconds
|
||||
- Vary user agent strings
|
||||
- Add mouse movements and scrolling
|
||||
- Use real browser cookies
|
||||
|
||||
6. THIRD-PARTY ALERTS:
|
||||
- Discord servers that track Pokemon Center
|
||||
- Stock alert services (NowInStock, etc.)
|
||||
- Browser extensions like Distill.io
|
||||
|
||||
Run this script to see what APIs are available:
|
||||
python api_discovery.py
|
||||
""")
|
||||
Reference in New Issue
Block a user