pokemon-stock-checker/tools/api_discovery.py

"""
API Discovery Tool for Pokemon Center
Attempts to find and test API endpoints that could be used instead of browser scraping.
"""

import requests
import json
from urllib.parse import urljoin

# Common API patterns to try
BASE_URL = "https://www.pokemoncenter.com"

# Headers to mimic a real browser
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "application/json, text/plain, */*",
    "Accept-Language": "en-US,en;q=0.9",
    "Referer": "https://www.pokemoncenter.com/",
    "Origin": "https://www.pokemoncenter.com",
}

# Common API endpoint patterns for e-commerce sites
API_PATTERNS = [
    # REST API patterns
    "/api/products",
    "/api/v1/products",
    "/api/v2/products",
    "/api/catalog/products",
    "/api/search",
    "/api/inventory",

    # GraphQL
    "/graphql",
    "/api/graphql",

    # Common e-commerce platforms
    "/rest/V1/products",  # Magento
    "/_api/products",  # Wix
    "/cdn/shop/products.json",  # Shopify pattern
    "/products.json",  # Shopify

    # Search APIs
    "/api/search/products",
    "/search/suggest",
    "/api/autocomplete",

    # Algolia (very common for e-commerce search)
    # Note: Algolia requires app ID and API key from the page
]

# Pokemon TCG specific search terms
SEARCH_TERMS = ["pokemon", "tcg", "cards", "booster", "etb"]


def test_endpoint(url: str, method: str = "GET", data: dict = None) -> dict:
    """Test an API endpoint"""
    try:
        if method == "GET":
            response = requests.get(url, headers=HEADERS, timeout=10)
        else:
            response = requests.post(url, headers=HEADERS, json=data, timeout=10)

        return {
            "url": url,
            "status": response.status_code,
            "content_type": response.headers.get("content-type", ""),
            "size": len(response.content),
            "sample": response.text[:500] if response.status_code == 200 else None
        }
    except Exception as e:
        return {
            "url": url,
            "status": "error",
            "error": str(e)
        }


def discover_apis():
    """Attempt to discover API endpoints"""
    print("=" * 60)
    print("Pokemon Center API Discovery")
    print("=" * 60)
    print()

    results = []

    # Test common patterns
    print("Testing common API patterns...")
    for pattern in API_PATTERNS:
        url = urljoin(BASE_URL, pattern)
        result = test_endpoint(url)
        results.append(result)

        if result["status"] == 200:
            print(f"  [OK] {url}")
            print(f"       Content-Type: {result['content_type']}")
            print(f"       Size: {result['size']} bytes")
        elif result["status"] != "error" and result["status"] < 500:
            print(f"  [{result['status']}] {url}")

    # Test search with query params
    print()
    print("Testing search endpoints...")
    search_patterns = [
        "/api/search?q=pokemon",
        "/api/products?search=tcg",
        "/api/catalog?category=tcg-cards",
        "/search?q=pokemon+tcg",
    ]

    for pattern in search_patterns:
        url = urljoin(BASE_URL, pattern)
        result = test_endpoint(url)
        results.append(result)

        if result["status"] == 200:
            print(f"  [OK] {url}")

    # Look for Algolia configuration
    print()
    print("Checking for Algolia search...")
    # Algolia is often exposed in page source
    try:
        response = requests.get(BASE_URL, headers=HEADERS, timeout=15)
        if "algolia" in response.text.lower():
            print("  [!] Algolia detected in page source")
            # Try to extract app ID and search key
            import re
            app_id = re.search(r'["\']?algolia[_-]?app[_-]?id["\']?\s*[:=]\s*["\']([A-Z0-9]+)["\']', response.text, re.I)
            api_key = re.search(r'["\']?algolia[_-]?(?:search[_-]?)?(?:api[_-]?)?key["\']?\s*[:=]\s*["\']([a-f0-9]+)["\']', response.text, re.I)

            if app_id:
                print(f"      App ID: {app_id.group(1)}")
            if api_key:
                print(f"      Search Key: {api_key.group(1)}")

        # Check for other API clues
        if "graphql" in response.text.lower():
            print("  [!] GraphQL detected in page source")

        if "__NEXT_DATA__" in response.text:
            print("  [!] Next.js detected - may have API routes at /api/*")

        if "window.__INITIAL_STATE__" in response.text or "window.__PRELOADED_STATE__" in response.text:
            print("  [!] Pre-rendered state detected - data may be in page source")

    except Exception as e:
        print(f"  Error checking main page: {e}")

    # Summary
    print()
    print("=" * 60)
    print("Summary")
    print("=" * 60)

    working = [r for r in results if r.get("status") == 200]
    if working:
        print(f"Found {len(working)} potentially working endpoints:")
        for r in working:
            print(f"  - {r['url']}")
    else:
        print("No direct API endpoints found.")
        print()
        print("Alternative approaches to consider:")
        print("  1. Monitor sitemap.xml for new products")
        print("  2. Use Google Shopping API or similar aggregators")
        print("  3. Check if they have an RSS feed")
        print("  4. Use a service like Distill.io for change detection")
        print("  5. Proxy rotation with residential IPs")
        print("  6. Lower check frequency + add human-like delays")

    return results


def check_sitemap():
    """Check sitemap for product URLs"""
    print()
    print("Checking sitemap...")

    sitemap_urls = [
        "/sitemap.xml",
        "/sitemap_index.xml",
        "/sitemaps/sitemap.xml",
        "/robots.txt",  # Often contains sitemap location
    ]

    for pattern in sitemap_urls:
        url = urljoin(BASE_URL, pattern)
        result = test_endpoint(url)

        if result["status"] == 200:
            print(f"  [OK] {url}")
            if "sitemap" in result.get("sample", "").lower():
                print(f"       Contains sitemap references")
            if "product" in result.get("sample", "").lower():
                print(f"       Contains product references")


def check_rss():
    """Check for RSS feeds"""
    print()
    print("Checking for RSS/Atom feeds...")

    feed_urls = [
        "/feed",
        "/rss",
        "/feed.xml",
        "/rss.xml",
        "/atom.xml",
        "/blog/feed",
        "/news/feed",
    ]

    for pattern in feed_urls:
        url = urljoin(BASE_URL, pattern)
        result = test_endpoint(url)

        if result["status"] == 200 and ("xml" in result.get("content_type", "") or "rss" in result.get("content_type", "")):
            print(f"  [OK] {url}")


if __name__ == "__main__":
    discover_apis()
    check_sitemap()
    check_rss()

    print()
    print("=" * 60)
    print("Next Steps")
    print("=" * 60)
    print("""
To avoid bot detection, consider these strategies:

1. API-BASED MONITORING (if endpoints found):
   - Call API endpoints directly with requests
   - Much faster and less detectable than browser
   - Can check more frequently

2. SITEMAP MONITORING:
   - Parse sitemap.xml periodically
   - Detect new product URLs without visiting pages
   - Very low detection risk

3. HASH-BASED CHANGE DETECTION:
   - Fetch page, hash content
   - Only alert when hash changes
   - Reduces unnecessary processing

4. RESIDENTIAL PROXY ROTATION:
   - Use services like Bright Data, Oxylabs
   - Rotate IPs to avoid blocks
   - More expensive but reliable

5. HUMAN-LIKE BEHAVIOR:
   - Random delays between 60-180 seconds
   - Vary user agent strings
   - Add mouse movements and scrolling
   - Use real browser cookies

6. THIRD-PARTY ALERTS:
   - Discord servers that track Pokemon Center
   - Stock alert services (NowInStock, etc.)
   - Browser extensions like Distill.io

Run this script to see what APIs are available:
  python api_discovery.py
""")