feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
Test Pokemon Center API endpoints directly
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
|
||||
# Headers extracted from HAR capture
|
||||
HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/category/tcg-cards",
|
||||
}
|
||||
|
||||
# Test endpoints
|
||||
ENDPOINTS = [
|
||||
# Category listing
|
||||
("GET", "https://www.pokemoncenter.com/site/resourceapi/category/new-releases"),
|
||||
|
||||
# Product details (example SKU)
|
||||
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/699-17113"),
|
||||
|
||||
# Product status
|
||||
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/status/qgqvbkjwhe4s2mjxgeytg="),
|
||||
]
|
||||
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Test")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
for method, url in ENDPOINTS:
|
||||
print(f"[{method}] {url[:80]}...")
|
||||
|
||||
try:
|
||||
if method == "GET":
|
||||
response = requests.get(url, headers=HEADERS, timeout=10)
|
||||
else:
|
||||
response = requests.post(url, headers=HEADERS, timeout=10)
|
||||
|
||||
print(f" Status: {response.status_code}")
|
||||
print(f" Content-Type: {response.headers.get('Content-Type', 'N/A')}")
|
||||
|
||||
if response.status_code == 200:
|
||||
try:
|
||||
data = response.json()
|
||||
print(f" Response keys: {list(data.keys())[:5]}...")
|
||||
|
||||
# Show a preview
|
||||
preview = json.dumps(data, indent=2)[:500]
|
||||
print(f" Preview:\n{preview}")
|
||||
except:
|
||||
print(f" Raw: {response.text[:200]}")
|
||||
else:
|
||||
print(f" Response: {response.text[:200]}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("If APIs return 200, we can monitor without a browser!")
|
||||
print("=" * 70)
|
||||
Reference in New Issue
Block a user