feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
+68
View File
@@ -0,0 +1,68 @@
"""
Test Pokemon Center API endpoints directly
"""
import requests
import json
# Headers extracted from HAR capture
HEADERS = {
"Accept": "application/json",
"Accept-Version": "1",
"Content-Type": "application/json",
"X-Store-Locale": "en-us",
"X-Store-Scope": "pokemon",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
"Referer": "https://www.pokemoncenter.com/category/tcg-cards",
}
# Test endpoints
ENDPOINTS = [
# Category listing
("GET", "https://www.pokemoncenter.com/site/resourceapi/category/new-releases"),
# Product details (example SKU)
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/699-17113"),
# Product status
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/status/qgqvbkjwhe4s2mjxgeytg="),
]
print("=" * 70)
print("Pokemon Center API Test")
print("=" * 70)
print()
for method, url in ENDPOINTS:
print(f"[{method}] {url[:80]}...")
try:
if method == "GET":
response = requests.get(url, headers=HEADERS, timeout=10)
else:
response = requests.post(url, headers=HEADERS, timeout=10)
print(f" Status: {response.status_code}")
print(f" Content-Type: {response.headers.get('Content-Type', 'N/A')}")
if response.status_code == 200:
try:
data = response.json()
print(f" Response keys: {list(data.keys())[:5]}...")
# Show a preview
preview = json.dumps(data, indent=2)[:500]
print(f" Preview:\n{preview}")
except:
print(f" Raw: {response.text[:200]}")
else:
print(f" Response: {response.text[:200]}")
except Exception as e:
print(f" Error: {e}")
print()
print("=" * 70)
print("If APIs return 200, we can monitor without a browser!")
print("=" * 70)