feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -5,14 +5,38 @@ Update DISCORD_WEBHOOK_URL with your actual webhook URL
|
||||
|
||||
# Discord webhook URL - GET THIS FROM YOUR DISCORD SERVER
|
||||
# Server Settings -> Integrations -> Webhooks -> New Webhook -> Copy Webhook URL
|
||||
DISCORD_WEBHOOK_URL = "YOUR_WEBHOOK_URL_HERE"
|
||||
DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1471927173353963570/8Xh4Y_D8zRDi5MAP6ZlX0rf2Fc5wEtRKCup74kBZltk2qaatxArFp-yrk7ZWh5EOZbHp"
|
||||
|
||||
# Enable/disable Discord notifications
|
||||
NOTIFICATIONS_ENABLED = True # Set to False to disable all Discord alerts
|
||||
NEW_DROP_NOTIFICATIONS = True # Set to False to silence new drop notifications
|
||||
SKIP_DUPLICATE_SKUS = True # Skip notifications for products we've already notified about
|
||||
|
||||
# Discord Bot Token (for interactive bot commands)
|
||||
# Create a bot at https://discord.com/developers/applications
|
||||
# Required for !setlocation, !stores, !stock commands
|
||||
DISCORD_BOT_TOKEN = "MTQ4NjQ2ODM2NDkyMTczNzM3OQ.Gxl8pq.wIvK2eVbp1G0SJhW4XTYD3zopR1gohaNv5sxlQ"
|
||||
|
||||
# How often to check each site (in seconds)
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
|
||||
# Pagination settings - how many pages to scrape per site
|
||||
# Set to 1 for fastest checks, higher for more thorough scraping
|
||||
MAX_PAGES = {
|
||||
"target": 1, # Each page has ~24 products
|
||||
"bestbuy": 1,
|
||||
"gamestop": 1,
|
||||
"walmart": 1,
|
||||
"pokemoncenter": 1,
|
||||
}
|
||||
|
||||
# Sort by newest - ensures new drops appear first
|
||||
SORT_BY_NEWEST = True
|
||||
|
||||
# Which sites to monitor
|
||||
# Note: PokemonCenter has strong bot protection (Imperva) - may need manual workarounds
|
||||
SITES_ENABLED = ["target"] # Options: "pokemoncenter", "target", "walmart", "bestbuy"
|
||||
# Note: Walmart has aggressive bot protection (PerimeterX) - may need stealth mode
|
||||
SITES_ENABLED = ["target", "bestbuy"] # Options: "pokemoncenter", "target", "gamestop", "bestbuy" (disabled - needs rework), "walmart"
|
||||
|
||||
# PokemonCenter URLs to monitor
|
||||
POKEMON_CENTER_URLS = [
|
||||
@@ -24,6 +48,21 @@ TARGET_URLS = [
|
||||
"https://www.target.com/s?searchTerm=pokemon+tcg",
|
||||
]
|
||||
|
||||
# GameStop URLs to monitor
|
||||
GAMESTOP_URLS = [
|
||||
"https://www.gamestop.com/search/?q=pokemon+tcg&lang=en_US",
|
||||
]
|
||||
|
||||
# Best Buy URLs to monitor
|
||||
BESTBUY_URLS = [
|
||||
"https://www.bestbuy.com/site/searchpage.jsp?st=pokemon+tcg",
|
||||
]
|
||||
|
||||
# Walmart URLs to monitor
|
||||
WALMART_URLS = [
|
||||
"https://www.walmart.com/search?q=pokemon+tcg",
|
||||
]
|
||||
|
||||
# Keyword filtering (disabled by default)
|
||||
KEYWORD_FILTER_ENABLED = False
|
||||
KEYWORDS = [
|
||||
@@ -46,10 +85,77 @@ SLOW_MO = 0 # Milliseconds to slow down browser actions (for debugging)
|
||||
# Leave as None to use a fresh browser profile
|
||||
# Set to None to use a fresh profile, or a path to use an existing profile
|
||||
# Note: Chrome must be closed when using an existing profile
|
||||
CHROME_USER_DATA_DIR = None
|
||||
CHROME_USER_DATA_DIR = None # Disabled - causes issues with Playwright persistent context
|
||||
|
||||
# Use real Chrome instead of Playwright's Chromium (helps with bot detection)
|
||||
USE_REAL_CHROME = True
|
||||
|
||||
# =============================================================================
|
||||
# STEALTH MODE SETTINGS (for Pokemon Center and other protected sites)
|
||||
# =============================================================================
|
||||
|
||||
# Use undetected-chromedriver instead of Playwright (better for Imperva/Cloudflare)
|
||||
USE_STEALTH_BROWSER = True
|
||||
|
||||
# Stealth browser settings
|
||||
STEALTH_HEADLESS = False # Keep False - headless is more detectable
|
||||
STEALTH_SESSION_NAME = "pokemoncenter" # Name for cookie/session persistence
|
||||
|
||||
# Human-like behavior settings
|
||||
MIN_CHECK_INTERVAL = 180 # Minimum seconds between checks for protected sites
|
||||
MAX_CHECK_INTERVAL = 300 # Maximum seconds (will randomize between min/max)
|
||||
HUMAN_DELAY_MIN = 2.0 # Minimum delay between actions (seconds)
|
||||
HUMAN_DELAY_MAX = 5.0 # Maximum delay between actions (seconds)
|
||||
|
||||
# Proxy settings (optional but recommended for heavy use)
|
||||
USE_PROXIES = False # Set to True to enable proxy rotation
|
||||
# Configure proxies in proxies.json file
|
||||
|
||||
# Auto-solve CAPTCHA settings
|
||||
CAPTCHA_WAIT_TIMEOUT = 120 # Seconds to wait for manual CAPTCHA solve
|
||||
PAUSE_ON_CAPTCHA = True # Pause monitoring when CAPTCHA detected (requires manual solve)
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL = "INFO"
|
||||
|
||||
# Dashboard settings
|
||||
DASHBOARD_HOST = "0.0.0.0" # Listen on all interfaces
|
||||
DASHBOARD_PORT = 5000
|
||||
DASHBOARD_DEBUG = False # Set to True for development
|
||||
|
||||
# Database settings
|
||||
DATABASE_PATH = None # None = use default (stats.db in project root)
|
||||
|
||||
# =============================================================================
|
||||
# NEWS AGGREGATION SETTINGS
|
||||
# =============================================================================
|
||||
|
||||
# News sources configuration
|
||||
NEWS_CONFIG = {
|
||||
'twitter': {
|
||||
'enabled': False, # Enable when you have API access
|
||||
'bearer_token': 'YOUR_TWITTER_BEARER_TOKEN', # Get from https://developer.twitter.com
|
||||
'accounts': [
|
||||
'pokepullzhq', # Poke Pullz
|
||||
'PokemonRestocks', # Pokemon TCG Restocks & News
|
||||
'PokemonDealsTCG', # Pokemon Deals, Alerts & News!
|
||||
'PokeNotifyX' # PokeNotify
|
||||
],
|
||||
'fetch_interval': 300 # 5 minutes (free tier: 1500 tweets/month)
|
||||
},
|
||||
'manual_sources': {
|
||||
'enabled': True,
|
||||
'sources': ['Poke Pullz Discord', 'PokePings Discord', 'Other']
|
||||
},
|
||||
'pokemon_official': {
|
||||
'enabled': True,
|
||||
'url': 'https://www.pokemon.com/us/pokemon-tcg-news/',
|
||||
'fetch_interval': 3600 # 1 hour (official news updates less frequently)
|
||||
}
|
||||
}
|
||||
|
||||
# Auto-fetch news on dashboard start
|
||||
NEWS_AUTO_FETCH = True
|
||||
|
||||
# Clean up old news articles after this many days
|
||||
NEWS_RETENTION_DAYS = 30
|
||||
|
||||
Reference in New Issue
Block a user