feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
Scraper State Manager
|
||||
Manages runtime state for scrapers - enabling/disabling, running status, etc.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json"
|
||||
|
||||
# Default state
|
||||
DEFAULT_STATE = {
|
||||
"scrapers": {
|
||||
"pokemoncenter": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||
"target": {"enabled": True, "running": False, "last_run": None, "last_error": None},
|
||||
"gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||
"walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||
"bestbuy": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||
},
|
||||
"monitor_running": False,
|
||||
"monitor_pid": None,
|
||||
"check_interval": 60,
|
||||
"last_check": None,
|
||||
}
|
||||
|
||||
|
||||
class ScraperStateManager:
|
||||
"""Manages scraper state with file persistence"""
|
||||
|
||||
_instance = None
|
||||
_lock = threading.Lock()
|
||||
_monitor_process = None
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
cls._instance._initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._initialized:
|
||||
return
|
||||
self._initialized = True
|
||||
self.state = self._load_state()
|
||||
|
||||
def _load_state(self) -> dict:
|
||||
"""Load state from file or return defaults"""
|
||||
try:
|
||||
if STATE_FILE.exists():
|
||||
with open(STATE_FILE, 'r') as f:
|
||||
saved = json.load(f)
|
||||
# Merge with defaults in case new fields added
|
||||
state = DEFAULT_STATE.copy()
|
||||
state.update(saved)
|
||||
# Ensure all scrapers exist
|
||||
for scraper in DEFAULT_STATE["scrapers"]:
|
||||
if scraper not in state["scrapers"]:
|
||||
state["scrapers"][scraper] = DEFAULT_STATE["scrapers"][scraper]
|
||||
return state
|
||||
except Exception:
|
||||
pass
|
||||
return DEFAULT_STATE.copy()
|
||||
|
||||
def _save_state(self):
|
||||
"""Save state to file"""
|
||||
with self._lock:
|
||||
try:
|
||||
with open(STATE_FILE, 'w') as f:
|
||||
json.dump(self.state, f, indent=2, default=str)
|
||||
except Exception as e:
|
||||
print(f"Error saving scraper state: {e}")
|
||||
|
||||
def get_state(self) -> dict:
|
||||
"""Get current state"""
|
||||
# Check if monitor is still running
|
||||
if self.state["monitor_pid"]:
|
||||
try:
|
||||
# Check if process exists
|
||||
import psutil
|
||||
if not psutil.pid_exists(self.state["monitor_pid"]):
|
||||
self.state["monitor_running"] = False
|
||||
self.state["monitor_pid"] = None
|
||||
for scraper in self.state["scrapers"]:
|
||||
self.state["scrapers"][scraper]["running"] = False
|
||||
self._save_state()
|
||||
except ImportError:
|
||||
pass # psutil not installed, skip check
|
||||
|
||||
return self.state.copy()
|
||||
|
||||
def set_scraper_enabled(self, scraper: str, enabled: bool) -> bool:
|
||||
"""Enable or disable a scraper"""
|
||||
if scraper not in self.state["scrapers"]:
|
||||
return False
|
||||
|
||||
self.state["scrapers"][scraper]["enabled"] = enabled
|
||||
self._save_state()
|
||||
return True
|
||||
|
||||
def get_enabled_scrapers(self) -> list:
|
||||
"""Get list of enabled scraper names"""
|
||||
return [
|
||||
name for name, data in self.state["scrapers"].items()
|
||||
if data.get("enabled", False)
|
||||
]
|
||||
|
||||
def set_check_interval(self, seconds: int) -> bool:
|
||||
"""Set check interval"""
|
||||
if seconds < 10:
|
||||
return False
|
||||
self.state["check_interval"] = seconds
|
||||
self._save_state()
|
||||
return True
|
||||
|
||||
def record_check(self, scraper: str, success: bool, error: str = None):
|
||||
"""Record a scraper check"""
|
||||
if scraper in self.state["scrapers"]:
|
||||
self.state["scrapers"][scraper]["last_run"] = datetime.now().isoformat()
|
||||
if not success:
|
||||
self.state["scrapers"][scraper]["last_error"] = error
|
||||
else:
|
||||
self.state["scrapers"][scraper]["last_error"] = None
|
||||
self.state["last_check"] = datetime.now().isoformat()
|
||||
self._save_state()
|
||||
|
||||
def start_monitor(self) -> dict:
|
||||
"""Start the monitor process"""
|
||||
if self.state["monitor_running"]:
|
||||
return {"success": False, "error": "Monitor already running"}
|
||||
|
||||
enabled = self.get_enabled_scrapers()
|
||||
if not enabled:
|
||||
return {"success": False, "error": "No scrapers enabled"}
|
||||
|
||||
try:
|
||||
# Start main.py as a subprocess
|
||||
project_dir = Path(__file__).parent.parent
|
||||
main_script = project_dir / "main.py"
|
||||
|
||||
# Create a modified config for enabled scrapers
|
||||
self._update_config_file(enabled)
|
||||
|
||||
# Start the process
|
||||
if sys.platform == 'win32':
|
||||
# Windows: use CREATE_NEW_PROCESS_GROUP for proper handling
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(main_script)],
|
||||
cwd=str(project_dir),
|
||||
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
else:
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(main_script)],
|
||||
cwd=str(project_dir),
|
||||
start_new_session=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
self._monitor_process = process
|
||||
self.state["monitor_running"] = True
|
||||
self.state["monitor_pid"] = process.pid
|
||||
|
||||
for scraper in enabled:
|
||||
self.state["scrapers"][scraper]["running"] = True
|
||||
|
||||
self._save_state()
|
||||
|
||||
return {"success": True, "pid": process.pid, "scrapers": enabled}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def stop_monitor(self) -> dict:
|
||||
"""Stop the monitor process"""
|
||||
if not self.state["monitor_running"] and not self.state["monitor_pid"]:
|
||||
return {"success": False, "error": "Monitor not running"}
|
||||
|
||||
try:
|
||||
pid = self.state["monitor_pid"]
|
||||
|
||||
if pid:
|
||||
# On Windows, use taskkill for reliable process termination
|
||||
if sys.platform == 'win32':
|
||||
try:
|
||||
# Kill the process tree forcefully
|
||||
subprocess.run(
|
||||
['taskkill', '/F', '/T', '/PID', str(pid)],
|
||||
capture_output=True,
|
||||
timeout=10
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# On Unix, use psutil
|
||||
try:
|
||||
import psutil
|
||||
proc = psutil.Process(pid)
|
||||
# Kill child processes first
|
||||
children = proc.children(recursive=True)
|
||||
for child in children:
|
||||
try:
|
||||
child.kill() # Use kill instead of terminate
|
||||
except:
|
||||
pass
|
||||
# Kill main process
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
self.state["monitor_running"] = False
|
||||
self.state["monitor_pid"] = None
|
||||
for scraper in self.state["scrapers"]:
|
||||
self.state["scrapers"][scraper]["running"] = False
|
||||
self._save_state()
|
||||
|
||||
return {"success": True}
|
||||
|
||||
except Exception as e:
|
||||
# Still update state even if kill failed
|
||||
self.state["monitor_running"] = False
|
||||
self.state["monitor_pid"] = None
|
||||
for scraper in self.state["scrapers"]:
|
||||
self.state["scrapers"][scraper]["running"] = False
|
||||
self._save_state()
|
||||
return {"success": True, "warning": f"Process may need manual cleanup: {str(e)}"}
|
||||
|
||||
def _update_config_file(self, enabled_scrapers: list):
|
||||
"""Update config.py with enabled scrapers"""
|
||||
config_path = Path(__file__).parent.parent / "config.py"
|
||||
|
||||
try:
|
||||
with open(config_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find and replace SITES_ENABLED line
|
||||
import re
|
||||
new_line = f'SITES_ENABLED = {json.dumps(enabled_scrapers)}'
|
||||
pattern = r'SITES_ENABLED\s*=\s*\[.*?\]'
|
||||
content = re.sub(pattern, new_line, content)
|
||||
|
||||
# Also update CHECK_INTERVAL_SECONDS
|
||||
interval_line = f'CHECK_INTERVAL_SECONDS = {self.state["check_interval"]}'
|
||||
interval_pattern = r'CHECK_INTERVAL_SECONDS\s*=\s*\d+'
|
||||
content = re.sub(interval_pattern, interval_line, content)
|
||||
|
||||
with open(config_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error updating config: {e}")
|
||||
|
||||
|
||||
# Singleton instance
|
||||
scraper_state = ScraperStateManager()
|
||||
Reference in New Issue
Block a user