""" Scraper State Manager Manages runtime state for scrapers - enabling/disabling, running status, etc. """ import json import os import subprocess import sys import threading import time from datetime import datetime from pathlib import Path STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json" # Default state # Note: Pokemon Center uses Chrome Extension, not a Python scraper DEFAULT_STATE = { "scrapers": { "target": {"enabled": True, "running": False, "last_run": None, "last_error": None}, "gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None}, "walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None}, "bestbuy": {"enabled": False, "running": False, "last_run": None, "last_error": None}, }, "monitor_running": False, "monitor_pid": None, "check_interval": 60, "last_check": None, } class ScraperStateManager: """Manages scraper state with file persistence""" _instance = None _lock = threading.Lock() _monitor_process = None def __new__(cls): if cls._instance is None: cls._instance = super().__new__(cls) cls._instance._initialized = False return cls._instance def __init__(self): if self._initialized: return self._initialized = True self.state = self._load_state() def _load_state(self) -> dict: """Load state from file or return defaults""" try: if STATE_FILE.exists(): with open(STATE_FILE, 'r') as f: saved = json.load(f) # Merge with defaults in case new fields added state = DEFAULT_STATE.copy() state.update(saved) # Ensure all scrapers exist for scraper in DEFAULT_STATE["scrapers"]: if scraper not in state["scrapers"]: state["scrapers"][scraper] = DEFAULT_STATE["scrapers"][scraper] return state except Exception: pass return DEFAULT_STATE.copy() def _save_state(self): """Save state to file using atomic write to prevent corruption""" with self._lock: try: tmp_file = STATE_FILE.with_suffix('.json.tmp') with open(tmp_file, 'w') as f: json.dump(self.state, f, indent=2, default=str) os.replace(tmp_file, STATE_FILE) except Exception as e: print(f"Error saving scraper state: {e}") def get_state(self) -> dict: """Get current state, reloading from file to pick up changes from other processes""" self.state = self._load_state() # Check if monitor is still running if self.state["monitor_pid"]: try: # Check if process exists import psutil if not psutil.pid_exists(self.state["monitor_pid"]): self.state["monitor_running"] = False self.state["monitor_pid"] = None for scraper in self.state["scrapers"]: self.state["scrapers"][scraper]["running"] = False self._save_state() except ImportError: pass # psutil not installed, skip check return self.state.copy() def set_scraper_enabled(self, scraper: str, enabled: bool) -> bool: """Enable or disable a scraper""" if scraper not in self.state["scrapers"]: return False self.state["scrapers"][scraper]["enabled"] = enabled self._save_state() return True def get_enabled_scrapers(self) -> list: """Get list of enabled scraper names""" return [ name for name, data in self.state["scrapers"].items() if data.get("enabled", False) ] def set_check_interval(self, seconds: int) -> bool: """Set check interval""" if seconds < 10: return False self.state["check_interval"] = seconds self._save_state() return True def record_check(self, scraper: str, success: bool, error: str = None): """Record a scraper check""" if scraper in self.state["scrapers"]: self.state["scrapers"][scraper]["last_run"] = datetime.now().isoformat() if not success: self.state["scrapers"][scraper]["last_error"] = error else: self.state["scrapers"][scraper]["last_error"] = None self.state["last_check"] = datetime.now().isoformat() self._save_state() def start_monitor(self) -> dict: """Start the monitor process""" if self.state["monitor_running"]: return {"success": False, "error": "Monitor already running"} enabled = self.get_enabled_scrapers() if not enabled: return {"success": False, "error": "No scrapers enabled"} try: # Start main.py as a subprocess project_dir = Path(__file__).parent.parent main_script = project_dir / "main.py" # Create a modified config for enabled scrapers self._update_config_file(enabled) # Start the process if sys.platform == 'win32': # Windows: use CREATE_NEW_PROCESS_GROUP for proper handling process = subprocess.Popen( [sys.executable, str(main_script)], cwd=str(project_dir), creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) else: process = subprocess.Popen( [sys.executable, str(main_script)], cwd=str(project_dir), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) self._monitor_process = process self.state["monitor_running"] = True self.state["monitor_pid"] = process.pid for scraper in enabled: self.state["scrapers"][scraper]["running"] = True self._save_state() return {"success": True, "pid": process.pid, "scrapers": enabled} except Exception as e: return {"success": False, "error": str(e)} def stop_monitor(self) -> dict: """Stop the monitor process""" if not self.state["monitor_running"] and not self.state["monitor_pid"]: return {"success": False, "error": "Monitor not running"} try: pid = self.state["monitor_pid"] if pid: # On Windows, use taskkill for reliable process termination if sys.platform == 'win32': try: # Kill the process tree forcefully subprocess.run( ['taskkill', '/F', '/T', '/PID', str(pid)], capture_output=True, timeout=10 ) except Exception: pass else: # On Unix, use psutil try: import psutil proc = psutil.Process(pid) # Kill child processes first children = proc.children(recursive=True) for child in children: try: child.kill() # Use kill instead of terminate except: pass # Kill main process proc.kill() proc.wait(timeout=5) except: pass self.state["monitor_running"] = False self.state["monitor_pid"] = None for scraper in self.state["scrapers"]: self.state["scrapers"][scraper]["running"] = False self._save_state() return {"success": True} except Exception as e: # Still update state even if kill failed self.state["monitor_running"] = False self.state["monitor_pid"] = None for scraper in self.state["scrapers"]: self.state["scrapers"][scraper]["running"] = False self._save_state() return {"success": True, "warning": f"Process may need manual cleanup: {str(e)}"} def _update_config_file(self, enabled_scrapers: list): """Update config.py with enabled scrapers""" config_path = Path(__file__).parent.parent / "config.py" try: with open(config_path, 'r') as f: content = f.read() # Find and replace SITES_ENABLED line import re new_line = f'SITES_ENABLED = {json.dumps(enabled_scrapers)}' pattern = r'SITES_ENABLED\s*=\s*\[.*?\]' content = re.sub(pattern, new_line, content) # Also update CHECK_INTERVAL_SECONDS interval_line = f'CHECK_INTERVAL_SECONDS = {self.state["check_interval"]}' interval_pattern = r'CHECK_INTERVAL_SECONDS\s*=\s*\d+' content = re.sub(interval_pattern, interval_line, content) with open(config_path, 'w') as f: f.write(content) except Exception as e: print(f"Error updating config: {e}") # Singleton instance scraper_state = ScraperStateManager()