269 lines
9.6 KiB
Python
269 lines
9.6 KiB
Python
"""
|
|
Scraper State Manager
|
|
Manages runtime state for scrapers - enabling/disabling, running status, etc.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json"
|
|
|
|
# Default state
|
|
# Note: Pokemon Center uses Chrome Extension, not a Python scraper
|
|
DEFAULT_STATE = {
|
|
"scrapers": {
|
|
"target": {"enabled": True, "running": False, "last_run": None, "last_error": None},
|
|
"gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
|
"walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
|
"bestbuy": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
|
},
|
|
"monitor_running": False,
|
|
"monitor_pid": None,
|
|
"check_interval": 60,
|
|
"last_check": None,
|
|
}
|
|
|
|
|
|
class ScraperStateManager:
|
|
"""Manages scraper state with file persistence"""
|
|
|
|
_instance = None
|
|
_lock = threading.Lock()
|
|
_monitor_process = None
|
|
|
|
def __new__(cls):
|
|
if cls._instance is None:
|
|
cls._instance = super().__new__(cls)
|
|
cls._instance._initialized = False
|
|
return cls._instance
|
|
|
|
def __init__(self):
|
|
if self._initialized:
|
|
return
|
|
self._initialized = True
|
|
self.state = self._load_state()
|
|
|
|
def _load_state(self) -> dict:
|
|
"""Load state from file or return defaults"""
|
|
try:
|
|
if STATE_FILE.exists():
|
|
with open(STATE_FILE, 'r') as f:
|
|
saved = json.load(f)
|
|
# Merge with defaults in case new fields added
|
|
state = DEFAULT_STATE.copy()
|
|
state.update(saved)
|
|
# Ensure all scrapers exist
|
|
for scraper in DEFAULT_STATE["scrapers"]:
|
|
if scraper not in state["scrapers"]:
|
|
state["scrapers"][scraper] = DEFAULT_STATE["scrapers"][scraper]
|
|
return state
|
|
except Exception:
|
|
pass
|
|
return DEFAULT_STATE.copy()
|
|
|
|
def _save_state(self):
|
|
"""Save state to file using atomic write to prevent corruption"""
|
|
with self._lock:
|
|
try:
|
|
tmp_file = STATE_FILE.with_suffix('.json.tmp')
|
|
with open(tmp_file, 'w') as f:
|
|
json.dump(self.state, f, indent=2, default=str)
|
|
os.replace(tmp_file, STATE_FILE)
|
|
except Exception as e:
|
|
print(f"Error saving scraper state: {e}")
|
|
|
|
def get_state(self) -> dict:
|
|
"""Get current state, reloading from file to pick up changes from other processes"""
|
|
self.state = self._load_state()
|
|
|
|
# Check if monitor is still running
|
|
if self.state["monitor_pid"]:
|
|
try:
|
|
# Check if process exists
|
|
import psutil
|
|
if not psutil.pid_exists(self.state["monitor_pid"]):
|
|
self.state["monitor_running"] = False
|
|
self.state["monitor_pid"] = None
|
|
for scraper in self.state["scrapers"]:
|
|
self.state["scrapers"][scraper]["running"] = False
|
|
self._save_state()
|
|
except ImportError:
|
|
pass # psutil not installed, skip check
|
|
|
|
return self.state.copy()
|
|
|
|
def set_scraper_enabled(self, scraper: str, enabled: bool) -> bool:
|
|
"""Enable or disable a scraper"""
|
|
if scraper not in self.state["scrapers"]:
|
|
return False
|
|
|
|
self.state["scrapers"][scraper]["enabled"] = enabled
|
|
self._save_state()
|
|
return True
|
|
|
|
def get_enabled_scrapers(self) -> list:
|
|
"""Get list of enabled scraper names"""
|
|
return [
|
|
name for name, data in self.state["scrapers"].items()
|
|
if data.get("enabled", False)
|
|
]
|
|
|
|
def set_check_interval(self, seconds: int) -> bool:
|
|
"""Set check interval"""
|
|
if seconds < 10:
|
|
return False
|
|
self.state["check_interval"] = seconds
|
|
self._save_state()
|
|
return True
|
|
|
|
def record_check(self, scraper: str, success: bool, error: str = None):
|
|
"""Record a scraper check"""
|
|
if scraper in self.state["scrapers"]:
|
|
self.state["scrapers"][scraper]["last_run"] = datetime.now().isoformat()
|
|
if not success:
|
|
self.state["scrapers"][scraper]["last_error"] = error
|
|
else:
|
|
self.state["scrapers"][scraper]["last_error"] = None
|
|
self.state["last_check"] = datetime.now().isoformat()
|
|
self._save_state()
|
|
|
|
def start_monitor(self) -> dict:
|
|
"""Start the monitor process"""
|
|
if self.state["monitor_running"]:
|
|
return {"success": False, "error": "Monitor already running"}
|
|
|
|
enabled = self.get_enabled_scrapers()
|
|
if not enabled:
|
|
return {"success": False, "error": "No scrapers enabled"}
|
|
|
|
try:
|
|
# Start main.py as a subprocess
|
|
project_dir = Path(__file__).parent.parent
|
|
main_script = project_dir / "main.py"
|
|
|
|
# Create a modified config for enabled scrapers
|
|
self._update_config_file(enabled)
|
|
|
|
# Start the process
|
|
if sys.platform == 'win32':
|
|
# Windows: use CREATE_NEW_PROCESS_GROUP for proper handling
|
|
process = subprocess.Popen(
|
|
[sys.executable, str(main_script)],
|
|
cwd=str(project_dir),
|
|
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
else:
|
|
process = subprocess.Popen(
|
|
[sys.executable, str(main_script)],
|
|
cwd=str(project_dir),
|
|
start_new_session=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self._monitor_process = process
|
|
self.state["monitor_running"] = True
|
|
self.state["monitor_pid"] = process.pid
|
|
|
|
for scraper in enabled:
|
|
self.state["scrapers"][scraper]["running"] = True
|
|
|
|
self._save_state()
|
|
|
|
return {"success": True, "pid": process.pid, "scrapers": enabled}
|
|
|
|
except Exception as e:
|
|
return {"success": False, "error": str(e)}
|
|
|
|
def stop_monitor(self) -> dict:
|
|
"""Stop the monitor process"""
|
|
if not self.state["monitor_running"] and not self.state["monitor_pid"]:
|
|
return {"success": False, "error": "Monitor not running"}
|
|
|
|
try:
|
|
pid = self.state["monitor_pid"]
|
|
|
|
if pid:
|
|
# On Windows, use taskkill for reliable process termination
|
|
if sys.platform == 'win32':
|
|
try:
|
|
# Kill the process tree forcefully
|
|
subprocess.run(
|
|
['taskkill', '/F', '/T', '/PID', str(pid)],
|
|
capture_output=True,
|
|
timeout=10
|
|
)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
# On Unix, use psutil
|
|
try:
|
|
import psutil
|
|
proc = psutil.Process(pid)
|
|
# Kill child processes first
|
|
children = proc.children(recursive=True)
|
|
for child in children:
|
|
try:
|
|
child.kill() # Use kill instead of terminate
|
|
except:
|
|
pass
|
|
# Kill main process
|
|
proc.kill()
|
|
proc.wait(timeout=5)
|
|
except:
|
|
pass
|
|
|
|
self.state["monitor_running"] = False
|
|
self.state["monitor_pid"] = None
|
|
for scraper in self.state["scrapers"]:
|
|
self.state["scrapers"][scraper]["running"] = False
|
|
self._save_state()
|
|
|
|
return {"success": True}
|
|
|
|
except Exception as e:
|
|
# Still update state even if kill failed
|
|
self.state["monitor_running"] = False
|
|
self.state["monitor_pid"] = None
|
|
for scraper in self.state["scrapers"]:
|
|
self.state["scrapers"][scraper]["running"] = False
|
|
self._save_state()
|
|
return {"success": True, "warning": f"Process may need manual cleanup: {str(e)}"}
|
|
|
|
def _update_config_file(self, enabled_scrapers: list):
|
|
"""Update config.py with enabled scrapers"""
|
|
config_path = Path(__file__).parent.parent / "config.py"
|
|
|
|
try:
|
|
with open(config_path, 'r') as f:
|
|
content = f.read()
|
|
|
|
# Find and replace SITES_ENABLED line
|
|
import re
|
|
new_line = f'SITES_ENABLED = {json.dumps(enabled_scrapers)}'
|
|
pattern = r'SITES_ENABLED\s*=\s*\[.*?\]'
|
|
content = re.sub(pattern, new_line, content)
|
|
|
|
# Also update CHECK_INTERVAL_SECONDS
|
|
interval_line = f'CHECK_INTERVAL_SECONDS = {self.state["check_interval"]}'
|
|
interval_pattern = r'CHECK_INTERVAL_SECONDS\s*=\s*\d+'
|
|
content = re.sub(interval_pattern, interval_line, content)
|
|
|
|
with open(config_path, 'w') as f:
|
|
f.write(content)
|
|
|
|
except Exception as e:
|
|
print(f"Error updating config: {e}")
|
|
|
|
|
|
# Singleton instance
|
|
scraper_state = ScraperStateManager()
|