1152 lines
46 KiB
Python
1152 lines
46 KiB
Python
"""
|
|
SQLite database for Pokemon Stock Monitor stats tracking.
|
|
Stores product history, price changes, stock events, and favorites.
|
|
"""
|
|
|
|
import sqlite3
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Optional, List, Dict, Any
|
|
from contextlib import contextmanager
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Database file path
|
|
DB_PATH = Path(__file__).parent.parent / "data" / "stats.db"
|
|
|
|
|
|
class Database:
|
|
"""SQLite database wrapper for stats tracking"""
|
|
|
|
def __init__(self, db_path: str = None):
|
|
self.db_path = db_path or str(DB_PATH)
|
|
self._init_schema()
|
|
|
|
@contextmanager
|
|
def get_connection(self):
|
|
"""Context manager for database connections"""
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.row_factory = sqlite3.Row # Return rows as dictionaries
|
|
try:
|
|
yield conn
|
|
conn.commit()
|
|
except Exception as e:
|
|
conn.rollback()
|
|
raise e
|
|
finally:
|
|
conn.close()
|
|
|
|
def _init_schema(self):
|
|
"""Initialize database schema"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Products table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS products (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
url TEXT UNIQUE NOT NULL,
|
|
name TEXT NOT NULL,
|
|
site TEXT NOT NULL,
|
|
product_id TEXT,
|
|
image_url TEXT,
|
|
current_price TEXT,
|
|
in_stock BOOLEAN DEFAULT 0,
|
|
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
last_seen DATETIME,
|
|
category TEXT
|
|
)
|
|
""")
|
|
|
|
# Price history table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS price_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
product_id INTEGER NOT NULL,
|
|
price TEXT,
|
|
recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (product_id) REFERENCES products(id)
|
|
)
|
|
""")
|
|
|
|
# Stock events table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS stock_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
product_id INTEGER NOT NULL,
|
|
event_type TEXT NOT NULL,
|
|
recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (product_id) REFERENCES products(id)
|
|
)
|
|
""")
|
|
|
|
# Favorites table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS favorites (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
type TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
display_name TEXT,
|
|
priority TEXT DEFAULT 'high',
|
|
notify_discord BOOLEAN DEFAULT 1,
|
|
notify_sound BOOLEAN DEFAULT 0,
|
|
custom_webhook TEXT,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(type, value)
|
|
)
|
|
""")
|
|
|
|
# Check logs table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS check_logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
site TEXT,
|
|
products_found INTEGER DEFAULT 0,
|
|
new_products INTEGER DEFAULT 0,
|
|
restocks INTEGER DEFAULT 0,
|
|
duration_ms INTEGER,
|
|
success BOOLEAN DEFAULT 1,
|
|
error_message TEXT,
|
|
checked_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# News articles table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS news_articles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source TEXT NOT NULL,
|
|
source_account TEXT,
|
|
external_id TEXT UNIQUE,
|
|
title TEXT,
|
|
content TEXT NOT NULL,
|
|
url TEXT,
|
|
author TEXT,
|
|
image_url TEXT,
|
|
published_at DATETIME,
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
sentiment_score REAL,
|
|
sentiment_label TEXT,
|
|
keywords TEXT,
|
|
related_product_ids TEXT,
|
|
is_drop_related BOOLEAN DEFAULT 0,
|
|
is_restock_related BOOLEAN DEFAULT 0
|
|
)
|
|
""")
|
|
|
|
# Users table for multi-user support
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL UNIQUE,
|
|
zip_code TEXT,
|
|
radius_miles INTEGER DEFAULT 25,
|
|
discord_webhook TEXT,
|
|
notify_enabled BOOLEAN DEFAULT 1,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
last_active DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
# Checkout profiles table — stores AES-encrypted shipping + payment data
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS user_profiles (
|
|
user_id INTEGER PRIMARY KEY,
|
|
salt BLOB NOT NULL,
|
|
ciphertext BLOB NOT NULL,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
)
|
|
""")
|
|
|
|
# Create indexes for common queries
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_site ON products(site)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_product ON stock_events(product_id)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_type ON stock_events(event_type)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_check_logs_site ON check_logs(site)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_source ON news_articles(source)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_sentiment ON news_articles(sentiment_label)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_users_name ON users(name)")
|
|
|
|
logger.info("Database schema initialized")
|
|
|
|
# ==================== Product Methods ====================
|
|
|
|
def _normalize_url(self, url: str) -> str:
|
|
"""Normalize URL to prevent duplicates from minor variations."""
|
|
if not url:
|
|
return url
|
|
try:
|
|
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
|
parsed = urlparse(url)
|
|
netloc = parsed.netloc.lower()
|
|
path = parsed.path.rstrip('/')
|
|
# Filter out tracking/pagination params
|
|
query_params = parse_qs(parsed.query)
|
|
exclude_params = {
|
|
'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
|
|
'utm_term', 'gclid', 'fbclid', 'msclkid', 'srsltid', 'skuId',
|
|
'preselect', 'intl', 'lnk', 'nao', 'start', 'page', 'sort', 'sortby',
|
|
}
|
|
filtered = {k: v for k, v in query_params.items() if k.lower() not in exclude_params}
|
|
query = urlencode(sorted(filtered.items()), doseq=True) if filtered else ''
|
|
return urlunparse((parsed.scheme, netloc, path, '', query, ''))
|
|
except Exception:
|
|
return url
|
|
|
|
def get_or_create_product(self, url: str, name: str, site: str,
|
|
product_id: str = None, image_url: str = None,
|
|
price: str = None, in_stock: bool = False) -> int:
|
|
"""Get existing product or create new one. Returns product ID."""
|
|
category = self._detect_category(name)
|
|
normalized_url = self._normalize_url(url)
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Try to get existing product by URL (check both original and normalized)
|
|
cursor.execute("SELECT id FROM products WHERE url = ? OR url = ?", (url, normalized_url))
|
|
row = cursor.fetchone()
|
|
|
|
# Also check by product_id if provided (more reliable for some sites)
|
|
if not row and product_id and site:
|
|
cursor.execute("SELECT id FROM products WHERE product_id = ? AND site = ?", (product_id, site))
|
|
row = cursor.fetchone()
|
|
|
|
if row:
|
|
# Update existing product
|
|
cursor.execute("""
|
|
UPDATE products
|
|
SET name = ?, current_price = ?, in_stock = ?,
|
|
last_seen = CURRENT_TIMESTAMP, image_url = COALESCE(?, image_url),
|
|
category = COALESCE(?, category)
|
|
WHERE id = ?
|
|
""", (name, price, in_stock, image_url, category, row['id']))
|
|
return row['id']
|
|
else:
|
|
# Create new product with normalized URL
|
|
cursor.execute("""
|
|
INSERT INTO products (url, name, site, product_id, image_url,
|
|
current_price, in_stock, last_seen, category)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
|
|
""", (normalized_url, name, site, product_id, image_url, price, in_stock, category))
|
|
return cursor.lastrowid
|
|
|
|
def get_product(self, product_id: int) -> Optional[Dict]:
|
|
"""Get product by ID"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM products WHERE id = ?", (product_id,))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def get_product_by_url(self, url: str) -> Optional[Dict]:
|
|
"""Get product by URL (checks both original and normalized)"""
|
|
normalized_url = self._normalize_url(url)
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM products WHERE url = ? OR url = ?", (url, normalized_url))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def update_product_price(self, product_id: int, price: str):
|
|
"""Update product price and record in history"""
|
|
if not price:
|
|
return
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE products SET current_price = ?, last_seen = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (price, product_id))
|
|
|
|
# Record in price history (only if changed)
|
|
self.record_price(product_id, price)
|
|
|
|
def update_product_stock(self, product_id: int, in_stock: bool):
|
|
"""Update product stock status"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE products SET in_stock = ?, last_seen = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (in_stock, product_id))
|
|
|
|
def get_products(self, site: str = None, category: str = None,
|
|
in_stock: bool = None, favorites_only: bool = False,
|
|
event_type: str = None, period: str = None,
|
|
limit: int = 100, offset: int = 0) -> List[Dict]:
|
|
"""Get products with optional filters
|
|
|
|
Args:
|
|
event_type: Filter by recent event type ('new_drop' or 'restock')
|
|
period: Time period for event filter ('today' or 'week')
|
|
"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# If filtering by event type, use a join with stock_events
|
|
if event_type and period:
|
|
period_sql = "-1 day" if period == "today" else "-7 days"
|
|
query = """
|
|
SELECT DISTINCT p.* FROM products p
|
|
INNER JOIN stock_events e ON p.id = e.product_id
|
|
WHERE e.event_type = ?
|
|
AND e.recorded_at >= datetime('now', ?)
|
|
"""
|
|
params = [event_type, period_sql]
|
|
else:
|
|
query = "SELECT * FROM products WHERE 1=1"
|
|
params = []
|
|
|
|
if site:
|
|
query += " AND p.site = ?" if event_type else " AND site = ?"
|
|
params.append(site)
|
|
if category:
|
|
query += " AND p.category = ?" if event_type else " AND category = ?"
|
|
params.append(category)
|
|
if in_stock is not None:
|
|
query += " AND p.in_stock = ?" if event_type else " AND in_stock = ?"
|
|
params.append(in_stock)
|
|
if favorites_only:
|
|
prefix = "p." if event_type else ""
|
|
query += f""" AND (
|
|
{prefix}url IN (SELECT value FROM favorites WHERE type = 'product')
|
|
OR {prefix}category IN (SELECT value FROM favorites WHERE type = 'category')
|
|
)"""
|
|
|
|
order_col = "p.last_seen" if event_type else "last_seen"
|
|
query += f" ORDER BY {order_col} DESC LIMIT ? OFFSET ?"
|
|
params.extend([limit, offset])
|
|
|
|
cursor.execute(query, params)
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_product_count(self, site: str = None, in_stock: bool = None) -> int:
|
|
"""Get count of products matching filters"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = "SELECT COUNT(*) FROM products WHERE 1=1"
|
|
params = []
|
|
|
|
if site:
|
|
query += " AND site = ?"
|
|
params.append(site)
|
|
if in_stock is not None:
|
|
query += " AND in_stock = ?"
|
|
params.append(in_stock)
|
|
|
|
cursor.execute(query, params)
|
|
return cursor.fetchone()[0]
|
|
|
|
def _detect_category(self, name: str) -> str:
|
|
"""Auto-detect product category from name"""
|
|
name_lower = name.lower()
|
|
|
|
if "elite trainer" in name_lower or "etb" in name_lower:
|
|
return "ETB"
|
|
elif "booster bundle" in name_lower:
|
|
return "Booster Bundle"
|
|
elif "booster box" in name_lower:
|
|
return "Booster Box"
|
|
elif "booster pack" in name_lower or "sleeved booster" in name_lower:
|
|
return "Booster Pack"
|
|
elif "collection" in name_lower:
|
|
return "Collection Box"
|
|
elif "tin" in name_lower:
|
|
return "Tin"
|
|
elif "blister" in name_lower:
|
|
return "Blister"
|
|
elif "binder" in name_lower or "album" in name_lower:
|
|
return "Accessories"
|
|
else:
|
|
return "Other"
|
|
|
|
# ==================== Price History Methods ====================
|
|
|
|
def record_price(self, product_id: int, price: str):
|
|
"""Record a price point for a product"""
|
|
if not price:
|
|
return
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Check if price changed from last record
|
|
cursor.execute("""
|
|
SELECT price FROM price_history
|
|
WHERE product_id = ?
|
|
ORDER BY recorded_at DESC LIMIT 1
|
|
""", (product_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Only record if price changed or no history exists
|
|
if not row or row['price'] != price:
|
|
cursor.execute("""
|
|
INSERT INTO price_history (product_id, price)
|
|
VALUES (?, ?)
|
|
""", (product_id, price))
|
|
logger.debug(f"Recorded price change for product {product_id}: {price}")
|
|
|
|
def get_price_history(self, product_id: int, days: int = 30) -> List[Dict]:
|
|
"""Get price history for a product"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT price, recorded_at FROM price_history
|
|
WHERE product_id = ? AND recorded_at >= datetime('now', ?)
|
|
ORDER BY recorded_at ASC
|
|
""", (product_id, f"-{days} days"))
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
# ==================== Stock Event Methods ====================
|
|
|
|
def record_stock_event(self, product_id: int, event_type: str):
|
|
"""Record a stock event (new_drop, restock, in_stock, out_of_stock).
|
|
Prevents duplicate events within 1 hour for the same product/event_type."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Check if same event was recorded recently (within 1 hour)
|
|
cursor.execute("""
|
|
SELECT id FROM stock_events
|
|
WHERE product_id = ? AND event_type = ?
|
|
AND recorded_at > datetime('now', '-1 hour')
|
|
LIMIT 1
|
|
""", (product_id, event_type))
|
|
|
|
if cursor.fetchone():
|
|
logger.debug(f"Skipping duplicate {event_type} event for product {product_id}")
|
|
return
|
|
|
|
cursor.execute("""
|
|
INSERT INTO stock_events (product_id, event_type)
|
|
VALUES (?, ?)
|
|
""", (product_id, event_type))
|
|
logger.debug(f"Recorded {event_type} event for product {product_id}")
|
|
|
|
def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
|
|
"""Get recent stock events with product info.
|
|
Deduplicates by product name to avoid showing same product multiple times."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Use a subquery to get only the most recent event per product
|
|
# This prevents the same product from appearing multiple times
|
|
params = []
|
|
|
|
if event_types:
|
|
placeholders = ",".join("?" * len(event_types))
|
|
event_filter = f"e.event_type IN ({placeholders})"
|
|
event_filter2 = f"e2.event_type IN ({placeholders})"
|
|
params = list(event_types)
|
|
else:
|
|
event_filter = "1=1"
|
|
event_filter2 = "1=1"
|
|
|
|
query = f"""
|
|
SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
|
|
FROM stock_events e
|
|
JOIN products p ON e.product_id = p.id
|
|
WHERE {event_filter}
|
|
AND e.id IN (
|
|
SELECT MAX(e2.id)
|
|
FROM stock_events e2
|
|
JOIN products p2 ON e2.product_id = p2.id
|
|
WHERE {event_filter2}
|
|
GROUP BY LOWER(TRIM(p2.name))
|
|
)
|
|
ORDER BY e.recorded_at DESC
|
|
LIMIT ?
|
|
"""
|
|
# Add params for subquery event filter if needed
|
|
if event_types:
|
|
params.extend(event_types)
|
|
params.append(limit)
|
|
|
|
cursor.execute(query, params)
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_events_today(self, event_type: str = None) -> int:
|
|
"""Count events from today"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = """
|
|
SELECT COUNT(*) FROM stock_events
|
|
WHERE DATE(recorded_at) = DATE('now')
|
|
"""
|
|
params = []
|
|
|
|
if event_type:
|
|
query += " AND event_type = ?"
|
|
params.append(event_type)
|
|
|
|
cursor.execute(query, params)
|
|
return cursor.fetchone()[0]
|
|
|
|
def get_events_this_week(self, event_type: str = None) -> int:
|
|
"""Count events from this week"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = """
|
|
SELECT COUNT(*) FROM stock_events
|
|
WHERE recorded_at >= datetime('now', '-7 days')
|
|
"""
|
|
params = []
|
|
|
|
if event_type:
|
|
query += " AND event_type = ?"
|
|
params.append(event_type)
|
|
|
|
cursor.execute(query, params)
|
|
return cursor.fetchone()[0]
|
|
|
|
# ==================== Analytics Methods ====================
|
|
|
|
def get_drop_timing_stats(self, days: int = 30) -> List[Dict]:
|
|
"""Get drop timing by hour of day"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT
|
|
strftime('%H', recorded_at) as hour,
|
|
COUNT(*) as count
|
|
FROM stock_events
|
|
WHERE event_type IN ('new_drop', 'restock')
|
|
AND recorded_at >= datetime('now', ?)
|
|
GROUP BY hour
|
|
ORDER BY hour
|
|
""", (f"-{days} days",))
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_stock_duration_stats(self) -> List[Dict]:
|
|
"""Get average time items stay in stock"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
# Calculate duration between in_stock and out_of_stock events
|
|
cursor.execute("""
|
|
SELECT
|
|
p.category,
|
|
AVG(
|
|
CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER)
|
|
) as avg_minutes_in_stock
|
|
FROM stock_events in_event
|
|
JOIN stock_events out_event ON in_event.product_id = out_event.product_id
|
|
AND out_event.event_type = 'out_of_stock'
|
|
AND out_event.recorded_at > in_event.recorded_at
|
|
JOIN products p ON in_event.product_id = p.id
|
|
WHERE in_event.event_type IN ('restock', 'new_drop')
|
|
GROUP BY p.category
|
|
""")
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_site_stats(self) -> List[Dict]:
|
|
"""Get stats per site"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT
|
|
site,
|
|
COUNT(*) as total_products,
|
|
SUM(CASE WHEN in_stock = 1 THEN 1 ELSE 0 END) as in_stock_count
|
|
FROM products
|
|
GROUP BY site
|
|
""")
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
# ==================== Check Log Methods ====================
|
|
|
|
def log_check(self, site: str, products_found: int, new_products: int,
|
|
restocks: int, duration_ms: int, success: bool = True,
|
|
error_message: str = None):
|
|
"""Log a monitoring check"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT INTO check_logs (site, products_found, new_products,
|
|
restocks, duration_ms, success, error_message)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""", (site, products_found, new_products, restocks, duration_ms,
|
|
success, error_message))
|
|
|
|
def get_last_check(self, site: str = None) -> Optional[Dict]:
|
|
"""Get the most recent check log"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = "SELECT * FROM check_logs"
|
|
params = []
|
|
|
|
if site:
|
|
query += " WHERE site = ?"
|
|
params.append(site)
|
|
|
|
query += " ORDER BY checked_at DESC LIMIT 1"
|
|
cursor.execute(query, params)
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def get_check_history(self, site: str = None, limit: int = 100) -> List[Dict]:
|
|
"""Get check history"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = "SELECT * FROM check_logs"
|
|
params = []
|
|
|
|
if site:
|
|
query += " WHERE site = ?"
|
|
params.append(site)
|
|
|
|
query += " ORDER BY checked_at DESC LIMIT ?"
|
|
params.append(limit)
|
|
|
|
cursor.execute(query, params)
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
# ==================== Favorites Methods ====================
|
|
|
|
def add_favorite(self, fav_type: str, value: str, display_name: str = None,
|
|
priority: str = "high", notify_discord: bool = True,
|
|
notify_sound: bool = False, custom_webhook: str = None) -> int:
|
|
"""Add a favorite. Returns favorite ID."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT OR REPLACE INTO favorites
|
|
(type, value, display_name, priority, notify_discord, notify_sound, custom_webhook)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""", (fav_type, value, display_name or value, priority,
|
|
notify_discord, notify_sound, custom_webhook))
|
|
return cursor.lastrowid
|
|
|
|
def remove_favorite(self, favorite_id: int):
|
|
"""Remove a favorite by ID"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM favorites WHERE id = ?", (favorite_id,))
|
|
|
|
def get_favorites(self, fav_type: str = None) -> List[Dict]:
|
|
"""Get all favorites, optionally filtered by type"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
if fav_type:
|
|
cursor.execute("SELECT * FROM favorites WHERE type = ? ORDER BY created_at DESC",
|
|
(fav_type,))
|
|
else:
|
|
cursor.execute("SELECT * FROM favorites ORDER BY created_at DESC")
|
|
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_favorite(self, favorite_id: int) -> Optional[Dict]:
|
|
"""Get a favorite by ID"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM favorites WHERE id = ?", (favorite_id,))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def update_favorite(self, favorite_id: int, **kwargs):
|
|
"""Update favorite settings"""
|
|
allowed_fields = ['display_name', 'priority', 'notify_discord',
|
|
'notify_sound', 'custom_webhook']
|
|
updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
|
|
|
|
if not updates:
|
|
return
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
|
|
cursor.execute(
|
|
f"UPDATE favorites SET {set_clause} WHERE id = ?",
|
|
list(updates.values()) + [favorite_id]
|
|
)
|
|
|
|
def check_is_favorite(self, url: str = None, category: str = None) -> Optional[Dict]:
|
|
"""Check if a product URL or category is favorited"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
if url:
|
|
cursor.execute("""
|
|
SELECT * FROM favorites
|
|
WHERE type = 'product' AND value = ?
|
|
""", (url,))
|
|
row = cursor.fetchone()
|
|
if row:
|
|
return dict(row)
|
|
|
|
if category:
|
|
cursor.execute("""
|
|
SELECT * FROM favorites
|
|
WHERE type = 'category' AND value = ?
|
|
""", (category,))
|
|
row = cursor.fetchone()
|
|
if row:
|
|
return dict(row)
|
|
|
|
return None
|
|
|
|
# ==================== News Article Methods ====================
|
|
|
|
def add_news_article(self, source: str, content: str, source_account: str = None,
|
|
external_id: str = None, title: str = None, url: str = None,
|
|
author: str = None, image_url: str = None, published_at: str = None,
|
|
sentiment_score: float = None, sentiment_label: str = None,
|
|
keywords: List[str] = None, related_product_ids: List[int] = None,
|
|
is_drop_related: bool = False, is_restock_related: bool = False) -> int:
|
|
"""Add a news article. Returns article ID."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Check if article already exists by external_id
|
|
if external_id:
|
|
cursor.execute("SELECT id FROM news_articles WHERE external_id = ?", (external_id,))
|
|
existing = cursor.fetchone()
|
|
if existing:
|
|
return existing['id']
|
|
|
|
cursor.execute("""
|
|
INSERT INTO news_articles (source, source_account, external_id, title, content,
|
|
url, author, image_url, published_at, sentiment_score,
|
|
sentiment_label, keywords, related_product_ids,
|
|
is_drop_related, is_restock_related)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
source, source_account, external_id, title, content,
|
|
url, author, image_url, published_at, sentiment_score,
|
|
sentiment_label,
|
|
json.dumps(keywords) if keywords else None,
|
|
json.dumps(related_product_ids) if related_product_ids else None,
|
|
is_drop_related, is_restock_related
|
|
))
|
|
return cursor.lastrowid
|
|
|
|
def get_news_articles(self, source: str = None, sentiment: str = None,
|
|
drop_related: bool = None, limit: int = 50,
|
|
offset: int = 0) -> List[Dict]:
|
|
"""Get news articles with optional filters"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
query = "SELECT * FROM news_articles WHERE 1=1"
|
|
params = []
|
|
|
|
if source:
|
|
query += " AND source = ?"
|
|
params.append(source)
|
|
if sentiment:
|
|
query += " AND sentiment_label = ?"
|
|
params.append(sentiment)
|
|
if drop_related is not None:
|
|
query += " AND (is_drop_related = ? OR is_restock_related = ?)"
|
|
params.extend([drop_related, drop_related])
|
|
|
|
query += " ORDER BY published_at DESC, fetched_at DESC LIMIT ? OFFSET ?"
|
|
params.extend([limit, offset])
|
|
|
|
cursor.execute(query, params)
|
|
articles = []
|
|
for row in cursor.fetchall():
|
|
article = dict(row)
|
|
# Parse JSON fields
|
|
if article.get('keywords'):
|
|
article['keywords'] = json.loads(article['keywords'])
|
|
if article.get('related_product_ids'):
|
|
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
|
articles.append(article)
|
|
return articles
|
|
|
|
def get_news_article(self, article_id: int) -> Optional[Dict]:
|
|
"""Get a news article by ID"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM news_articles WHERE id = ?", (article_id,))
|
|
row = cursor.fetchone()
|
|
if row:
|
|
article = dict(row)
|
|
if article.get('keywords'):
|
|
article['keywords'] = json.loads(article['keywords'])
|
|
if article.get('related_product_ids'):
|
|
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
|
return article
|
|
return None
|
|
|
|
def get_news_stats(self) -> Dict:
|
|
"""Get news statistics"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Total articles
|
|
cursor.execute("SELECT COUNT(*) FROM news_articles")
|
|
total = cursor.fetchone()[0]
|
|
|
|
# By source
|
|
cursor.execute("""
|
|
SELECT source, COUNT(*) as count
|
|
FROM news_articles GROUP BY source
|
|
""")
|
|
by_source = {row['source']: row['count'] for row in cursor.fetchall()}
|
|
|
|
# By sentiment
|
|
cursor.execute("""
|
|
SELECT sentiment_label, COUNT(*) as count
|
|
FROM news_articles WHERE sentiment_label IS NOT NULL
|
|
GROUP BY sentiment_label
|
|
""")
|
|
by_sentiment = {row['sentiment_label']: row['count'] for row in cursor.fetchall()}
|
|
|
|
# Today's articles
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM news_articles
|
|
WHERE DATE(fetched_at) = DATE('now')
|
|
""")
|
|
today = cursor.fetchone()[0]
|
|
|
|
# Drop related
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM news_articles
|
|
WHERE is_drop_related = 1 OR is_restock_related = 1
|
|
""")
|
|
drop_related = cursor.fetchone()[0]
|
|
|
|
return {
|
|
'total': total,
|
|
'today': today,
|
|
'by_source': by_source,
|
|
'by_sentiment': by_sentiment,
|
|
'drop_related': drop_related
|
|
}
|
|
|
|
def update_news_correlation(self, article_id: int, related_product_ids: List[int],
|
|
is_drop_related: bool, is_restock_related: bool):
|
|
"""Update news article with correlation data"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE news_articles
|
|
SET related_product_ids = ?, is_drop_related = ?, is_restock_related = ?
|
|
WHERE id = ?
|
|
""", (json.dumps(related_product_ids), is_drop_related, is_restock_related, article_id))
|
|
|
|
def get_correlated_news(self, hours: int = 24) -> List[Dict]:
|
|
"""Get news articles correlated with recent drops"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT * FROM news_articles
|
|
WHERE (is_drop_related = 1 OR is_restock_related = 1)
|
|
AND fetched_at >= datetime('now', ?)
|
|
ORDER BY published_at DESC
|
|
""", (f"-{hours} hours",))
|
|
articles = []
|
|
for row in cursor.fetchall():
|
|
article = dict(row)
|
|
if article.get('keywords'):
|
|
article['keywords'] = json.loads(article['keywords'])
|
|
if article.get('related_product_ids'):
|
|
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
|
articles.append(article)
|
|
return articles
|
|
|
|
def delete_old_news(self, days: int = 30):
|
|
"""Delete news articles older than specified days"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
DELETE FROM news_articles
|
|
WHERE fetched_at < datetime('now', ?)
|
|
""", (f"-{days} days",))
|
|
deleted = cursor.rowcount
|
|
logger.info(f"Deleted {deleted} old news articles")
|
|
return deleted
|
|
|
|
# ==================== Migration Methods ====================
|
|
|
|
def migrate_from_json(self, json_path: str):
|
|
"""Migrate existing products.json to database"""
|
|
try:
|
|
with open(json_path, 'r') as f:
|
|
products = json.load(f)
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
logger.warning(f"Could not load {json_path} for migration")
|
|
return
|
|
|
|
migrated = 0
|
|
for url, data in products.items():
|
|
product_id = self.get_or_create_product(
|
|
url=url,
|
|
name=data.get('name', 'Unknown'),
|
|
site=data.get('site', 'unknown'),
|
|
product_id=data.get('product_id'),
|
|
image_url=data.get('image_url'),
|
|
price=data.get('price'),
|
|
in_stock=data.get('in_stock', False)
|
|
)
|
|
|
|
# Record initial price if available
|
|
if data.get('price'):
|
|
self.record_price(product_id, data['price'])
|
|
|
|
# Record as existing product (not new_drop since it's historical)
|
|
if data.get('in_stock'):
|
|
self.record_stock_event(product_id, 'in_stock')
|
|
|
|
migrated += 1
|
|
|
|
logger.info(f"Migrated {migrated} products from {json_path}")
|
|
|
|
def deduplicate_products(self) -> Dict:
|
|
"""
|
|
Find and remove duplicate products based on normalized URL or product_id.
|
|
Keeps the product with the most recent last_seen date.
|
|
Returns stats about what was deduplicated.
|
|
"""
|
|
stats = {'duplicates_found': 0, 'products_removed': 0, 'by_site': {}}
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Find duplicates by product_id (same site)
|
|
cursor.execute("""
|
|
SELECT site, product_id, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
|
FROM products
|
|
WHERE product_id IS NOT NULL AND product_id != ''
|
|
GROUP BY site, product_id
|
|
HAVING count > 1
|
|
""")
|
|
product_id_dupes = cursor.fetchall()
|
|
|
|
for row in product_id_dupes:
|
|
site = row['site']
|
|
ids = [int(i) for i in row['ids'].split(',')]
|
|
stats['duplicates_found'] += 1
|
|
|
|
# Keep the one with the most recent last_seen
|
|
cursor.execute("""
|
|
SELECT id FROM products
|
|
WHERE id IN ({})
|
|
ORDER BY last_seen DESC, first_seen ASC
|
|
LIMIT 1
|
|
""".format(','.join('?' * len(ids))), ids)
|
|
keep_id = cursor.fetchone()['id']
|
|
|
|
# Delete the others
|
|
delete_ids = [i for i in ids if i != keep_id]
|
|
if delete_ids:
|
|
cursor.execute("""
|
|
DELETE FROM products WHERE id IN ({})
|
|
""".format(','.join('?' * len(delete_ids))), delete_ids)
|
|
stats['products_removed'] += len(delete_ids)
|
|
stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
|
|
|
|
# Find duplicates by similar name (same site, fuzzy match)
|
|
cursor.execute("""
|
|
SELECT site, LOWER(TRIM(name)) as clean_name, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
|
FROM products
|
|
GROUP BY site, clean_name
|
|
HAVING count > 1
|
|
""")
|
|
name_dupes = cursor.fetchall()
|
|
|
|
for row in name_dupes:
|
|
site = row['site']
|
|
ids = [int(i) for i in row['ids'].split(',')]
|
|
stats['duplicates_found'] += 1
|
|
|
|
# Keep the one with the most recent last_seen
|
|
cursor.execute("""
|
|
SELECT id FROM products
|
|
WHERE id IN ({})
|
|
ORDER BY last_seen DESC, first_seen ASC
|
|
LIMIT 1
|
|
""".format(','.join('?' * len(ids))), ids)
|
|
keep_id = cursor.fetchone()['id']
|
|
|
|
# Delete the others
|
|
delete_ids = [i for i in ids if i != keep_id]
|
|
if delete_ids:
|
|
cursor.execute("""
|
|
DELETE FROM products WHERE id IN ({})
|
|
""".format(','.join('?' * len(delete_ids))), delete_ids)
|
|
stats['products_removed'] += len(delete_ids)
|
|
stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
|
|
|
|
logger.info(f"Deduplication complete: {stats}")
|
|
return stats
|
|
|
|
def deduplicate_events(self) -> Dict:
|
|
"""
|
|
Remove duplicate stock events (same product, same event type within 1 hour).
|
|
Keeps only the earliest event in each duplicate group.
|
|
Returns stats about what was removed.
|
|
"""
|
|
stats = {'duplicate_groups': 0, 'events_removed': 0}
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Find events that have duplicates within 1 hour
|
|
# We'll keep the earliest one in each group
|
|
cursor.execute("""
|
|
DELETE FROM stock_events
|
|
WHERE id NOT IN (
|
|
SELECT MIN(id)
|
|
FROM stock_events
|
|
GROUP BY product_id, event_type,
|
|
strftime('%Y-%m-%d %H', recorded_at)
|
|
)
|
|
""")
|
|
stats['events_removed'] = cursor.rowcount
|
|
|
|
logger.info(f"Event deduplication complete: removed {stats['events_removed']} duplicate events")
|
|
return stats
|
|
|
|
# ==================== User Methods ====================
|
|
|
|
def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,
|
|
discord_webhook: str = None, notify_enabled: bool = True) -> int:
|
|
"""Create a new user. Returns user ID."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT INTO users (name, zip_code, radius_miles, discord_webhook, notify_enabled)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""", (name, zip_code, radius_miles, discord_webhook, notify_enabled))
|
|
logger.info(f"Created user: {name}")
|
|
return cursor.lastrowid
|
|
|
|
def get_user(self, user_id: int) -> Optional[Dict]:
|
|
"""Get user by ID"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def get_user_by_name(self, name: str) -> Optional[Dict]:
|
|
"""Get user by name"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM users WHERE name = ?", (name,))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def get_all_users(self) -> List[Dict]:
|
|
"""Get all users"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT * FROM users ORDER BY name")
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def update_user(self, user_id: int, **kwargs) -> bool:
|
|
"""Update user settings"""
|
|
allowed_fields = ['name', 'zip_code', 'radius_miles', 'discord_webhook', 'notify_enabled']
|
|
updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
|
|
|
|
if not updates:
|
|
return False
|
|
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
|
|
cursor.execute(
|
|
f"UPDATE users SET {set_clause}, last_active = CURRENT_TIMESTAMP WHERE id = ?",
|
|
list(updates.values()) + [user_id]
|
|
)
|
|
logger.info(f"Updated user {user_id}: {updates}")
|
|
return cursor.rowcount > 0
|
|
|
|
def delete_user(self, user_id: int) -> bool:
|
|
"""Delete a user"""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM users WHERE id = ?", (user_id,))
|
|
return cursor.rowcount > 0
|
|
|
|
def update_user_location(self, user_id: int, zip_code: str, radius_miles: int = 25) -> bool:
|
|
"""Update user's location settings"""
|
|
return self.update_user(user_id, zip_code=zip_code, radius_miles=radius_miles)
|
|
|
|
# ==================== Checkout Profile Methods ====================
|
|
|
|
def save_user_profile(self, user_id: int, salt: bytes, ciphertext: bytes):
|
|
"""Save or replace an encrypted checkout profile for a user."""
|
|
with self.get_connection() as conn:
|
|
conn.execute(
|
|
"""INSERT INTO user_profiles (user_id, salt, ciphertext, updated_at)
|
|
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(user_id) DO UPDATE SET
|
|
salt=excluded.salt,
|
|
ciphertext=excluded.ciphertext,
|
|
updated_at=CURRENT_TIMESTAMP""",
|
|
(user_id, salt, ciphertext),
|
|
)
|
|
|
|
def get_user_profile(self, user_id: int) -> Optional[Dict]:
|
|
"""Return {salt, ciphertext} for a user, or None if no profile saved."""
|
|
with self.get_connection() as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
row = conn.execute(
|
|
"SELECT salt, ciphertext, updated_at FROM user_profiles WHERE user_id = ?",
|
|
(user_id,),
|
|
).fetchone()
|
|
if not row:
|
|
return None
|
|
return {"salt": row["salt"], "ciphertext": row["ciphertext"], "updated_at": row["updated_at"]}
|
|
|
|
def delete_user_profile(self, user_id: int) -> bool:
|
|
"""Delete a user's checkout profile."""
|
|
with self.get_connection() as conn:
|
|
cursor = conn.execute(
|
|
"DELETE FROM user_profiles WHERE user_id = ?", (user_id,)
|
|
)
|
|
return cursor.rowcount > 0
|
|
|
|
def user_has_profile(self, user_id: int) -> bool:
|
|
"""Return True if a saved (encrypted) profile exists for this user."""
|
|
with self.get_connection() as conn:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM user_profiles WHERE user_id = ?", (user_id,)
|
|
).fetchone()
|
|
return row is not None
|
|
|
|
# ==================== Stats Summary ====================
|
|
|
|
def get_dashboard_stats(self) -> Dict:
|
|
"""Get summary stats for dashboard"""
|
|
return {
|
|
'total_products': self.get_product_count(),
|
|
'in_stock_count': self.get_product_count(in_stock=True),
|
|
'new_drops_today': self.get_events_today('new_drop'),
|
|
'new_drops_week': self.get_events_this_week('new_drop'),
|
|
'restocks_today': self.get_events_today('restock'),
|
|
'restocks_week': self.get_events_this_week('restock'),
|
|
'last_check': self.get_last_check(),
|
|
'sites': self.get_site_stats(),
|
|
'favorites_count': len(self.get_favorites())
|
|
}
|
|
|
|
|
|
# Global database instance
|
|
_db: Database = None
|
|
|
|
|
|
def get_database() -> Database:
|
|
"""Get the global database instance"""
|
|
global _db
|
|
if _db is None:
|
|
_db = Database()
|
|
return _db
|