feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
+929
@@ -0,0 +1,929 @@
|
||||
"""
|
||||
SQLite database for Pokemon Stock Monitor stats tracking.
|
||||
Stores product history, price changes, stock events, and favorites.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
from contextlib import contextmanager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Database file path
|
||||
DB_PATH = Path(__file__).parent.parent / "data" / "stats.db"
|
||||
|
||||
|
||||
class Database:
|
||||
"""SQLite database wrapper for stats tracking"""
|
||||
|
||||
def __init__(self, db_path: str = None):
|
||||
self.db_path = db_path or str(DB_PATH)
|
||||
self._init_schema()
|
||||
|
||||
@contextmanager
|
||||
def get_connection(self):
|
||||
"""Context manager for database connections"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row # Return rows as dictionaries
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
raise e
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _init_schema(self):
|
||||
"""Initialize database schema"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Products table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS products (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
site TEXT NOT NULL,
|
||||
product_id TEXT,
|
||||
image_url TEXT,
|
||||
current_price TEXT,
|
||||
in_stock BOOLEAN DEFAULT 0,
|
||||
first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen DATETIME,
|
||||
category TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Price history table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS price_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
product_id INTEGER NOT NULL,
|
||||
price TEXT,
|
||||
recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (product_id) REFERENCES products(id)
|
||||
)
|
||||
""")
|
||||
|
||||
# Stock events table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS stock_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
product_id INTEGER NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (product_id) REFERENCES products(id)
|
||||
)
|
||||
""")
|
||||
|
||||
# Favorites table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS favorites (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
type TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
display_name TEXT,
|
||||
priority TEXT DEFAULT 'high',
|
||||
notify_discord BOOLEAN DEFAULT 1,
|
||||
notify_sound BOOLEAN DEFAULT 0,
|
||||
custom_webhook TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(type, value)
|
||||
)
|
||||
""")
|
||||
|
||||
# Check logs table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS check_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
site TEXT,
|
||||
products_found INTEGER DEFAULT 0,
|
||||
new_products INTEGER DEFAULT 0,
|
||||
restocks INTEGER DEFAULT 0,
|
||||
duration_ms INTEGER,
|
||||
success BOOLEAN DEFAULT 1,
|
||||
error_message TEXT,
|
||||
checked_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# News articles table
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS news_articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
source_account TEXT,
|
||||
external_id TEXT UNIQUE,
|
||||
title TEXT,
|
||||
content TEXT NOT NULL,
|
||||
url TEXT,
|
||||
author TEXT,
|
||||
image_url TEXT,
|
||||
published_at DATETIME,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
sentiment_score REAL,
|
||||
sentiment_label TEXT,
|
||||
keywords TEXT,
|
||||
related_product_ids TEXT,
|
||||
is_drop_related BOOLEAN DEFAULT 0,
|
||||
is_restock_related BOOLEAN DEFAULT 0
|
||||
)
|
||||
""")
|
||||
|
||||
# Users table for multi-user support
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
zip_code TEXT,
|
||||
radius_miles INTEGER DEFAULT 25,
|
||||
discord_webhook TEXT,
|
||||
notify_enabled BOOLEAN DEFAULT 1,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_active DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Create indexes for common queries
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_site ON products(site)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_product ON stock_events(product_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_type ON stock_events(event_type)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_check_logs_site ON check_logs(site)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_source ON news_articles(source)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_sentiment ON news_articles(sentiment_label)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_users_name ON users(name)")
|
||||
|
||||
logger.info("Database schema initialized")
|
||||
|
||||
# ==================== Product Methods ====================
|
||||
|
||||
def get_or_create_product(self, url: str, name: str, site: str,
|
||||
product_id: str = None, image_url: str = None,
|
||||
price: str = None, in_stock: bool = False) -> int:
|
||||
"""Get existing product or create new one. Returns product ID."""
|
||||
category = self._detect_category(name)
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Try to get existing product
|
||||
cursor.execute("SELECT id FROM products WHERE url = ?", (url,))
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
# Update existing product
|
||||
cursor.execute("""
|
||||
UPDATE products
|
||||
SET name = ?, current_price = ?, in_stock = ?,
|
||||
last_seen = CURRENT_TIMESTAMP, image_url = COALESCE(?, image_url),
|
||||
category = COALESCE(?, category)
|
||||
WHERE id = ?
|
||||
""", (name, price, in_stock, image_url, category, row['id']))
|
||||
return row['id']
|
||||
else:
|
||||
# Create new product
|
||||
cursor.execute("""
|
||||
INSERT INTO products (url, name, site, product_id, image_url,
|
||||
current_price, in_stock, last_seen, category)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
|
||||
""", (url, name, site, product_id, image_url, price, in_stock, category))
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_product(self, product_id: int) -> Optional[Dict]:
|
||||
"""Get product by ID"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM products WHERE id = ?", (product_id,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_product_by_url(self, url: str) -> Optional[Dict]:
|
||||
"""Get product by URL"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM products WHERE url = ?", (url,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def update_product_price(self, product_id: int, price: str):
|
||||
"""Update product price and record in history"""
|
||||
if not price:
|
||||
return
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE products SET current_price = ?, last_seen = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (price, product_id))
|
||||
|
||||
# Record in price history (only if changed)
|
||||
self.record_price(product_id, price)
|
||||
|
||||
def update_product_stock(self, product_id: int, in_stock: bool):
|
||||
"""Update product stock status"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE products SET in_stock = ?, last_seen = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (in_stock, product_id))
|
||||
|
||||
def get_products(self, site: str = None, category: str = None,
|
||||
in_stock: bool = None, favorites_only: bool = False,
|
||||
event_type: str = None, period: str = None,
|
||||
limit: int = 100, offset: int = 0) -> List[Dict]:
|
||||
"""Get products with optional filters
|
||||
|
||||
Args:
|
||||
event_type: Filter by recent event type ('new_drop' or 'restock')
|
||||
period: Time period for event filter ('today' or 'week')
|
||||
"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# If filtering by event type, use a join with stock_events
|
||||
if event_type and period:
|
||||
period_sql = "-1 day" if period == "today" else "-7 days"
|
||||
query = """
|
||||
SELECT DISTINCT p.* FROM products p
|
||||
INNER JOIN stock_events e ON p.id = e.product_id
|
||||
WHERE e.event_type = ?
|
||||
AND e.recorded_at >= datetime('now', ?)
|
||||
"""
|
||||
params = [event_type, period_sql]
|
||||
else:
|
||||
query = "SELECT * FROM products WHERE 1=1"
|
||||
params = []
|
||||
|
||||
if site:
|
||||
query += " AND p.site = ?" if event_type else " AND site = ?"
|
||||
params.append(site)
|
||||
if category:
|
||||
query += " AND p.category = ?" if event_type else " AND category = ?"
|
||||
params.append(category)
|
||||
if in_stock is not None:
|
||||
query += " AND p.in_stock = ?" if event_type else " AND in_stock = ?"
|
||||
params.append(in_stock)
|
||||
if favorites_only:
|
||||
prefix = "p." if event_type else ""
|
||||
query += f""" AND (
|
||||
{prefix}url IN (SELECT value FROM favorites WHERE type = 'product')
|
||||
OR {prefix}category IN (SELECT value FROM favorites WHERE type = 'category')
|
||||
)"""
|
||||
|
||||
order_col = "p.last_seen" if event_type else "last_seen"
|
||||
query += f" ORDER BY {order_col} DESC LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
cursor.execute(query, params)
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_product_count(self, site: str = None, in_stock: bool = None) -> int:
|
||||
"""Get count of products matching filters"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = "SELECT COUNT(*) FROM products WHERE 1=1"
|
||||
params = []
|
||||
|
||||
if site:
|
||||
query += " AND site = ?"
|
||||
params.append(site)
|
||||
if in_stock is not None:
|
||||
query += " AND in_stock = ?"
|
||||
params.append(in_stock)
|
||||
|
||||
cursor.execute(query, params)
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
def _detect_category(self, name: str) -> str:
|
||||
"""Auto-detect product category from name"""
|
||||
name_lower = name.lower()
|
||||
|
||||
if "elite trainer" in name_lower or "etb" in name_lower:
|
||||
return "ETB"
|
||||
elif "booster bundle" in name_lower:
|
||||
return "Booster Bundle"
|
||||
elif "booster box" in name_lower:
|
||||
return "Booster Box"
|
||||
elif "booster pack" in name_lower or "sleeved booster" in name_lower:
|
||||
return "Booster Pack"
|
||||
elif "collection" in name_lower:
|
||||
return "Collection Box"
|
||||
elif "tin" in name_lower:
|
||||
return "Tin"
|
||||
elif "blister" in name_lower:
|
||||
return "Blister"
|
||||
elif "binder" in name_lower or "album" in name_lower:
|
||||
return "Accessories"
|
||||
else:
|
||||
return "Other"
|
||||
|
||||
# ==================== Price History Methods ====================
|
||||
|
||||
def record_price(self, product_id: int, price: str):
|
||||
"""Record a price point for a product"""
|
||||
if not price:
|
||||
return
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if price changed from last record
|
||||
cursor.execute("""
|
||||
SELECT price FROM price_history
|
||||
WHERE product_id = ?
|
||||
ORDER BY recorded_at DESC LIMIT 1
|
||||
""", (product_id,))
|
||||
row = cursor.fetchone()
|
||||
|
||||
# Only record if price changed or no history exists
|
||||
if not row or row['price'] != price:
|
||||
cursor.execute("""
|
||||
INSERT INTO price_history (product_id, price)
|
||||
VALUES (?, ?)
|
||||
""", (product_id, price))
|
||||
logger.debug(f"Recorded price change for product {product_id}: {price}")
|
||||
|
||||
def get_price_history(self, product_id: int, days: int = 30) -> List[Dict]:
|
||||
"""Get price history for a product"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT price, recorded_at FROM price_history
|
||||
WHERE product_id = ? AND recorded_at >= datetime('now', ?)
|
||||
ORDER BY recorded_at ASC
|
||||
""", (product_id, f"-{days} days"))
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
# ==================== Stock Event Methods ====================
|
||||
|
||||
def record_stock_event(self, product_id: int, event_type: str):
|
||||
"""Record a stock event (new_drop, restock, in_stock, out_of_stock)"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO stock_events (product_id, event_type)
|
||||
VALUES (?, ?)
|
||||
""", (product_id, event_type))
|
||||
logger.debug(f"Recorded {event_type} event for product {product_id}")
|
||||
|
||||
def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
|
||||
"""Get recent stock events with product info"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = """
|
||||
SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
|
||||
FROM stock_events e
|
||||
JOIN products p ON e.product_id = p.id
|
||||
"""
|
||||
params = []
|
||||
|
||||
if event_types:
|
||||
placeholders = ",".join("?" * len(event_types))
|
||||
query += f" WHERE e.event_type IN ({placeholders})"
|
||||
params.extend(event_types)
|
||||
|
||||
query += " ORDER BY e.recorded_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
|
||||
cursor.execute(query, params)
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_events_today(self, event_type: str = None) -> int:
|
||||
"""Count events from today"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = """
|
||||
SELECT COUNT(*) FROM stock_events
|
||||
WHERE DATE(recorded_at) = DATE('now')
|
||||
"""
|
||||
params = []
|
||||
|
||||
if event_type:
|
||||
query += " AND event_type = ?"
|
||||
params.append(event_type)
|
||||
|
||||
cursor.execute(query, params)
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
def get_events_this_week(self, event_type: str = None) -> int:
|
||||
"""Count events from this week"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = """
|
||||
SELECT COUNT(*) FROM stock_events
|
||||
WHERE recorded_at >= datetime('now', '-7 days')
|
||||
"""
|
||||
params = []
|
||||
|
||||
if event_type:
|
||||
query += " AND event_type = ?"
|
||||
params.append(event_type)
|
||||
|
||||
cursor.execute(query, params)
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
# ==================== Analytics Methods ====================
|
||||
|
||||
def get_drop_timing_stats(self, days: int = 30) -> List[Dict]:
|
||||
"""Get drop timing by hour of day"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
strftime('%H', recorded_at) as hour,
|
||||
COUNT(*) as count
|
||||
FROM stock_events
|
||||
WHERE event_type IN ('new_drop', 'restock')
|
||||
AND recorded_at >= datetime('now', ?)
|
||||
GROUP BY hour
|
||||
ORDER BY hour
|
||||
""", (f"-{days} days",))
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_stock_duration_stats(self) -> List[Dict]:
|
||||
"""Get average time items stay in stock"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
# Calculate duration between in_stock and out_of_stock events
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
p.category,
|
||||
AVG(
|
||||
CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER)
|
||||
) as avg_minutes_in_stock
|
||||
FROM stock_events in_event
|
||||
JOIN stock_events out_event ON in_event.product_id = out_event.product_id
|
||||
AND out_event.event_type = 'out_of_stock'
|
||||
AND out_event.recorded_at > in_event.recorded_at
|
||||
JOIN products p ON in_event.product_id = p.id
|
||||
WHERE in_event.event_type IN ('restock', 'new_drop')
|
||||
GROUP BY p.category
|
||||
""")
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_site_stats(self) -> List[Dict]:
|
||||
"""Get stats per site"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
site,
|
||||
COUNT(*) as total_products,
|
||||
SUM(CASE WHEN in_stock = 1 THEN 1 ELSE 0 END) as in_stock_count
|
||||
FROM products
|
||||
GROUP BY site
|
||||
""")
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
# ==================== Check Log Methods ====================
|
||||
|
||||
def log_check(self, site: str, products_found: int, new_products: int,
|
||||
restocks: int, duration_ms: int, success: bool = True,
|
||||
error_message: str = None):
|
||||
"""Log a monitoring check"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO check_logs (site, products_found, new_products,
|
||||
restocks, duration_ms, success, error_message)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (site, products_found, new_products, restocks, duration_ms,
|
||||
success, error_message))
|
||||
|
||||
def get_last_check(self, site: str = None) -> Optional[Dict]:
|
||||
"""Get the most recent check log"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = "SELECT * FROM check_logs"
|
||||
params = []
|
||||
|
||||
if site:
|
||||
query += " WHERE site = ?"
|
||||
params.append(site)
|
||||
|
||||
query += " ORDER BY checked_at DESC LIMIT 1"
|
||||
cursor.execute(query, params)
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_check_history(self, site: str = None, limit: int = 100) -> List[Dict]:
|
||||
"""Get check history"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = "SELECT * FROM check_logs"
|
||||
params = []
|
||||
|
||||
if site:
|
||||
query += " WHERE site = ?"
|
||||
params.append(site)
|
||||
|
||||
query += " ORDER BY checked_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
|
||||
cursor.execute(query, params)
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
# ==================== Favorites Methods ====================
|
||||
|
||||
def add_favorite(self, fav_type: str, value: str, display_name: str = None,
|
||||
priority: str = "high", notify_discord: bool = True,
|
||||
notify_sound: bool = False, custom_webhook: str = None) -> int:
|
||||
"""Add a favorite. Returns favorite ID."""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO favorites
|
||||
(type, value, display_name, priority, notify_discord, notify_sound, custom_webhook)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (fav_type, value, display_name or value, priority,
|
||||
notify_discord, notify_sound, custom_webhook))
|
||||
return cursor.lastrowid
|
||||
|
||||
def remove_favorite(self, favorite_id: int):
|
||||
"""Remove a favorite by ID"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM favorites WHERE id = ?", (favorite_id,))
|
||||
|
||||
def get_favorites(self, fav_type: str = None) -> List[Dict]:
|
||||
"""Get all favorites, optionally filtered by type"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if fav_type:
|
||||
cursor.execute("SELECT * FROM favorites WHERE type = ? ORDER BY created_at DESC",
|
||||
(fav_type,))
|
||||
else:
|
||||
cursor.execute("SELECT * FROM favorites ORDER BY created_at DESC")
|
||||
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_favorite(self, favorite_id: int) -> Optional[Dict]:
|
||||
"""Get a favorite by ID"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM favorites WHERE id = ?", (favorite_id,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def update_favorite(self, favorite_id: int, **kwargs):
|
||||
"""Update favorite settings"""
|
||||
allowed_fields = ['display_name', 'priority', 'notify_discord',
|
||||
'notify_sound', 'custom_webhook']
|
||||
updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
|
||||
|
||||
if not updates:
|
||||
return
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
|
||||
cursor.execute(
|
||||
f"UPDATE favorites SET {set_clause} WHERE id = ?",
|
||||
list(updates.values()) + [favorite_id]
|
||||
)
|
||||
|
||||
def check_is_favorite(self, url: str = None, category: str = None) -> Optional[Dict]:
|
||||
"""Check if a product URL or category is favorited"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if url:
|
||||
cursor.execute("""
|
||||
SELECT * FROM favorites
|
||||
WHERE type = 'product' AND value = ?
|
||||
""", (url,))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return dict(row)
|
||||
|
||||
if category:
|
||||
cursor.execute("""
|
||||
SELECT * FROM favorites
|
||||
WHERE type = 'category' AND value = ?
|
||||
""", (category,))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return dict(row)
|
||||
|
||||
return None
|
||||
|
||||
# ==================== News Article Methods ====================
|
||||
|
||||
def add_news_article(self, source: str, content: str, source_account: str = None,
|
||||
external_id: str = None, title: str = None, url: str = None,
|
||||
author: str = None, image_url: str = None, published_at: str = None,
|
||||
sentiment_score: float = None, sentiment_label: str = None,
|
||||
keywords: List[str] = None, related_product_ids: List[int] = None,
|
||||
is_drop_related: bool = False, is_restock_related: bool = False) -> int:
|
||||
"""Add a news article. Returns article ID."""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if article already exists by external_id
|
||||
if external_id:
|
||||
cursor.execute("SELECT id FROM news_articles WHERE external_id = ?", (external_id,))
|
||||
existing = cursor.fetchone()
|
||||
if existing:
|
||||
return existing['id']
|
||||
|
||||
cursor.execute("""
|
||||
INSERT INTO news_articles (source, source_account, external_id, title, content,
|
||||
url, author, image_url, published_at, sentiment_score,
|
||||
sentiment_label, keywords, related_product_ids,
|
||||
is_drop_related, is_restock_related)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
source, source_account, external_id, title, content,
|
||||
url, author, image_url, published_at, sentiment_score,
|
||||
sentiment_label,
|
||||
json.dumps(keywords) if keywords else None,
|
||||
json.dumps(related_product_ids) if related_product_ids else None,
|
||||
is_drop_related, is_restock_related
|
||||
))
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_news_articles(self, source: str = None, sentiment: str = None,
|
||||
drop_related: bool = None, limit: int = 50,
|
||||
offset: int = 0) -> List[Dict]:
|
||||
"""Get news articles with optional filters"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = "SELECT * FROM news_articles WHERE 1=1"
|
||||
params = []
|
||||
|
||||
if source:
|
||||
query += " AND source = ?"
|
||||
params.append(source)
|
||||
if sentiment:
|
||||
query += " AND sentiment_label = ?"
|
||||
params.append(sentiment)
|
||||
if drop_related is not None:
|
||||
query += " AND (is_drop_related = ? OR is_restock_related = ?)"
|
||||
params.extend([drop_related, drop_related])
|
||||
|
||||
query += " ORDER BY published_at DESC, fetched_at DESC LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
cursor.execute(query, params)
|
||||
articles = []
|
||||
for row in cursor.fetchall():
|
||||
article = dict(row)
|
||||
# Parse JSON fields
|
||||
if article.get('keywords'):
|
||||
article['keywords'] = json.loads(article['keywords'])
|
||||
if article.get('related_product_ids'):
|
||||
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
||||
articles.append(article)
|
||||
return articles
|
||||
|
||||
def get_news_article(self, article_id: int) -> Optional[Dict]:
|
||||
"""Get a news article by ID"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM news_articles WHERE id = ?", (article_id,))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
article = dict(row)
|
||||
if article.get('keywords'):
|
||||
article['keywords'] = json.loads(article['keywords'])
|
||||
if article.get('related_product_ids'):
|
||||
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
||||
return article
|
||||
return None
|
||||
|
||||
def get_news_stats(self) -> Dict:
|
||||
"""Get news statistics"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Total articles
|
||||
cursor.execute("SELECT COUNT(*) FROM news_articles")
|
||||
total = cursor.fetchone()[0]
|
||||
|
||||
# By source
|
||||
cursor.execute("""
|
||||
SELECT source, COUNT(*) as count
|
||||
FROM news_articles GROUP BY source
|
||||
""")
|
||||
by_source = {row['source']: row['count'] for row in cursor.fetchall()}
|
||||
|
||||
# By sentiment
|
||||
cursor.execute("""
|
||||
SELECT sentiment_label, COUNT(*) as count
|
||||
FROM news_articles WHERE sentiment_label IS NOT NULL
|
||||
GROUP BY sentiment_label
|
||||
""")
|
||||
by_sentiment = {row['sentiment_label']: row['count'] for row in cursor.fetchall()}
|
||||
|
||||
# Today's articles
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM news_articles
|
||||
WHERE DATE(fetched_at) = DATE('now')
|
||||
""")
|
||||
today = cursor.fetchone()[0]
|
||||
|
||||
# Drop related
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM news_articles
|
||||
WHERE is_drop_related = 1 OR is_restock_related = 1
|
||||
""")
|
||||
drop_related = cursor.fetchone()[0]
|
||||
|
||||
return {
|
||||
'total': total,
|
||||
'today': today,
|
||||
'by_source': by_source,
|
||||
'by_sentiment': by_sentiment,
|
||||
'drop_related': drop_related
|
||||
}
|
||||
|
||||
def update_news_correlation(self, article_id: int, related_product_ids: List[int],
|
||||
is_drop_related: bool, is_restock_related: bool):
|
||||
"""Update news article with correlation data"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE news_articles
|
||||
SET related_product_ids = ?, is_drop_related = ?, is_restock_related = ?
|
||||
WHERE id = ?
|
||||
""", (json.dumps(related_product_ids), is_drop_related, is_restock_related, article_id))
|
||||
|
||||
def get_correlated_news(self, hours: int = 24) -> List[Dict]:
|
||||
"""Get news articles correlated with recent drops"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT * FROM news_articles
|
||||
WHERE (is_drop_related = 1 OR is_restock_related = 1)
|
||||
AND fetched_at >= datetime('now', ?)
|
||||
ORDER BY published_at DESC
|
||||
""", (f"-{hours} hours",))
|
||||
articles = []
|
||||
for row in cursor.fetchall():
|
||||
article = dict(row)
|
||||
if article.get('keywords'):
|
||||
article['keywords'] = json.loads(article['keywords'])
|
||||
if article.get('related_product_ids'):
|
||||
article['related_product_ids'] = json.loads(article['related_product_ids'])
|
||||
articles.append(article)
|
||||
return articles
|
||||
|
||||
def delete_old_news(self, days: int = 30):
|
||||
"""Delete news articles older than specified days"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
DELETE FROM news_articles
|
||||
WHERE fetched_at < datetime('now', ?)
|
||||
""", (f"-{days} days",))
|
||||
deleted = cursor.rowcount
|
||||
logger.info(f"Deleted {deleted} old news articles")
|
||||
return deleted
|
||||
|
||||
# ==================== Migration Methods ====================
|
||||
|
||||
def migrate_from_json(self, json_path: str):
|
||||
"""Migrate existing products.json to database"""
|
||||
try:
|
||||
with open(json_path, 'r') as f:
|
||||
products = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
logger.warning(f"Could not load {json_path} for migration")
|
||||
return
|
||||
|
||||
migrated = 0
|
||||
for url, data in products.items():
|
||||
product_id = self.get_or_create_product(
|
||||
url=url,
|
||||
name=data.get('name', 'Unknown'),
|
||||
site=data.get('site', 'unknown'),
|
||||
product_id=data.get('product_id'),
|
||||
image_url=data.get('image_url'),
|
||||
price=data.get('price'),
|
||||
in_stock=data.get('in_stock', False)
|
||||
)
|
||||
|
||||
# Record initial price if available
|
||||
if data.get('price'):
|
||||
self.record_price(product_id, data['price'])
|
||||
|
||||
# Record as existing product (not new_drop since it's historical)
|
||||
if data.get('in_stock'):
|
||||
self.record_stock_event(product_id, 'in_stock')
|
||||
|
||||
migrated += 1
|
||||
|
||||
logger.info(f"Migrated {migrated} products from {json_path}")
|
||||
|
||||
# ==================== User Methods ====================
|
||||
|
||||
def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,
|
||||
discord_webhook: str = None, notify_enabled: bool = True) -> int:
|
||||
"""Create a new user. Returns user ID."""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO users (name, zip_code, radius_miles, discord_webhook, notify_enabled)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (name, zip_code, radius_miles, discord_webhook, notify_enabled))
|
||||
logger.info(f"Created user: {name}")
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_user(self, user_id: int) -> Optional[Dict]:
|
||||
"""Get user by ID"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_user_by_name(self, name: str) -> Optional[Dict]:
|
||||
"""Get user by name"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM users WHERE name = ?", (name,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_all_users(self) -> List[Dict]:
|
||||
"""Get all users"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM users ORDER BY name")
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def update_user(self, user_id: int, **kwargs) -> bool:
|
||||
"""Update user settings"""
|
||||
allowed_fields = ['name', 'zip_code', 'radius_miles', 'discord_webhook', 'notify_enabled']
|
||||
updates = {k: v for k, v in kwargs.items() if k in allowed_fields}
|
||||
|
||||
if not updates:
|
||||
return False
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
|
||||
cursor.execute(
|
||||
f"UPDATE users SET {set_clause}, last_active = CURRENT_TIMESTAMP WHERE id = ?",
|
||||
list(updates.values()) + [user_id]
|
||||
)
|
||||
logger.info(f"Updated user {user_id}: {updates}")
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def delete_user(self, user_id: int) -> bool:
|
||||
"""Delete a user"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM users WHERE id = ?", (user_id,))
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def update_user_location(self, user_id: int, zip_code: str, radius_miles: int = 25) -> bool:
|
||||
"""Update user's location settings"""
|
||||
return self.update_user(user_id, zip_code=zip_code, radius_miles=radius_miles)
|
||||
|
||||
# ==================== Stats Summary ====================
|
||||
|
||||
def get_dashboard_stats(self) -> Dict:
|
||||
"""Get summary stats for dashboard"""
|
||||
return {
|
||||
'total_products': self.get_product_count(),
|
||||
'in_stock_count': self.get_product_count(in_stock=True),
|
||||
'new_drops_today': self.get_events_today('new_drop'),
|
||||
'new_drops_week': self.get_events_this_week('new_drop'),
|
||||
'restocks_today': self.get_events_today('restock'),
|
||||
'restocks_week': self.get_events_this_week('restock'),
|
||||
'last_check': self.get_last_check(),
|
||||
'sites': self.get_site_stats(),
|
||||
'favorites_count': len(self.get_favorites())
|
||||
}
|
||||
|
||||
|
||||
# Global database instance
|
||||
_db: Database = None
|
||||
|
||||
|
||||
def get_database() -> Database:
|
||||
"""Get the global database instance"""
|
||||
global _db
|
||||
if _db is None:
|
||||
_db = Database()
|
||||
return _db
|
||||
Reference in New Issue
Block a user