"""
SQLite database for Pokemon Stock Monitor stats tracking.
Stores product history, price changes, stock events, and favorites.
"""

import sqlite3
import json
import logging
from datetime import datetime, timedelta
from pathlib import Path
from typing import Optional, List, Dict, Any
from contextlib import contextmanager

logger = logging.getLogger(__name__)

# Database file path
DB_PATH = Path(__file__).parent.parent / "data" / "stats.db"


class Database:
    """SQLite database wrapper for stats tracking"""

    def __init__(self, db_path: str = None):
        self.db_path = db_path or str(DB_PATH)
        self._init_schema()

    @contextmanager
    def get_connection(self):
        """Context manager for database connections"""
        conn = sqlite3.connect(self.db_path)
        conn.row_factory = sqlite3.Row  # Return rows as dictionaries
        try:
            yield conn
            conn.commit()
        except Exception as e:
            conn.rollback()
            raise e
        finally:
            conn.close()

    def _init_schema(self):
        """Initialize database schema"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Products table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS products (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    url TEXT UNIQUE NOT NULL,
                    name TEXT NOT NULL,
                    site TEXT NOT NULL,
                    product_id TEXT,
                    image_url TEXT,
                    current_price TEXT,
                    in_stock BOOLEAN DEFAULT 0,
                    first_seen DATETIME DEFAULT CURRENT_TIMESTAMP,
                    last_seen DATETIME,
                    category TEXT
                )
            """)

            # Price history table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS price_history (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    product_id INTEGER NOT NULL,
                    price TEXT,
                    recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    FOREIGN KEY (product_id) REFERENCES products(id)
                )
            """)

            # Stock events table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS stock_events (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    product_id INTEGER NOT NULL,
                    event_type TEXT NOT NULL,
                    recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    FOREIGN KEY (product_id) REFERENCES products(id)
                )
            """)

            # Favorites table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS favorites (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    type TEXT NOT NULL,
                    value TEXT NOT NULL,
                    display_name TEXT,
                    priority TEXT DEFAULT 'high',
                    notify_discord BOOLEAN DEFAULT 1,
                    notify_sound BOOLEAN DEFAULT 0,
                    custom_webhook TEXT,
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    UNIQUE(type, value)
                )
            """)

            # Check logs table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS check_logs (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    site TEXT,
                    products_found INTEGER DEFAULT 0,
                    new_products INTEGER DEFAULT 0,
                    restocks INTEGER DEFAULT 0,
                    duration_ms INTEGER,
                    success BOOLEAN DEFAULT 1,
                    error_message TEXT,
                    checked_at DATETIME DEFAULT CURRENT_TIMESTAMP
                )
            """)

            # News articles table
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS news_articles (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    source TEXT NOT NULL,
                    source_account TEXT,
                    external_id TEXT UNIQUE,
                    title TEXT,
                    content TEXT NOT NULL,
                    url TEXT,
                    author TEXT,
                    image_url TEXT,
                    published_at DATETIME,
                    fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    sentiment_score REAL,
                    sentiment_label TEXT,
                    keywords TEXT,
                    related_product_ids TEXT,
                    is_drop_related BOOLEAN DEFAULT 0,
                    is_restock_related BOOLEAN DEFAULT 0
                )
            """)

            # Users table for multi-user support
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS users (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    name TEXT NOT NULL UNIQUE,
                    zip_code TEXT,
                    radius_miles INTEGER DEFAULT 25,
                    discord_webhook TEXT,
                    notify_enabled BOOLEAN DEFAULT 1,
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    last_active DATETIME DEFAULT CURRENT_TIMESTAMP
                )
            """)

            # Checkout profiles table — stores AES-encrypted shipping + payment data
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS user_profiles (
                    user_id INTEGER PRIMARY KEY,
                    salt BLOB NOT NULL,
                    ciphertext BLOB NOT NULL,
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
                    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
                )
            """)

            # Create indexes for common queries
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_site ON products(site)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_product ON stock_events(product_id)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_stock_events_type ON stock_events(event_type)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_price_history_product ON price_history(product_id)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_check_logs_site ON check_logs(site)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_source ON news_articles(source)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_news_sentiment ON news_articles(sentiment_label)")
            cursor.execute("CREATE INDEX IF NOT EXISTS idx_users_name ON users(name)")

            logger.info("Database schema initialized")

    # ==================== Product Methods ====================

    def _normalize_url(self, url: str) -> str:
        """Normalize URL to prevent duplicates from minor variations."""
        if not url:
            return url
        try:
            from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
            parsed = urlparse(url)
            netloc = parsed.netloc.lower()
            path = parsed.path.rstrip('/')
            # Filter out tracking/pagination params
            query_params = parse_qs(parsed.query)
            exclude_params = {
                'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
                'utm_term', 'gclid', 'fbclid', 'msclkid', 'srsltid', 'skuId',
                'preselect', 'intl', 'lnk', 'nao', 'start', 'page', 'sort', 'sortby',
            }
            filtered = {k: v for k, v in query_params.items() if k.lower() not in exclude_params}
            query = urlencode(sorted(filtered.items()), doseq=True) if filtered else ''
            return urlunparse((parsed.scheme, netloc, path, '', query, ''))
        except Exception:
            return url

    def get_or_create_product(self, url: str, name: str, site: str,
                               product_id: str = None, image_url: str = None,
                               price: str = None, in_stock: bool = False) -> int:
        """Get existing product or create new one. Returns product ID."""
        category = self._detect_category(name)
        normalized_url = self._normalize_url(url)

        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Try to get existing product by URL (check both original and normalized)
            cursor.execute("SELECT id FROM products WHERE url = ? OR url = ?", (url, normalized_url))
            row = cursor.fetchone()

            # Also check by product_id if provided (more reliable for some sites)
            if not row and product_id and site:
                cursor.execute("SELECT id FROM products WHERE product_id = ? AND site = ?", (product_id, site))
                row = cursor.fetchone()

            if row:
                # Update existing product
                cursor.execute("""
                    UPDATE products
                    SET name = ?, current_price = ?, in_stock = ?,
                        last_seen = CURRENT_TIMESTAMP, image_url = COALESCE(?, image_url),
                        category = COALESCE(?, category)
                    WHERE id = ?
                """, (name, price, in_stock, image_url, category, row['id']))
                return row['id']
            else:
                # Create new product with normalized URL
                cursor.execute("""
                    INSERT INTO products (url, name, site, product_id, image_url,
                                         current_price, in_stock, last_seen, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
                """, (normalized_url, name, site, product_id, image_url, price, in_stock, category))
                return cursor.lastrowid

    def get_product(self, product_id: int) -> Optional[Dict]:
        """Get product by ID"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM products WHERE id = ?", (product_id,))
            row = cursor.fetchone()
            return dict(row) if row else None

    def get_product_by_url(self, url: str) -> Optional[Dict]:
        """Get product by URL (checks both original and normalized)"""
        normalized_url = self._normalize_url(url)
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM products WHERE url = ? OR url = ?", (url, normalized_url))
            row = cursor.fetchone()
            return dict(row) if row else None

    def update_product_price(self, product_id: int, price: str):
        """Update product price and record in history"""
        if not price:
            return

        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                UPDATE products SET current_price = ?, last_seen = CURRENT_TIMESTAMP
                WHERE id = ?
            """, (price, product_id))

        # Record in price history (only if changed)
        self.record_price(product_id, price)

    def update_product_stock(self, product_id: int, in_stock: bool):
        """Update product stock status"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                UPDATE products SET in_stock = ?, last_seen = CURRENT_TIMESTAMP
                WHERE id = ?
            """, (in_stock, product_id))

    def get_products(self, site: str = None, category: str = None,
                     in_stock: bool = None, favorites_only: bool = False,
                     event_type: str = None, period: str = None,
                     limit: int = 100, offset: int = 0) -> List[Dict]:
        """Get products with optional filters

        Args:
            event_type: Filter by recent event type ('new_drop' or 'restock')
            period: Time period for event filter ('today' or 'week')
        """
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # If filtering by event type, use a join with stock_events
            if event_type and period:
                period_sql = "-1 day" if period == "today" else "-7 days"
                query = """
                    SELECT DISTINCT p.* FROM products p
                    INNER JOIN stock_events e ON p.id = e.product_id
                    WHERE e.event_type = ?
                    AND e.recorded_at >= datetime('now', ?)
                """
                params = [event_type, period_sql]
            else:
                query = "SELECT * FROM products WHERE 1=1"
                params = []

            if site:
                query += " AND p.site = ?" if event_type else " AND site = ?"
                params.append(site)
            if category:
                query += " AND p.category = ?" if event_type else " AND category = ?"
                params.append(category)
            if in_stock is not None:
                query += " AND p.in_stock = ?" if event_type else " AND in_stock = ?"
                params.append(in_stock)
            if favorites_only:
                prefix = "p." if event_type else ""
                query += f""" AND (
                    {prefix}url IN (SELECT value FROM favorites WHERE type = 'product')
                    OR {prefix}category IN (SELECT value FROM favorites WHERE type = 'category')
                )"""

            order_col = "p.last_seen" if event_type else "last_seen"
            query += f" ORDER BY {order_col} DESC LIMIT ? OFFSET ?"
            params.extend([limit, offset])

            cursor.execute(query, params)
            return [dict(row) for row in cursor.fetchall()]

    def get_product_count(self, site: str = None, in_stock: bool = None) -> int:
        """Get count of products matching filters"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT COUNT(*) FROM products WHERE 1=1"
            params = []

            if site:
                query += " AND site = ?"
                params.append(site)
            if in_stock is not None:
                query += " AND in_stock = ?"
                params.append(in_stock)

            cursor.execute(query, params)
            return cursor.fetchone()[0]

    def _detect_category(self, name: str) -> str:
        """Auto-detect product category from name"""
        name_lower = name.lower()

        if "elite trainer" in name_lower or "etb" in name_lower:
            return "ETB"
        elif "booster bundle" in name_lower:
            return "Booster Bundle"
        elif "booster box" in name_lower:
            return "Booster Box"
        elif "booster pack" in name_lower or "sleeved booster" in name_lower:
            return "Booster Pack"
        elif "collection" in name_lower:
            return "Collection Box"
        elif "tin" in name_lower:
            return "Tin"
        elif "blister" in name_lower:
            return "Blister"
        elif "binder" in name_lower or "album" in name_lower:
            return "Accessories"
        else:
            return "Other"

    # ==================== Price History Methods ====================

    def record_price(self, product_id: int, price: str):
        """Record a price point for a product"""
        if not price:
            return

        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Check if price changed from last record
            cursor.execute("""
                SELECT price FROM price_history
                WHERE product_id = ?
                ORDER BY recorded_at DESC LIMIT 1
            """, (product_id,))
            row = cursor.fetchone()

            # Only record if price changed or no history exists
            if not row or row['price'] != price:
                cursor.execute("""
                    INSERT INTO price_history (product_id, price)
                    VALUES (?, ?)
                """, (product_id, price))
                logger.debug(f"Recorded price change for product {product_id}: {price}")

    def get_price_history(self, product_id: int, days: int = 30) -> List[Dict]:
        """Get price history for a product"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                SELECT price, recorded_at FROM price_history
                WHERE product_id = ? AND recorded_at >= datetime('now', ?)
                ORDER BY recorded_at ASC
            """, (product_id, f"-{days} days"))
            return [dict(row) for row in cursor.fetchall()]

    # ==================== Stock Event Methods ====================

    def record_stock_event(self, product_id: int, event_type: str):
        """Record a stock event (new_drop, restock, in_stock, out_of_stock).
        Prevents duplicate events within 1 hour for the same product/event_type."""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Check if same event was recorded recently (within 1 hour)
            cursor.execute("""
                SELECT id FROM stock_events
                WHERE product_id = ? AND event_type = ?
                AND recorded_at > datetime('now', '-1 hour')
                LIMIT 1
            """, (product_id, event_type))

            if cursor.fetchone():
                logger.debug(f"Skipping duplicate {event_type} event for product {product_id}")
                return

            cursor.execute("""
                INSERT INTO stock_events (product_id, event_type)
                VALUES (?, ?)
            """, (product_id, event_type))
            logger.debug(f"Recorded {event_type} event for product {product_id}")

    def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
        """Get recent stock events with product info.
        Deduplicates by product name to avoid showing same product multiple times."""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Use a subquery to get only the most recent event per product
            # This prevents the same product from appearing multiple times
            params = []

            if event_types:
                placeholders = ",".join("?" * len(event_types))
                event_filter = f"e.event_type IN ({placeholders})"
                event_filter2 = f"e2.event_type IN ({placeholders})"
                params = list(event_types)
            else:
                event_filter = "1=1"
                event_filter2 = "1=1"

            query = f"""
                SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
                FROM stock_events e
                JOIN products p ON e.product_id = p.id
                WHERE {event_filter}
                AND e.id IN (
                    SELECT MAX(e2.id)
                    FROM stock_events e2
                    JOIN products p2 ON e2.product_id = p2.id
                    WHERE {event_filter2}
                    GROUP BY LOWER(TRIM(p2.name))
                )
                ORDER BY e.recorded_at DESC
                LIMIT ?
            """
            # Add params for subquery event filter if needed
            if event_types:
                params.extend(event_types)
            params.append(limit)

            cursor.execute(query, params)
            return [dict(row) for row in cursor.fetchall()]

    def get_events_today(self, event_type: str = None) -> int:
        """Count events from today"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = """
                SELECT COUNT(*) FROM stock_events
                WHERE DATE(recorded_at) = DATE('now')
            """
            params = []

            if event_type:
                query += " AND event_type = ?"
                params.append(event_type)

            cursor.execute(query, params)
            return cursor.fetchone()[0]

    def get_events_this_week(self, event_type: str = None) -> int:
        """Count events from this week"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = """
                SELECT COUNT(*) FROM stock_events
                WHERE recorded_at >= datetime('now', '-7 days')
            """
            params = []

            if event_type:
                query += " AND event_type = ?"
                params.append(event_type)

            cursor.execute(query, params)
            return cursor.fetchone()[0]

    # ==================== Analytics Methods ====================

    def get_drop_timing_stats(self, days: int = 30) -> List[Dict]:
        """Get drop timing by hour of day"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                SELECT
                    strftime('%H', recorded_at) as hour,
                    COUNT(*) as count
                FROM stock_events
                WHERE event_type IN ('new_drop', 'restock')
                AND recorded_at >= datetime('now', ?)
                GROUP BY hour
                ORDER BY hour
            """, (f"-{days} days",))
            return [dict(row) for row in cursor.fetchall()]

    def get_stock_duration_stats(self) -> List[Dict]:
        """Get average time items stay in stock"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            # Calculate duration between in_stock and out_of_stock events
            cursor.execute("""
                SELECT
                    p.category,
                    AVG(
                        CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER)
                    ) as avg_minutes_in_stock
                FROM stock_events in_event
                JOIN stock_events out_event ON in_event.product_id = out_event.product_id
                    AND out_event.event_type = 'out_of_stock'
                    AND out_event.recorded_at > in_event.recorded_at
                JOIN products p ON in_event.product_id = p.id
                WHERE in_event.event_type IN ('restock', 'new_drop')
                GROUP BY p.category
            """)
            return [dict(row) for row in cursor.fetchall()]

    def get_site_stats(self) -> List[Dict]:
        """Get stats per site"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                SELECT
                    site,
                    COUNT(*) as total_products,
                    SUM(CASE WHEN in_stock = 1 THEN 1 ELSE 0 END) as in_stock_count
                FROM products
                GROUP BY site
            """)
            return [dict(row) for row in cursor.fetchall()]

    # ==================== Check Log Methods ====================

    def log_check(self, site: str, products_found: int, new_products: int,
                  restocks: int, duration_ms: int, success: bool = True,
                  error_message: str = None):
        """Log a monitoring check"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                INSERT INTO check_logs (site, products_found, new_products,
                                       restocks, duration_ms, success, error_message)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (site, products_found, new_products, restocks, duration_ms,
                  success, error_message))

    def get_last_check(self, site: str = None) -> Optional[Dict]:
        """Get the most recent check log"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM check_logs"
            params = []

            if site:
                query += " WHERE site = ?"
                params.append(site)

            query += " ORDER BY checked_at DESC LIMIT 1"
            cursor.execute(query, params)
            row = cursor.fetchone()
            return dict(row) if row else None

    def get_check_history(self, site: str = None, limit: int = 100) -> List[Dict]:
        """Get check history"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM check_logs"
            params = []

            if site:
                query += " WHERE site = ?"
                params.append(site)

            query += " ORDER BY checked_at DESC LIMIT ?"
            params.append(limit)

            cursor.execute(query, params)
            return [dict(row) for row in cursor.fetchall()]

    # ==================== Favorites Methods ====================

    def add_favorite(self, fav_type: str, value: str, display_name: str = None,
                     priority: str = "high", notify_discord: bool = True,
                     notify_sound: bool = False, custom_webhook: str = None) -> int:
        """Add a favorite. Returns favorite ID."""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                INSERT OR REPLACE INTO favorites
                (type, value, display_name, priority, notify_discord, notify_sound, custom_webhook)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (fav_type, value, display_name or value, priority,
                  notify_discord, notify_sound, custom_webhook))
            return cursor.lastrowid

    def remove_favorite(self, favorite_id: int):
        """Remove a favorite by ID"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("DELETE FROM favorites WHERE id = ?", (favorite_id,))

    def get_favorites(self, fav_type: str = None) -> List[Dict]:
        """Get all favorites, optionally filtered by type"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            if fav_type:
                cursor.execute("SELECT * FROM favorites WHERE type = ? ORDER BY created_at DESC",
                              (fav_type,))
            else:
                cursor.execute("SELECT * FROM favorites ORDER BY created_at DESC")

            return [dict(row) for row in cursor.fetchall()]

    def get_favorite(self, favorite_id: int) -> Optional[Dict]:
        """Get a favorite by ID"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM favorites WHERE id = ?", (favorite_id,))
            row = cursor.fetchone()
            return dict(row) if row else None

    def update_favorite(self, favorite_id: int, **kwargs):
        """Update favorite settings"""
        allowed_fields = ['display_name', 'priority', 'notify_discord',
                         'notify_sound', 'custom_webhook']
        updates = {k: v for k, v in kwargs.items() if k in allowed_fields}

        if not updates:
            return

        with self.get_connection() as conn:
            cursor = conn.cursor()
            set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
            cursor.execute(
                f"UPDATE favorites SET {set_clause} WHERE id = ?",
                list(updates.values()) + [favorite_id]
            )

    def check_is_favorite(self, url: str = None, category: str = None) -> Optional[Dict]:
        """Check if a product URL or category is favorited"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            if url:
                cursor.execute("""
                    SELECT * FROM favorites
                    WHERE type = 'product' AND value = ?
                """, (url,))
                row = cursor.fetchone()
                if row:
                    return dict(row)

            if category:
                cursor.execute("""
                    SELECT * FROM favorites
                    WHERE type = 'category' AND value = ?
                """, (category,))
                row = cursor.fetchone()
                if row:
                    return dict(row)

            return None

    # ==================== News Article Methods ====================

    def add_news_article(self, source: str, content: str, source_account: str = None,
                         external_id: str = None, title: str = None, url: str = None,
                         author: str = None, image_url: str = None, published_at: str = None,
                         sentiment_score: float = None, sentiment_label: str = None,
                         keywords: List[str] = None, related_product_ids: List[int] = None,
                         is_drop_related: bool = False, is_restock_related: bool = False) -> int:
        """Add a news article. Returns article ID."""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Check if article already exists by external_id
            if external_id:
                cursor.execute("SELECT id FROM news_articles WHERE external_id = ?", (external_id,))
                existing = cursor.fetchone()
                if existing:
                    return existing['id']

            cursor.execute("""
                INSERT INTO news_articles (source, source_account, external_id, title, content,
                                          url, author, image_url, published_at, sentiment_score,
                                          sentiment_label, keywords, related_product_ids,
                                          is_drop_related, is_restock_related)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                source, source_account, external_id, title, content,
                url, author, image_url, published_at, sentiment_score,
                sentiment_label,
                json.dumps(keywords) if keywords else None,
                json.dumps(related_product_ids) if related_product_ids else None,
                is_drop_related, is_restock_related
            ))
            return cursor.lastrowid

    def get_news_articles(self, source: str = None, sentiment: str = None,
                          drop_related: bool = None, limit: int = 50,
                          offset: int = 0) -> List[Dict]:
        """Get news articles with optional filters"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            query = "SELECT * FROM news_articles WHERE 1=1"
            params = []

            if source:
                query += " AND source = ?"
                params.append(source)
            if sentiment:
                query += " AND sentiment_label = ?"
                params.append(sentiment)
            if drop_related is not None:
                query += " AND (is_drop_related = ? OR is_restock_related = ?)"
                params.extend([drop_related, drop_related])

            query += " ORDER BY published_at DESC, fetched_at DESC LIMIT ? OFFSET ?"
            params.extend([limit, offset])

            cursor.execute(query, params)
            articles = []
            for row in cursor.fetchall():
                article = dict(row)
                # Parse JSON fields
                if article.get('keywords'):
                    article['keywords'] = json.loads(article['keywords'])
                if article.get('related_product_ids'):
                    article['related_product_ids'] = json.loads(article['related_product_ids'])
                articles.append(article)
            return articles

    def get_news_article(self, article_id: int) -> Optional[Dict]:
        """Get a news article by ID"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM news_articles WHERE id = ?", (article_id,))
            row = cursor.fetchone()
            if row:
                article = dict(row)
                if article.get('keywords'):
                    article['keywords'] = json.loads(article['keywords'])
                if article.get('related_product_ids'):
                    article['related_product_ids'] = json.loads(article['related_product_ids'])
                return article
            return None

    def get_news_stats(self) -> Dict:
        """Get news statistics"""
        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Total articles
            cursor.execute("SELECT COUNT(*) FROM news_articles")
            total = cursor.fetchone()[0]

            # By source
            cursor.execute("""
                SELECT source, COUNT(*) as count
                FROM news_articles GROUP BY source
            """)
            by_source = {row['source']: row['count'] for row in cursor.fetchall()}

            # By sentiment
            cursor.execute("""
                SELECT sentiment_label, COUNT(*) as count
                FROM news_articles WHERE sentiment_label IS NOT NULL
                GROUP BY sentiment_label
            """)
            by_sentiment = {row['sentiment_label']: row['count'] for row in cursor.fetchall()}

            # Today's articles
            cursor.execute("""
                SELECT COUNT(*) FROM news_articles
                WHERE DATE(fetched_at) = DATE('now')
            """)
            today = cursor.fetchone()[0]

            # Drop related
            cursor.execute("""
                SELECT COUNT(*) FROM news_articles
                WHERE is_drop_related = 1 OR is_restock_related = 1
            """)
            drop_related = cursor.fetchone()[0]

            return {
                'total': total,
                'today': today,
                'by_source': by_source,
                'by_sentiment': by_sentiment,
                'drop_related': drop_related
            }

    def update_news_correlation(self, article_id: int, related_product_ids: List[int],
                                 is_drop_related: bool, is_restock_related: bool):
        """Update news article with correlation data"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                UPDATE news_articles
                SET related_product_ids = ?, is_drop_related = ?, is_restock_related = ?
                WHERE id = ?
            """, (json.dumps(related_product_ids), is_drop_related, is_restock_related, article_id))

    def get_correlated_news(self, hours: int = 24) -> List[Dict]:
        """Get news articles correlated with recent drops"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                SELECT * FROM news_articles
                WHERE (is_drop_related = 1 OR is_restock_related = 1)
                AND fetched_at >= datetime('now', ?)
                ORDER BY published_at DESC
            """, (f"-{hours} hours",))
            articles = []
            for row in cursor.fetchall():
                article = dict(row)
                if article.get('keywords'):
                    article['keywords'] = json.loads(article['keywords'])
                if article.get('related_product_ids'):
                    article['related_product_ids'] = json.loads(article['related_product_ids'])
                articles.append(article)
            return articles

    def delete_old_news(self, days: int = 30):
        """Delete news articles older than specified days"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                DELETE FROM news_articles
                WHERE fetched_at < datetime('now', ?)
            """, (f"-{days} days",))
            deleted = cursor.rowcount
            logger.info(f"Deleted {deleted} old news articles")
            return deleted

    # ==================== Migration Methods ====================

    def migrate_from_json(self, json_path: str):
        """Migrate existing products.json to database"""
        try:
            with open(json_path, 'r') as f:
                products = json.load(f)
        except (FileNotFoundError, json.JSONDecodeError):
            logger.warning(f"Could not load {json_path} for migration")
            return

        migrated = 0
        for url, data in products.items():
            product_id = self.get_or_create_product(
                url=url,
                name=data.get('name', 'Unknown'),
                site=data.get('site', 'unknown'),
                product_id=data.get('product_id'),
                image_url=data.get('image_url'),
                price=data.get('price'),
                in_stock=data.get('in_stock', False)
            )

            # Record initial price if available
            if data.get('price'):
                self.record_price(product_id, data['price'])

            # Record as existing product (not new_drop since it's historical)
            if data.get('in_stock'):
                self.record_stock_event(product_id, 'in_stock')

            migrated += 1

        logger.info(f"Migrated {migrated} products from {json_path}")

    def deduplicate_products(self) -> Dict:
        """
        Find and remove duplicate products based on normalized URL or product_id.
        Keeps the product with the most recent last_seen date.
        Returns stats about what was deduplicated.
        """
        stats = {'duplicates_found': 0, 'products_removed': 0, 'by_site': {}}

        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Find duplicates by product_id (same site)
            cursor.execute("""
                SELECT site, product_id, COUNT(*) as count, GROUP_CONCAT(id) as ids
                FROM products
                WHERE product_id IS NOT NULL AND product_id != ''
                GROUP BY site, product_id
                HAVING count > 1
            """)
            product_id_dupes = cursor.fetchall()

            for row in product_id_dupes:
                site = row['site']
                ids = [int(i) for i in row['ids'].split(',')]
                stats['duplicates_found'] += 1

                # Keep the one with the most recent last_seen
                cursor.execute("""
                    SELECT id FROM products
                    WHERE id IN ({})
                    ORDER BY last_seen DESC, first_seen ASC
                    LIMIT 1
                """.format(','.join('?' * len(ids))), ids)
                keep_id = cursor.fetchone()['id']

                # Delete the others
                delete_ids = [i for i in ids if i != keep_id]
                if delete_ids:
                    cursor.execute("""
                        DELETE FROM products WHERE id IN ({})
                    """.format(','.join('?' * len(delete_ids))), delete_ids)
                    stats['products_removed'] += len(delete_ids)
                    stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)

            # Find duplicates by similar name (same site, fuzzy match)
            cursor.execute("""
                SELECT site, LOWER(TRIM(name)) as clean_name, COUNT(*) as count, GROUP_CONCAT(id) as ids
                FROM products
                GROUP BY site, clean_name
                HAVING count > 1
            """)
            name_dupes = cursor.fetchall()

            for row in name_dupes:
                site = row['site']
                ids = [int(i) for i in row['ids'].split(',')]
                stats['duplicates_found'] += 1

                # Keep the one with the most recent last_seen
                cursor.execute("""
                    SELECT id FROM products
                    WHERE id IN ({})
                    ORDER BY last_seen DESC, first_seen ASC
                    LIMIT 1
                """.format(','.join('?' * len(ids))), ids)
                keep_id = cursor.fetchone()['id']

                # Delete the others
                delete_ids = [i for i in ids if i != keep_id]
                if delete_ids:
                    cursor.execute("""
                        DELETE FROM products WHERE id IN ({})
                    """.format(','.join('?' * len(delete_ids))), delete_ids)
                    stats['products_removed'] += len(delete_ids)
                    stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)

        logger.info(f"Deduplication complete: {stats}")
        return stats

    def deduplicate_events(self) -> Dict:
        """
        Remove duplicate stock events (same product, same event type within 1 hour).
        Keeps only the earliest event in each duplicate group.
        Returns stats about what was removed.
        """
        stats = {'duplicate_groups': 0, 'events_removed': 0}

        with self.get_connection() as conn:
            cursor = conn.cursor()

            # Find events that have duplicates within 1 hour
            # We'll keep the earliest one in each group
            cursor.execute("""
                DELETE FROM stock_events
                WHERE id NOT IN (
                    SELECT MIN(id)
                    FROM stock_events
                    GROUP BY product_id, event_type,
                             strftime('%Y-%m-%d %H', recorded_at)
                )
            """)
            stats['events_removed'] = cursor.rowcount

        logger.info(f"Event deduplication complete: removed {stats['events_removed']} duplicate events")
        return stats

    # ==================== User Methods ====================

    def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,
                    discord_webhook: str = None, notify_enabled: bool = True) -> int:
        """Create a new user. Returns user ID."""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("""
                INSERT INTO users (name, zip_code, radius_miles, discord_webhook, notify_enabled)
                VALUES (?, ?, ?, ?, ?)
            """, (name, zip_code, radius_miles, discord_webhook, notify_enabled))
            logger.info(f"Created user: {name}")
            return cursor.lastrowid

    def get_user(self, user_id: int) -> Optional[Dict]:
        """Get user by ID"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,))
            row = cursor.fetchone()
            return dict(row) if row else None

    def get_user_by_name(self, name: str) -> Optional[Dict]:
        """Get user by name"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM users WHERE name = ?", (name,))
            row = cursor.fetchone()
            return dict(row) if row else None

    def get_all_users(self) -> List[Dict]:
        """Get all users"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM users ORDER BY name")
            return [dict(row) for row in cursor.fetchall()]

    def update_user(self, user_id: int, **kwargs) -> bool:
        """Update user settings"""
        allowed_fields = ['name', 'zip_code', 'radius_miles', 'discord_webhook', 'notify_enabled']
        updates = {k: v for k, v in kwargs.items() if k in allowed_fields}

        if not updates:
            return False

        with self.get_connection() as conn:
            cursor = conn.cursor()
            set_clause = ", ".join(f"{k} = ?" for k in updates.keys())
            cursor.execute(
                f"UPDATE users SET {set_clause}, last_active = CURRENT_TIMESTAMP WHERE id = ?",
                list(updates.values()) + [user_id]
            )
            logger.info(f"Updated user {user_id}: {updates}")
            return cursor.rowcount > 0

    def delete_user(self, user_id: int) -> bool:
        """Delete a user"""
        with self.get_connection() as conn:
            cursor = conn.cursor()
            cursor.execute("DELETE FROM users WHERE id = ?", (user_id,))
            return cursor.rowcount > 0

    def update_user_location(self, user_id: int, zip_code: str, radius_miles: int = 25) -> bool:
        """Update user's location settings"""
        return self.update_user(user_id, zip_code=zip_code, radius_miles=radius_miles)

    # ==================== Checkout Profile Methods ====================

    def save_user_profile(self, user_id: int, salt: bytes, ciphertext: bytes):
        """Save or replace an encrypted checkout profile for a user."""
        with self.get_connection() as conn:
            conn.execute(
                """INSERT INTO user_profiles (user_id, salt, ciphertext, updated_at)
                   VALUES (?, ?, ?, CURRENT_TIMESTAMP)
                   ON CONFLICT(user_id) DO UPDATE SET
                     salt=excluded.salt,
                     ciphertext=excluded.ciphertext,
                     updated_at=CURRENT_TIMESTAMP""",
                (user_id, salt, ciphertext),
            )

    def get_user_profile(self, user_id: int) -> Optional[Dict]:
        """Return {salt, ciphertext} for a user, or None if no profile saved."""
        with self.get_connection() as conn:
            conn.row_factory = sqlite3.Row
            row = conn.execute(
                "SELECT salt, ciphertext, updated_at FROM user_profiles WHERE user_id = ?",
                (user_id,),
            ).fetchone()
            if not row:
                return None
            return {"salt": row["salt"], "ciphertext": row["ciphertext"], "updated_at": row["updated_at"]}

    def delete_user_profile(self, user_id: int) -> bool:
        """Delete a user's checkout profile."""
        with self.get_connection() as conn:
            cursor = conn.execute(
                "DELETE FROM user_profiles WHERE user_id = ?", (user_id,)
            )
            return cursor.rowcount > 0

    def user_has_profile(self, user_id: int) -> bool:
        """Return True if a saved (encrypted) profile exists for this user."""
        with self.get_connection() as conn:
            row = conn.execute(
                "SELECT 1 FROM user_profiles WHERE user_id = ?", (user_id,)
            ).fetchone()
            return row is not None

    # ==================== Stats Summary ====================

    def get_dashboard_stats(self) -> Dict:
        """Get summary stats for dashboard"""
        return {
            'total_products': self.get_product_count(),
            'in_stock_count': self.get_product_count(in_stock=True),
            'new_drops_today': self.get_events_today('new_drop'),
            'new_drops_week': self.get_events_this_week('new_drop'),
            'restocks_today': self.get_events_today('restock'),
            'restocks_week': self.get_events_this_week('restock'),
            'last_check': self.get_last_check(),
            'sites': self.get_site_stats(),
            'favorites_count': len(self.get_favorites())
        }


# Global database instance
_db: Database = None


def get_database() -> Database:
    """Get the global database instance"""
    global _db
    if _db is None:
        _db = Database()
    return _db