Add deduplication and normalize names/URLs

Add product name cleanup and URL normalization across backend, scrapers, and frontend; introduce product/event deduplication. Key changes:
- Clean up SVG/CSS garbage, trailing "X Reviews", mojibake and extra spaces in product names (api, scrapers, JS).
- Normalize URLs (lowercase host, strip tracking/query params, trim trailing slashes) in scrapers and Database and use normalized URL for comparisons/storage.
- Database: use normalized URLs when creating/getting products, prevent duplicate stock events within 1 hour, adjust recent events query to avoid duplicate products.
- Implement deduplication routines in Database: deduplicate_products (by product_id and normalized/clean name) and deduplicate_events (remove duplicate events within same hour).
- API: add POST endpoints /products/deduplicate and /events/deduplicate to trigger deduplication and return stats.
- Frontend: add UI buttons and handlers to call deduplication endpoints and display results.

These changes reduce duplicate product records/events caused by minor URL/name variations and stray CSS/svg artifacts.

scrapers/base.py

@@ -3,13 +3,79 @@ Base scraper class with common functionality
 """
 
 import logging
+import re
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import List, Optional
+from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
 
 logger = logging.getLogger(__name__)
 
 
+def clean_product_name(name: str) -> str:
+    """Clean garbage from product names (SVG classes, CSS, etc.)"""
+    if not name:
+        return name
+
+    import re
+
+    # Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    name = re.sub(r'[\w-]*_svg__[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\.review-full-[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\{[^}]*\}', '', name)  # Remove any remaining {css} blocks
+    name = re.sub(r'\d+\s*Reviews?$', '', name, flags=re.IGNORECASE)  # Remove trailing "X Reviews"
+
+    # Clean up multiple spaces
+    name = re.sub(r'\s+', ' ', name)
+
+    return name.strip()
+
+
+def normalize_url(url: str) -> str:
+    """
+    Normalize a URL to prevent duplicates from minor variations.
+    - Removes tracking/session query parameters
+    - Removes trailing slashes
+    - Lowercases the domain
+    - Keeps only essential path components
+    """
+    if not url:
+        return url
+
+    try:
+        parsed = urlparse(url)
+
+        # Lowercase the domain
+        netloc = parsed.netloc.lower()
+
+        # Remove trailing slash from path
+        path = parsed.path.rstrip('/')
+
+        # Filter out common tracking/non-essential query params
+        query_params = parse_qs(parsed.query)
+        exclude_params = {
+            'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
+            'utm_term', 'gclid', 'fbclid', 'msclkid', 'dclid', 'zanpid',
+            'src', 'srsltid', 'skuId', 'preselect', 'intl', 'lnk',
+            'Nao', 'start', 'page', 'sort', 'sortBy', 'facets',
+        }
+        filtered_params = {
+            k: v for k, v in query_params.items()
+            if k.lower() not in {p.lower() for p in exclude_params}
+        }
+
+        # Rebuild query string (sorted for consistency)
+        query = urlencode(sorted(filtered_params.items()), doseq=True) if filtered_params else ''
+
+        # Rebuild URL without fragment
+        normalized = urlunparse((parsed.scheme, netloc, path, '', query, ''))
+        return normalized
+
+    except Exception as e:
+        logger.debug(f"Error normalizing URL {url}: {e}")
+        return url
+
+
 @dataclass
 class Product:
     """Represents a product listing"""
@@ -20,13 +86,31 @@ class Product:
     image_url: Optional[str] = None
     site: str = ""
     product_id: Optional[str] = None  # Unique identifier for tracking
+    _normalized_url: str = field(default="", repr=False, compare=False)
+
+    def __post_init__(self):
+        self._normalized_url = normalize_url(self.url)
+        # Clean the name of any SVG/CSS garbage
+        if self.name:
+            self.name = clean_product_name(self.name)
+
+    @property
+    def normalized_url(self) -> str:
+        """Get the normalized URL for deduplication"""
+        if not self._normalized_url:
+            self._normalized_url = normalize_url(self.url)
+        return self._normalized_url
 
     def __hash__(self):
-        return hash(self.url)
+        # Use normalized URL for hashing to prevent duplicates
+        return hash(self.normalized_url)
 
     def __eq__(self, other):
         if isinstance(other, Product):
-            return self.url == other.url
+            # Compare by normalized URL OR by product_id if both have one
+            if self.product_id and other.product_id and self.site == other.site:
+                return self.product_id == other.product_id
+            return self.normalized_url == other.normalized_url
         return False