Add deduplication and normalize names/URLs

Add product name cleanup and URL normalization across backend, scrapers, and frontend; introduce product/event deduplication. Key changes:
- Clean up SVG/CSS garbage, trailing "X Reviews", mojibake and extra spaces in product names (api, scrapers, JS).
- Normalize URLs (lowercase host, strip tracking/query params, trim trailing slashes) in scrapers and Database and use normalized URL for comparisons/storage.
- Database: use normalized URLs when creating/getting products, prevent duplicate stock events within 1 hour, adjust recent events query to avoid duplicate products.
- Implement deduplication routines in Database: deduplicate_products (by product_id and normalized/clean name) and deduplicate_events (remove duplicate events within same hour).
- API: add POST endpoints /products/deduplicate and /events/deduplicate to trigger deduplication and return stats.
- Frontend: add UI buttons and handlers to call deduplication endpoints and display results.

These changes reduce duplicate product records/events caused by minor URL/name variations and stray CSS/svg artifacts.

dashboard/api.py

@@ -18,10 +18,16 @@ api_bp = Blueprint('api', __name__)
 
 
 def clean_product_name(name: str) -> str:
-    """Fix common encoding issues in product names"""
+    """Fix common encoding issues and garbage in product names"""
     if not name:
         return name
 
+    # Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    name = re.sub(r'[\w-]*_svg__[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\.review-full-[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\{[^}]*\}', '', name)  # Remove any remaining {css} blocks
+    name = re.sub(r'\d+\s*Reviews?$', '', name, flags=re.IGNORECASE)  # Remove trailing "X Reviews"
+
     # Fix Pokémon encoding issues
     name = re.sub(r'Pok[éÃ\u00c3\u00a9]+mon', 'Pokémon', name, flags=re.IGNORECASE)
     name = name.replace('Pokémon', 'Pokémon')
@@ -48,6 +54,9 @@ def clean_product_name(name: str) -> str:
     name = name.replace('®', '®')
     name = name.replace('Â', '')
 
+    # Clean up multiple spaces
+    name = re.sub(r'\s+', ' ', name)
+
     return name.strip()
 
 
@@ -908,6 +917,40 @@ def cleanup_broken_pokemoncenter_urls():
     })
 
 
+@api_bp.route('/products/deduplicate', methods=['POST'])
+def deduplicate_products():
+    """
+    Remove duplicate products from the database.
+    Duplicates are identified by:
+    - Same product_id and site
+    - Same name (case-insensitive) and site
+    """
+    db = get_database()
+    stats = db.deduplicate_products()
+
+    return jsonify({
+        'success': True,
+        'duplicates_found': stats['duplicates_found'],
+        'products_removed': stats['products_removed'],
+        'by_site': stats['by_site']
+    })
+
+
+@api_bp.route('/events/deduplicate', methods=['POST'])
+def deduplicate_events():
+    """
+    Remove duplicate events from the database.
+    Duplicates are events for the same product/type within the same hour.
+    """
+    db = get_database()
+    stats = db.deduplicate_events()
+
+    return jsonify({
+        'success': True,
+        'events_removed': stats['events_removed']
+    })
+
+
 @api_bp.route('/extension/clear', methods=['POST'])
 def clear_extension_data():
     """Clear all extension data"""

dashboard/static/app.js

@@ -674,6 +674,11 @@ function cleanProductName(name) {
 
   // Fix common UTF-8 encoding issues (mojibake)
   return name
+    // Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    .replace(/[\w-]*_svg__[\w\s{}:#.;-]+/gi, '')
+    .replace(/\.review-full-[\w\s{}:#.;-]+/gi, '')
+    .replace(/\{[^}]*\}/g, '')  // Remove any remaining {css} blocks
+    .replace(/\d+\s*Reviews?$/i, '')  // Remove trailing "X Reviews"
     // Fix Pokémon - multiple encoding patterns
     .replace(/Pok[éÃ\u00c3\u00a9]+mon/gi, 'Pokémon')
     .replace(/Pokémon/gi, 'Pokémon')
@@ -1278,6 +1283,8 @@ async function loadSettings() {
   if (!settingsInitialized) {
     document.getElementById('checkBrokenUrlsBtn')?.addEventListener('click', checkBrokenUrls);
     document.getElementById('cleanupBrokenUrlsBtn')?.addEventListener('click', cleanupBrokenUrls);
+    document.getElementById('deduplicateBtn')?.addEventListener('click', deduplicateProducts);
+    document.getElementById('deduplicateEventsBtn')?.addEventListener('click', deduplicateEvents);
     settingsInitialized = true;
   }
 }
@@ -1309,3 +1316,37 @@ async function cleanupBrokenUrls() {
     resultEl.textContent = 'Failed to cleanup broken URLs';
   }
 }
+
+async function deduplicateProducts() {
+  if (!confirm('This will remove duplicate products. Continue?')) return;
+
+  const resultEl = document.getElementById('deduplicateResult');
+  resultEl.textContent = 'Removing duplicates...';
+
+  const result = await api('/products/deduplicate', { method: 'POST' });
+
+  if (result && result.success) {
+    let message = `Found ${result.duplicates_found} duplicate groups, removed ${result.products_removed} products.`;
+    if (Object.keys(result.by_site).length > 0) {
+      message += ' By site: ' + Object.entries(result.by_site).map(([site, count]) => `${site}: ${count}`).join(', ');
+    }
+    resultEl.textContent = message;
+  } else {
+    resultEl.textContent = 'Failed to deduplicate products';
+  }
+}
+
+async function deduplicateEvents() {
+  if (!confirm('This will remove duplicate events from the activity feed. Continue?')) return;
+
+  const resultEl = document.getElementById('deduplicateEventsResult');
+  resultEl.textContent = 'Cleaning up events...';
+
+  const result = await api('/events/deduplicate', { method: 'POST' });
+
+  if (result && result.success) {
+    resultEl.textContent = `Removed ${result.events_removed} duplicate events.`;
+  } else {
+    resultEl.textContent = 'Failed to clean up events';
+  }
+}

dashboard/templates/index.html

@@ -356,6 +356,16 @@
           <button id="checkBrokenUrlsBtn" class="btn-secondary">Check Broken URLs</button>
           <button id="cleanupBrokenUrlsBtn" class="btn-danger">Cleanup Broken URLs</button>
           <p id="brokenUrlsResult" class="help-text"></p>
+
+          <h3>Duplicate Products</h3>
+          <p class="help-text">Remove duplicate products from Target, GameStop, and other stores</p>
+          <button id="deduplicateBtn" class="btn-secondary">Remove Duplicates</button>
+          <p id="deduplicateResult" class="help-text"></p>
+
+          <h3>Duplicate Events</h3>
+          <p class="help-text">Remove duplicate events from the activity feed (same product/type within 1 hour)</p>
+          <button id="deduplicateEventsBtn" class="btn-secondary">Clean Up Events</button>
+          <p id="deduplicateEventsResult" class="help-text"></p>
         </div>
       </section>
     </main>

scrapers/base.py

@@ -3,13 +3,79 @@ Base scraper class with common functionality
 """
 
 import logging
+import re
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import List, Optional
+from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
 
 logger = logging.getLogger(__name__)
 
 
+def clean_product_name(name: str) -> str:
+    """Clean garbage from product names (SVG classes, CSS, etc.)"""
+    if not name:
+        return name
+
+    import re
+
+    # Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    name = re.sub(r'[\w-]*_svg__[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\.review-full-[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\{[^}]*\}', '', name)  # Remove any remaining {css} blocks
+    name = re.sub(r'\d+\s*Reviews?$', '', name, flags=re.IGNORECASE)  # Remove trailing "X Reviews"
+
+    # Clean up multiple spaces
+    name = re.sub(r'\s+', ' ', name)
+
+    return name.strip()
+
+
+def normalize_url(url: str) -> str:
+    """
+    Normalize a URL to prevent duplicates from minor variations.
+    - Removes tracking/session query parameters
+    - Removes trailing slashes
+    - Lowercases the domain
+    - Keeps only essential path components
+    """
+    if not url:
+        return url
+
+    try:
+        parsed = urlparse(url)
+
+        # Lowercase the domain
+        netloc = parsed.netloc.lower()
+
+        # Remove trailing slash from path
+        path = parsed.path.rstrip('/')
+
+        # Filter out common tracking/non-essential query params
+        query_params = parse_qs(parsed.query)
+        exclude_params = {
+            'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
+            'utm_term', 'gclid', 'fbclid', 'msclkid', 'dclid', 'zanpid',
+            'src', 'srsltid', 'skuId', 'preselect', 'intl', 'lnk',
+            'Nao', 'start', 'page', 'sort', 'sortBy', 'facets',
+        }
+        filtered_params = {
+            k: v for k, v in query_params.items()
+            if k.lower() not in {p.lower() for p in exclude_params}
+        }
+
+        # Rebuild query string (sorted for consistency)
+        query = urlencode(sorted(filtered_params.items()), doseq=True) if filtered_params else ''
+
+        # Rebuild URL without fragment
+        normalized = urlunparse((parsed.scheme, netloc, path, '', query, ''))
+        return normalized
+
+    except Exception as e:
+        logger.debug(f"Error normalizing URL {url}: {e}")
+        return url
+
+
 @dataclass
 class Product:
     """Represents a product listing"""
@@ -20,13 +86,31 @@ class Product:
     image_url: Optional[str] = None
     site: str = ""
     product_id: Optional[str] = None  # Unique identifier for tracking
+    _normalized_url: str = field(default="", repr=False, compare=False)
+
+    def __post_init__(self):
+        self._normalized_url = normalize_url(self.url)
+        # Clean the name of any SVG/CSS garbage
+        if self.name:
+            self.name = clean_product_name(self.name)
+
+    @property
+    def normalized_url(self) -> str:
+        """Get the normalized URL for deduplication"""
+        if not self._normalized_url:
+            self._normalized_url = normalize_url(self.url)
+        return self._normalized_url
 
     def __hash__(self):
-        return hash(self.url)
+        # Use normalized URL for hashing to prevent duplicates
+        return hash(self.normalized_url)
 
     def __eq__(self, other):
         if isinstance(other, Product):
-            return self.url == other.url
+            # Compare by normalized URL OR by product_id if both have one
+            if self.product_id and other.product_id and self.site == other.site:
+                return self.product_id == other.product_id
+            return self.normalized_url == other.normalized_url
         return False
 
 

src/database.py

@@ -166,19 +166,47 @@ class Database:
 
     # ==================== Product Methods ====================
 
+    def _normalize_url(self, url: str) -> str:
+        """Normalize URL to prevent duplicates from minor variations."""
+        if not url:
+            return url
+        try:
+            from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
+            parsed = urlparse(url)
+            netloc = parsed.netloc.lower()
+            path = parsed.path.rstrip('/')
+            # Filter out tracking/pagination params
+            query_params = parse_qs(parsed.query)
+            exclude_params = {
+                'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
+                'utm_term', 'gclid', 'fbclid', 'msclkid', 'srsltid', 'skuId',
+                'preselect', 'intl', 'lnk', 'nao', 'start', 'page', 'sort', 'sortby',
+            }
+            filtered = {k: v for k, v in query_params.items() if k.lower() not in exclude_params}
+            query = urlencode(sorted(filtered.items()), doseq=True) if filtered else ''
+            return urlunparse((parsed.scheme, netloc, path, '', query, ''))
+        except Exception:
+            return url
+
     def get_or_create_product(self, url: str, name: str, site: str,
                                product_id: str = None, image_url: str = None,
                                price: str = None, in_stock: bool = False) -> int:
         """Get existing product or create new one. Returns product ID."""
         category = self._detect_category(name)
+        normalized_url = self._normalize_url(url)
 
         with self.get_connection() as conn:
             cursor = conn.cursor()
 
-            # Try to get existing product
-            cursor.execute("SELECT id FROM products WHERE url = ?", (url,))
+            # Try to get existing product by URL (check both original and normalized)
+            cursor.execute("SELECT id FROM products WHERE url = ? OR url = ?", (url, normalized_url))
             row = cursor.fetchone()
 
+            # Also check by product_id if provided (more reliable for some sites)
+            if not row and product_id and site:
+                cursor.execute("SELECT id FROM products WHERE product_id = ? AND site = ?", (product_id, site))
+                row = cursor.fetchone()
+
             if row:
                 # Update existing product
                 cursor.execute("""
@@ -190,12 +218,12 @@ class Database:
                 """, (name, price, in_stock, image_url, category, row['id']))
                 return row['id']
             else:
-                # Create new product
+                # Create new product with normalized URL
                 cursor.execute("""
                     INSERT INTO products (url, name, site, product_id, image_url,
                                          current_price, in_stock, last_seen, category)
                     VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
-                """, (url, name, site, product_id, image_url, price, in_stock, category))
+                """, (normalized_url, name, site, product_id, image_url, price, in_stock, category))
                 return cursor.lastrowid
 
     def get_product(self, product_id: int) -> Optional[Dict]:
@@ -207,10 +235,11 @@ class Database:
             return dict(row) if row else None
 
     def get_product_by_url(self, url: str) -> Optional[Dict]:
-        """Get product by URL"""
+        """Get product by URL (checks both original and normalized)"""
+        normalized_url = self._normalize_url(url)
         with self.get_connection() as conn:
             cursor = conn.cursor()
-            cursor.execute("SELECT * FROM products WHERE url = ?", (url,))
+            cursor.execute("SELECT * FROM products WHERE url = ? OR url = ?", (url, normalized_url))
             row = cursor.fetchone()
             return dict(row) if row else None
 
@@ -369,9 +398,23 @@ class Database:
     # ==================== Stock Event Methods ====================
 
     def record_stock_event(self, product_id: int, event_type: str):
-        """Record a stock event (new_drop, restock, in_stock, out_of_stock)"""
+        """Record a stock event (new_drop, restock, in_stock, out_of_stock).
+        Prevents duplicate events within 1 hour for the same product/event_type."""
         with self.get_connection() as conn:
             cursor = conn.cursor()
+
+            # Check if same event was recorded recently (within 1 hour)
+            cursor.execute("""
+                SELECT id FROM stock_events
+                WHERE product_id = ? AND event_type = ?
+                AND recorded_at > datetime('now', '-1 hour')
+                LIMIT 1
+            """, (product_id, event_type))
+
+            if cursor.fetchone():
+                logger.debug(f"Skipping duplicate {event_type} event for product {product_id}")
+                return
+
             cursor.execute("""
                 INSERT INTO stock_events (product_id, event_type)
                 VALUES (?, ?)
@@ -379,23 +422,42 @@ class Database:
             logger.debug(f"Recorded {event_type} event for product {product_id}")
 
     def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
-        """Get recent stock events with product info"""
+        """Get recent stock events with product info.
+        Deduplicates by product name to avoid showing same product multiple times."""
         with self.get_connection() as conn:
             cursor = conn.cursor()
 
-            query = """
-                SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
-                FROM stock_events e
-                JOIN products p ON e.product_id = p.id
-            """
+            # Use a subquery to get only the most recent event per product
+            # This prevents the same product from appearing multiple times
             params = []
 
             if event_types:
                 placeholders = ",".join("?" * len(event_types))
-                query += f" WHERE e.event_type IN ({placeholders})"
-                params.extend(event_types)
+                event_filter = f"e.event_type IN ({placeholders})"
+                event_filter2 = f"e2.event_type IN ({placeholders})"
+                params = list(event_types)
+            else:
+                event_filter = "1=1"
+                event_filter2 = "1=1"
 
-            query += " ORDER BY e.recorded_at DESC LIMIT ?"
+            query = f"""
+                SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
+                FROM stock_events e
+                JOIN products p ON e.product_id = p.id
+                WHERE {event_filter}
+                AND e.id IN (
+                    SELECT MAX(e2.id)
+                    FROM stock_events e2
+                    JOIN products p2 ON e2.product_id = p2.id
+                    WHERE {event_filter2}
+                    GROUP BY LOWER(TRIM(p2.name))
+                )
+                ORDER BY e.recorded_at DESC
+                LIMIT ?
+            """
+            # Add params for subquery event filter if needed
+            if event_types:
+                params.extend(event_types)
             params.append(limit)
 
             cursor.execute(query, params)
@@ -834,6 +896,112 @@ class Database:
 
         logger.info(f"Migrated {migrated} products from {json_path}")
 
+    def deduplicate_products(self) -> Dict:
+        """
+        Find and remove duplicate products based on normalized URL or product_id.
+        Keeps the product with the most recent last_seen date.
+        Returns stats about what was deduplicated.
+        """
+        stats = {'duplicates_found': 0, 'products_removed': 0, 'by_site': {}}
+
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Find duplicates by product_id (same site)
+            cursor.execute("""
+                SELECT site, product_id, COUNT(*) as count, GROUP_CONCAT(id) as ids
+                FROM products
+                WHERE product_id IS NOT NULL AND product_id != ''
+                GROUP BY site, product_id
+                HAVING count > 1
+            """)
+            product_id_dupes = cursor.fetchall()
+
+            for row in product_id_dupes:
+                site = row['site']
+                ids = [int(i) for i in row['ids'].split(',')]
+                stats['duplicates_found'] += 1
+
+                # Keep the one with the most recent last_seen
+                cursor.execute("""
+                    SELECT id FROM products
+                    WHERE id IN ({})
+                    ORDER BY last_seen DESC, first_seen ASC
+                    LIMIT 1
+                """.format(','.join('?' * len(ids))), ids)
+                keep_id = cursor.fetchone()['id']
+
+                # Delete the others
+                delete_ids = [i for i in ids if i != keep_id]
+                if delete_ids:
+                    cursor.execute("""
+                        DELETE FROM products WHERE id IN ({})
+                    """.format(','.join('?' * len(delete_ids))), delete_ids)
+                    stats['products_removed'] += len(delete_ids)
+                    stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
+
+            # Find duplicates by similar name (same site, fuzzy match)
+            cursor.execute("""
+                SELECT site, LOWER(TRIM(name)) as clean_name, COUNT(*) as count, GROUP_CONCAT(id) as ids
+                FROM products
+                GROUP BY site, clean_name
+                HAVING count > 1
+            """)
+            name_dupes = cursor.fetchall()
+
+            for row in name_dupes:
+                site = row['site']
+                ids = [int(i) for i in row['ids'].split(',')]
+                stats['duplicates_found'] += 1
+
+                # Keep the one with the most recent last_seen
+                cursor.execute("""
+                    SELECT id FROM products
+                    WHERE id IN ({})
+                    ORDER BY last_seen DESC, first_seen ASC
+                    LIMIT 1
+                """.format(','.join('?' * len(ids))), ids)
+                keep_id = cursor.fetchone()['id']
+
+                # Delete the others
+                delete_ids = [i for i in ids if i != keep_id]
+                if delete_ids:
+                    cursor.execute("""
+                        DELETE FROM products WHERE id IN ({})
+                    """.format(','.join('?' * len(delete_ids))), delete_ids)
+                    stats['products_removed'] += len(delete_ids)
+                    stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
+
+        logger.info(f"Deduplication complete: {stats}")
+        return stats
+
+    def deduplicate_events(self) -> Dict:
+        """
+        Remove duplicate stock events (same product, same event type within 1 hour).
+        Keeps only the earliest event in each duplicate group.
+        Returns stats about what was removed.
+        """
+        stats = {'duplicate_groups': 0, 'events_removed': 0}
+
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Find events that have duplicates within 1 hour
+            # We'll keep the earliest one in each group
+            cursor.execute("""
+                DELETE FROM stock_events
+                WHERE id NOT IN (
+                    SELECT MIN(id)
+                    FROM stock_events
+                    GROUP BY product_id, event_type,
+                             strftime('%Y-%m-%d %H', recorded_at)
+                )
+            """)
+            stats['events_removed'] = cursor.rowcount
+
+        logger.info(f"Event deduplication complete: removed {stats['events_removed']} duplicate events")
+        return stats
+
     # ==================== User Methods ====================
 
     def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,