Add deduplication and normalize names/URLs

Add product name cleanup and URL normalization across backend, scrapers, and frontend; introduce product/event deduplication. Key changes:
- Clean up SVG/CSS garbage, trailing "X Reviews", mojibake and extra spaces in product names (api, scrapers, JS).
- Normalize URLs (lowercase host, strip tracking/query params, trim trailing slashes) in scrapers and Database and use normalized URL for comparisons/storage.
- Database: use normalized URLs when creating/getting products, prevent duplicate stock events within 1 hour, adjust recent events query to avoid duplicate products.
- Implement deduplication routines in Database: deduplicate_products (by product_id and normalized/clean name) and deduplicate_events (remove duplicate events within same hour).
- API: add POST endpoints /products/deduplicate and /events/deduplicate to trigger deduplication and return stats.
- Frontend: add UI buttons and handlers to call deduplication endpoints and display results.

These changes reduce duplicate product records/events caused by minor URL/name variations and stray CSS/svg artifacts.

dashboard/api.py

@@ -18,10 +18,16 @@ api_bp = Blueprint('api', __name__)
 
 
 def clean_product_name(name: str) -> str:
-    """Fix common encoding issues in product names"""
+    """Fix common encoding issues and garbage in product names"""
     if not name:
         return name
 
+    # Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    name = re.sub(r'[\w-]*_svg__[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\.review-full-[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\{[^}]*\}', '', name)  # Remove any remaining {css} blocks
+    name = re.sub(r'\d+\s*Reviews?$', '', name, flags=re.IGNORECASE)  # Remove trailing "X Reviews"
+
     # Fix Pokémon encoding issues
     name = re.sub(r'Pok[éÃ\u00c3\u00a9]+mon', 'Pokémon', name, flags=re.IGNORECASE)
     name = name.replace('Pokémon', 'Pokémon')
@@ -48,6 +54,9 @@ def clean_product_name(name: str) -> str:
     name = name.replace('®', '®')
     name = name.replace('Â', '')
 
+    # Clean up multiple spaces
+    name = re.sub(r'\s+', ' ', name)
+
     return name.strip()
 
 
@@ -908,6 +917,40 @@ def cleanup_broken_pokemoncenter_urls():
     })
 
 
+@api_bp.route('/products/deduplicate', methods=['POST'])
+def deduplicate_products():
+    """
+    Remove duplicate products from the database.
+    Duplicates are identified by:
+    - Same product_id and site
+    - Same name (case-insensitive) and site
+    """
+    db = get_database()
+    stats = db.deduplicate_products()
+
+    return jsonify({
+        'success': True,
+        'duplicates_found': stats['duplicates_found'],
+        'products_removed': stats['products_removed'],
+        'by_site': stats['by_site']
+    })
+
+
+@api_bp.route('/events/deduplicate', methods=['POST'])
+def deduplicate_events():
+    """
+    Remove duplicate events from the database.
+    Duplicates are events for the same product/type within the same hour.
+    """
+    db = get_database()
+    stats = db.deduplicate_events()
+
+    return jsonify({
+        'success': True,
+        'events_removed': stats['events_removed']
+    })
+
+
 @api_bp.route('/extension/clear', methods=['POST'])
 def clear_extension_data():
     """Clear all extension data"""

dashboard/static/app.js

@@ -674,6 +674,11 @@ function cleanProductName(name) {
 
   // Fix common UTF-8 encoding issues (mojibake)
   return name
+    // Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...)
+    .replace(/[\w-]*_svg__[\w\s{}:#.;-]+/gi, '')
+    .replace(/\.review-full-[\w\s{}:#.;-]+/gi, '')
+    .replace(/\{[^}]*\}/g, '')  // Remove any remaining {css} blocks
+    .replace(/\d+\s*Reviews?$/i, '')  // Remove trailing "X Reviews"
     // Fix Pokémon - multiple encoding patterns
     .replace(/Pok[éÃ\u00c3\u00a9]+mon/gi, 'Pokémon')
     .replace(/Pokémon/gi, 'Pokémon')
@@ -1278,6 +1283,8 @@ async function loadSettings() {
   if (!settingsInitialized) {
     document.getElementById('checkBrokenUrlsBtn')?.addEventListener('click', checkBrokenUrls);
     document.getElementById('cleanupBrokenUrlsBtn')?.addEventListener('click', cleanupBrokenUrls);
+    document.getElementById('deduplicateBtn')?.addEventListener('click', deduplicateProducts);
+    document.getElementById('deduplicateEventsBtn')?.addEventListener('click', deduplicateEvents);
     settingsInitialized = true;
   }
 }
@@ -1309,3 +1316,37 @@ async function cleanupBrokenUrls() {
     resultEl.textContent = 'Failed to cleanup broken URLs';
   }
 }
+
+async function deduplicateProducts() {
+  if (!confirm('This will remove duplicate products. Continue?')) return;
+
+  const resultEl = document.getElementById('deduplicateResult');
+  resultEl.textContent = 'Removing duplicates...';
+
+  const result = await api('/products/deduplicate', { method: 'POST' });
+
+  if (result && result.success) {
+    let message = `Found ${result.duplicates_found} duplicate groups, removed ${result.products_removed} products.`;
+    if (Object.keys(result.by_site).length > 0) {
+      message += ' By site: ' + Object.entries(result.by_site).map(([site, count]) => `${site}: ${count}`).join(', ');
+    }
+    resultEl.textContent = message;
+  } else {
+    resultEl.textContent = 'Failed to deduplicate products';
+  }
+}
+
+async function deduplicateEvents() {
+  if (!confirm('This will remove duplicate events from the activity feed. Continue?')) return;
+
+  const resultEl = document.getElementById('deduplicateEventsResult');
+  resultEl.textContent = 'Cleaning up events...';
+
+  const result = await api('/events/deduplicate', { method: 'POST' });
+
+  if (result && result.success) {
+    resultEl.textContent = `Removed ${result.events_removed} duplicate events.`;
+  } else {
+    resultEl.textContent = 'Failed to clean up events';
+  }
+}

dashboard/templates/index.html

@@ -356,6 +356,16 @@
           <button id="checkBrokenUrlsBtn" class="btn-secondary">Check Broken URLs</button>
           <button id="cleanupBrokenUrlsBtn" class="btn-danger">Cleanup Broken URLs</button>
           <p id="brokenUrlsResult" class="help-text"></p>
+
+          <h3>Duplicate Products</h3>
+          <p class="help-text">Remove duplicate products from Target, GameStop, and other stores</p>
+          <button id="deduplicateBtn" class="btn-secondary">Remove Duplicates</button>
+          <p id="deduplicateResult" class="help-text"></p>
+
+          <h3>Duplicate Events</h3>
+          <p class="help-text">Remove duplicate events from the activity feed (same product/type within 1 hour)</p>
+          <button id="deduplicateEventsBtn" class="btn-secondary">Clean Up Events</button>
+          <p id="deduplicateEventsResult" class="help-text"></p>
         </div>
       </section>
     </main>