Add deduplication and normalize names/URLs
Add product name cleanup and URL normalization across backend, scrapers, and frontend; introduce product/event deduplication. Key changes: - Clean up SVG/CSS garbage, trailing "X Reviews", mojibake and extra spaces in product names (api, scrapers, JS). - Normalize URLs (lowercase host, strip tracking/query params, trim trailing slashes) in scrapers and Database and use normalized URL for comparisons/storage. - Database: use normalized URLs when creating/getting products, prevent duplicate stock events within 1 hour, adjust recent events query to avoid duplicate products. - Implement deduplication routines in Database: deduplicate_products (by product_id and normalized/clean name) and deduplicate_events (remove duplicate events within same hour). - API: add POST endpoints /products/deduplicate and /events/deduplicate to trigger deduplication and return stats. - Frontend: add UI buttons and handlers to call deduplication endpoints and display results. These changes reduce duplicate product records/events caused by minor URL/name variations and stray CSS/svg artifacts.
This commit is contained in:
+184
-16
@@ -166,19 +166,47 @@ class Database:
|
||||
|
||||
# ==================== Product Methods ====================
|
||||
|
||||
def _normalize_url(self, url: str) -> str:
|
||||
"""Normalize URL to prevent duplicates from minor variations."""
|
||||
if not url:
|
||||
return url
|
||||
try:
|
||||
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
||||
parsed = urlparse(url)
|
||||
netloc = parsed.netloc.lower()
|
||||
path = parsed.path.rstrip('/')
|
||||
# Filter out tracking/pagination params
|
||||
query_params = parse_qs(parsed.query)
|
||||
exclude_params = {
|
||||
'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
|
||||
'utm_term', 'gclid', 'fbclid', 'msclkid', 'srsltid', 'skuId',
|
||||
'preselect', 'intl', 'lnk', 'nao', 'start', 'page', 'sort', 'sortby',
|
||||
}
|
||||
filtered = {k: v for k, v in query_params.items() if k.lower() not in exclude_params}
|
||||
query = urlencode(sorted(filtered.items()), doseq=True) if filtered else ''
|
||||
return urlunparse((parsed.scheme, netloc, path, '', query, ''))
|
||||
except Exception:
|
||||
return url
|
||||
|
||||
def get_or_create_product(self, url: str, name: str, site: str,
|
||||
product_id: str = None, image_url: str = None,
|
||||
price: str = None, in_stock: bool = False) -> int:
|
||||
"""Get existing product or create new one. Returns product ID."""
|
||||
category = self._detect_category(name)
|
||||
normalized_url = self._normalize_url(url)
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Try to get existing product
|
||||
cursor.execute("SELECT id FROM products WHERE url = ?", (url,))
|
||||
# Try to get existing product by URL (check both original and normalized)
|
||||
cursor.execute("SELECT id FROM products WHERE url = ? OR url = ?", (url, normalized_url))
|
||||
row = cursor.fetchone()
|
||||
|
||||
# Also check by product_id if provided (more reliable for some sites)
|
||||
if not row and product_id and site:
|
||||
cursor.execute("SELECT id FROM products WHERE product_id = ? AND site = ?", (product_id, site))
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
# Update existing product
|
||||
cursor.execute("""
|
||||
@@ -190,12 +218,12 @@ class Database:
|
||||
""", (name, price, in_stock, image_url, category, row['id']))
|
||||
return row['id']
|
||||
else:
|
||||
# Create new product
|
||||
# Create new product with normalized URL
|
||||
cursor.execute("""
|
||||
INSERT INTO products (url, name, site, product_id, image_url,
|
||||
current_price, in_stock, last_seen, category)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)
|
||||
""", (url, name, site, product_id, image_url, price, in_stock, category))
|
||||
""", (normalized_url, name, site, product_id, image_url, price, in_stock, category))
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_product(self, product_id: int) -> Optional[Dict]:
|
||||
@@ -207,10 +235,11 @@ class Database:
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_product_by_url(self, url: str) -> Optional[Dict]:
|
||||
"""Get product by URL"""
|
||||
"""Get product by URL (checks both original and normalized)"""
|
||||
normalized_url = self._normalize_url(url)
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT * FROM products WHERE url = ?", (url,))
|
||||
cursor.execute("SELECT * FROM products WHERE url = ? OR url = ?", (url, normalized_url))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
@@ -369,9 +398,23 @@ class Database:
|
||||
# ==================== Stock Event Methods ====================
|
||||
|
||||
def record_stock_event(self, product_id: int, event_type: str):
|
||||
"""Record a stock event (new_drop, restock, in_stock, out_of_stock)"""
|
||||
"""Record a stock event (new_drop, restock, in_stock, out_of_stock).
|
||||
Prevents duplicate events within 1 hour for the same product/event_type."""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check if same event was recorded recently (within 1 hour)
|
||||
cursor.execute("""
|
||||
SELECT id FROM stock_events
|
||||
WHERE product_id = ? AND event_type = ?
|
||||
AND recorded_at > datetime('now', '-1 hour')
|
||||
LIMIT 1
|
||||
""", (product_id, event_type))
|
||||
|
||||
if cursor.fetchone():
|
||||
logger.debug(f"Skipping duplicate {event_type} event for product {product_id}")
|
||||
return
|
||||
|
||||
cursor.execute("""
|
||||
INSERT INTO stock_events (product_id, event_type)
|
||||
VALUES (?, ?)
|
||||
@@ -379,23 +422,42 @@ class Database:
|
||||
logger.debug(f"Recorded {event_type} event for product {product_id}")
|
||||
|
||||
def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]:
|
||||
"""Get recent stock events with product info"""
|
||||
"""Get recent stock events with product info.
|
||||
Deduplicates by product name to avoid showing same product multiple times."""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
query = """
|
||||
SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
|
||||
FROM stock_events e
|
||||
JOIN products p ON e.product_id = p.id
|
||||
"""
|
||||
# Use a subquery to get only the most recent event per product
|
||||
# This prevents the same product from appearing multiple times
|
||||
params = []
|
||||
|
||||
if event_types:
|
||||
placeholders = ",".join("?" * len(event_types))
|
||||
query += f" WHERE e.event_type IN ({placeholders})"
|
||||
params.extend(event_types)
|
||||
event_filter = f"e.event_type IN ({placeholders})"
|
||||
event_filter2 = f"e2.event_type IN ({placeholders})"
|
||||
params = list(event_types)
|
||||
else:
|
||||
event_filter = "1=1"
|
||||
event_filter2 = "1=1"
|
||||
|
||||
query += " ORDER BY e.recorded_at DESC LIMIT ?"
|
||||
query = f"""
|
||||
SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price
|
||||
FROM stock_events e
|
||||
JOIN products p ON e.product_id = p.id
|
||||
WHERE {event_filter}
|
||||
AND e.id IN (
|
||||
SELECT MAX(e2.id)
|
||||
FROM stock_events e2
|
||||
JOIN products p2 ON e2.product_id = p2.id
|
||||
WHERE {event_filter2}
|
||||
GROUP BY LOWER(TRIM(p2.name))
|
||||
)
|
||||
ORDER BY e.recorded_at DESC
|
||||
LIMIT ?
|
||||
"""
|
||||
# Add params for subquery event filter if needed
|
||||
if event_types:
|
||||
params.extend(event_types)
|
||||
params.append(limit)
|
||||
|
||||
cursor.execute(query, params)
|
||||
@@ -834,6 +896,112 @@ class Database:
|
||||
|
||||
logger.info(f"Migrated {migrated} products from {json_path}")
|
||||
|
||||
def deduplicate_products(self) -> Dict:
|
||||
"""
|
||||
Find and remove duplicate products based on normalized URL or product_id.
|
||||
Keeps the product with the most recent last_seen date.
|
||||
Returns stats about what was deduplicated.
|
||||
"""
|
||||
stats = {'duplicates_found': 0, 'products_removed': 0, 'by_site': {}}
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Find duplicates by product_id (same site)
|
||||
cursor.execute("""
|
||||
SELECT site, product_id, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
||||
FROM products
|
||||
WHERE product_id IS NOT NULL AND product_id != ''
|
||||
GROUP BY site, product_id
|
||||
HAVING count > 1
|
||||
""")
|
||||
product_id_dupes = cursor.fetchall()
|
||||
|
||||
for row in product_id_dupes:
|
||||
site = row['site']
|
||||
ids = [int(i) for i in row['ids'].split(',')]
|
||||
stats['duplicates_found'] += 1
|
||||
|
||||
# Keep the one with the most recent last_seen
|
||||
cursor.execute("""
|
||||
SELECT id FROM products
|
||||
WHERE id IN ({})
|
||||
ORDER BY last_seen DESC, first_seen ASC
|
||||
LIMIT 1
|
||||
""".format(','.join('?' * len(ids))), ids)
|
||||
keep_id = cursor.fetchone()['id']
|
||||
|
||||
# Delete the others
|
||||
delete_ids = [i for i in ids if i != keep_id]
|
||||
if delete_ids:
|
||||
cursor.execute("""
|
||||
DELETE FROM products WHERE id IN ({})
|
||||
""".format(','.join('?' * len(delete_ids))), delete_ids)
|
||||
stats['products_removed'] += len(delete_ids)
|
||||
stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
|
||||
|
||||
# Find duplicates by similar name (same site, fuzzy match)
|
||||
cursor.execute("""
|
||||
SELECT site, LOWER(TRIM(name)) as clean_name, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
||||
FROM products
|
||||
GROUP BY site, clean_name
|
||||
HAVING count > 1
|
||||
""")
|
||||
name_dupes = cursor.fetchall()
|
||||
|
||||
for row in name_dupes:
|
||||
site = row['site']
|
||||
ids = [int(i) for i in row['ids'].split(',')]
|
||||
stats['duplicates_found'] += 1
|
||||
|
||||
# Keep the one with the most recent last_seen
|
||||
cursor.execute("""
|
||||
SELECT id FROM products
|
||||
WHERE id IN ({})
|
||||
ORDER BY last_seen DESC, first_seen ASC
|
||||
LIMIT 1
|
||||
""".format(','.join('?' * len(ids))), ids)
|
||||
keep_id = cursor.fetchone()['id']
|
||||
|
||||
# Delete the others
|
||||
delete_ids = [i for i in ids if i != keep_id]
|
||||
if delete_ids:
|
||||
cursor.execute("""
|
||||
DELETE FROM products WHERE id IN ({})
|
||||
""".format(','.join('?' * len(delete_ids))), delete_ids)
|
||||
stats['products_removed'] += len(delete_ids)
|
||||
stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids)
|
||||
|
||||
logger.info(f"Deduplication complete: {stats}")
|
||||
return stats
|
||||
|
||||
def deduplicate_events(self) -> Dict:
|
||||
"""
|
||||
Remove duplicate stock events (same product, same event type within 1 hour).
|
||||
Keeps only the earliest event in each duplicate group.
|
||||
Returns stats about what was removed.
|
||||
"""
|
||||
stats = {'duplicate_groups': 0, 'events_removed': 0}
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Find events that have duplicates within 1 hour
|
||||
# We'll keep the earliest one in each group
|
||||
cursor.execute("""
|
||||
DELETE FROM stock_events
|
||||
WHERE id NOT IN (
|
||||
SELECT MIN(id)
|
||||
FROM stock_events
|
||||
GROUP BY product_id, event_type,
|
||||
strftime('%Y-%m-%d %H', recorded_at)
|
||||
)
|
||||
""")
|
||||
stats['events_removed'] = cursor.rowcount
|
||||
|
||||
logger.info(f"Event deduplication complete: removed {stats['events_removed']} duplicate events")
|
||||
return stats
|
||||
|
||||
# ==================== User Methods ====================
|
||||
|
||||
def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,
|
||||
|
||||
Reference in New Issue
Block a user