Remove duplicate products from Target, GameStop, and other stores
+ + + +Remove duplicate events from the activity feed (same product/type within 1 hour)
+ + diff --git a/scrapers/base.py b/scrapers/base.py index abde8a2..0b195d0 100644 --- a/scrapers/base.py +++ b/scrapers/base.py @@ -3,13 +3,79 @@ Base scraper class with common functionality """ import logging +import re from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Optional +from urllib.parse import urlparse, urlunparse, parse_qs, urlencode logger = logging.getLogger(__name__) +def clean_product_name(name: str) -> str: + """Clean garbage from product names (SVG classes, CSS, etc.)""" + if not name: + return name + + import re + + # Remove SVG/CSS class garbage (pikachu_svg__st1{fill:#7c888f}.review-full-...) + name = re.sub(r'[\w-]*_svg__[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE) + name = re.sub(r'\.review-full-[\w\s{}:#.;-]+', '', name, flags=re.IGNORECASE) + name = re.sub(r'\{[^}]*\}', '', name) # Remove any remaining {css} blocks + name = re.sub(r'\d+\s*Reviews?$', '', name, flags=re.IGNORECASE) # Remove trailing "X Reviews" + + # Clean up multiple spaces + name = re.sub(r'\s+', ' ', name) + + return name.strip() + + +def normalize_url(url: str) -> str: + """ + Normalize a URL to prevent duplicates from minor variations. + - Removes tracking/session query parameters + - Removes trailing slashes + - Lowercases the domain + - Keeps only essential path components + """ + if not url: + return url + + try: + parsed = urlparse(url) + + # Lowercase the domain + netloc = parsed.netloc.lower() + + # Remove trailing slash from path + path = parsed.path.rstrip('/') + + # Filter out common tracking/non-essential query params + query_params = parse_qs(parsed.query) + exclude_params = { + 'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content', + 'utm_term', 'gclid', 'fbclid', 'msclkid', 'dclid', 'zanpid', + 'src', 'srsltid', 'skuId', 'preselect', 'intl', 'lnk', + 'Nao', 'start', 'page', 'sort', 'sortBy', 'facets', + } + filtered_params = { + k: v for k, v in query_params.items() + if k.lower() not in {p.lower() for p in exclude_params} + } + + # Rebuild query string (sorted for consistency) + query = urlencode(sorted(filtered_params.items()), doseq=True) if filtered_params else '' + + # Rebuild URL without fragment + normalized = urlunparse((parsed.scheme, netloc, path, '', query, '')) + return normalized + + except Exception as e: + logger.debug(f"Error normalizing URL {url}: {e}") + return url + + @dataclass class Product: """Represents a product listing""" @@ -20,13 +86,31 @@ class Product: image_url: Optional[str] = None site: str = "" product_id: Optional[str] = None # Unique identifier for tracking + _normalized_url: str = field(default="", repr=False, compare=False) + + def __post_init__(self): + self._normalized_url = normalize_url(self.url) + # Clean the name of any SVG/CSS garbage + if self.name: + self.name = clean_product_name(self.name) + + @property + def normalized_url(self) -> str: + """Get the normalized URL for deduplication""" + if not self._normalized_url: + self._normalized_url = normalize_url(self.url) + return self._normalized_url def __hash__(self): - return hash(self.url) + # Use normalized URL for hashing to prevent duplicates + return hash(self.normalized_url) def __eq__(self, other): if isinstance(other, Product): - return self.url == other.url + # Compare by normalized URL OR by product_id if both have one + if self.product_id and other.product_id and self.site == other.site: + return self.product_id == other.product_id + return self.normalized_url == other.normalized_url return False diff --git a/src/database.py b/src/database.py index 465a668..5666844 100644 --- a/src/database.py +++ b/src/database.py @@ -166,19 +166,47 @@ class Database: # ==================== Product Methods ==================== + def _normalize_url(self, url: str) -> str: + """Normalize URL to prevent duplicates from minor variations.""" + if not url: + return url + try: + from urllib.parse import urlparse, urlunparse, parse_qs, urlencode + parsed = urlparse(url) + netloc = parsed.netloc.lower() + path = parsed.path.rstrip('/') + # Filter out tracking/pagination params + query_params = parse_qs(parsed.query) + exclude_params = { + 'ref', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_content', + 'utm_term', 'gclid', 'fbclid', 'msclkid', 'srsltid', 'skuId', + 'preselect', 'intl', 'lnk', 'nao', 'start', 'page', 'sort', 'sortby', + } + filtered = {k: v for k, v in query_params.items() if k.lower() not in exclude_params} + query = urlencode(sorted(filtered.items()), doseq=True) if filtered else '' + return urlunparse((parsed.scheme, netloc, path, '', query, '')) + except Exception: + return url + def get_or_create_product(self, url: str, name: str, site: str, product_id: str = None, image_url: str = None, price: str = None, in_stock: bool = False) -> int: """Get existing product or create new one. Returns product ID.""" category = self._detect_category(name) + normalized_url = self._normalize_url(url) with self.get_connection() as conn: cursor = conn.cursor() - # Try to get existing product - cursor.execute("SELECT id FROM products WHERE url = ?", (url,)) + # Try to get existing product by URL (check both original and normalized) + cursor.execute("SELECT id FROM products WHERE url = ? OR url = ?", (url, normalized_url)) row = cursor.fetchone() + # Also check by product_id if provided (more reliable for some sites) + if not row and product_id and site: + cursor.execute("SELECT id FROM products WHERE product_id = ? AND site = ?", (product_id, site)) + row = cursor.fetchone() + if row: # Update existing product cursor.execute(""" @@ -190,12 +218,12 @@ class Database: """, (name, price, in_stock, image_url, category, row['id'])) return row['id'] else: - # Create new product + # Create new product with normalized URL cursor.execute(""" INSERT INTO products (url, name, site, product_id, image_url, current_price, in_stock, last_seen, category) VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?) - """, (url, name, site, product_id, image_url, price, in_stock, category)) + """, (normalized_url, name, site, product_id, image_url, price, in_stock, category)) return cursor.lastrowid def get_product(self, product_id: int) -> Optional[Dict]: @@ -207,10 +235,11 @@ class Database: return dict(row) if row else None def get_product_by_url(self, url: str) -> Optional[Dict]: - """Get product by URL""" + """Get product by URL (checks both original and normalized)""" + normalized_url = self._normalize_url(url) with self.get_connection() as conn: cursor = conn.cursor() - cursor.execute("SELECT * FROM products WHERE url = ?", (url,)) + cursor.execute("SELECT * FROM products WHERE url = ? OR url = ?", (url, normalized_url)) row = cursor.fetchone() return dict(row) if row else None @@ -369,9 +398,23 @@ class Database: # ==================== Stock Event Methods ==================== def record_stock_event(self, product_id: int, event_type: str): - """Record a stock event (new_drop, restock, in_stock, out_of_stock)""" + """Record a stock event (new_drop, restock, in_stock, out_of_stock). + Prevents duplicate events within 1 hour for the same product/event_type.""" with self.get_connection() as conn: cursor = conn.cursor() + + # Check if same event was recorded recently (within 1 hour) + cursor.execute(""" + SELECT id FROM stock_events + WHERE product_id = ? AND event_type = ? + AND recorded_at > datetime('now', '-1 hour') + LIMIT 1 + """, (product_id, event_type)) + + if cursor.fetchone(): + logger.debug(f"Skipping duplicate {event_type} event for product {product_id}") + return + cursor.execute(""" INSERT INTO stock_events (product_id, event_type) VALUES (?, ?) @@ -379,23 +422,42 @@ class Database: logger.debug(f"Recorded {event_type} event for product {product_id}") def get_recent_events(self, limit: int = 20, event_types: List[str] = None) -> List[Dict]: - """Get recent stock events with product info""" + """Get recent stock events with product info. + Deduplicates by product name to avoid showing same product multiple times.""" with self.get_connection() as conn: cursor = conn.cursor() - query = """ - SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price - FROM stock_events e - JOIN products p ON e.product_id = p.id - """ + # Use a subquery to get only the most recent event per product + # This prevents the same product from appearing multiple times params = [] if event_types: placeholders = ",".join("?" * len(event_types)) - query += f" WHERE e.event_type IN ({placeholders})" - params.extend(event_types) + event_filter = f"e.event_type IN ({placeholders})" + event_filter2 = f"e2.event_type IN ({placeholders})" + params = list(event_types) + else: + event_filter = "1=1" + event_filter2 = "1=1" - query += " ORDER BY e.recorded_at DESC LIMIT ?" + query = f""" + SELECT e.*, p.name, p.url, p.site, p.image_url, p.current_price + FROM stock_events e + JOIN products p ON e.product_id = p.id + WHERE {event_filter} + AND e.id IN ( + SELECT MAX(e2.id) + FROM stock_events e2 + JOIN products p2 ON e2.product_id = p2.id + WHERE {event_filter2} + GROUP BY LOWER(TRIM(p2.name)) + ) + ORDER BY e.recorded_at DESC + LIMIT ? + """ + # Add params for subquery event filter if needed + if event_types: + params.extend(event_types) params.append(limit) cursor.execute(query, params) @@ -834,6 +896,112 @@ class Database: logger.info(f"Migrated {migrated} products from {json_path}") + def deduplicate_products(self) -> Dict: + """ + Find and remove duplicate products based on normalized URL or product_id. + Keeps the product with the most recent last_seen date. + Returns stats about what was deduplicated. + """ + stats = {'duplicates_found': 0, 'products_removed': 0, 'by_site': {}} + + with self.get_connection() as conn: + cursor = conn.cursor() + + # Find duplicates by product_id (same site) + cursor.execute(""" + SELECT site, product_id, COUNT(*) as count, GROUP_CONCAT(id) as ids + FROM products + WHERE product_id IS NOT NULL AND product_id != '' + GROUP BY site, product_id + HAVING count > 1 + """) + product_id_dupes = cursor.fetchall() + + for row in product_id_dupes: + site = row['site'] + ids = [int(i) for i in row['ids'].split(',')] + stats['duplicates_found'] += 1 + + # Keep the one with the most recent last_seen + cursor.execute(""" + SELECT id FROM products + WHERE id IN ({}) + ORDER BY last_seen DESC, first_seen ASC + LIMIT 1 + """.format(','.join('?' * len(ids))), ids) + keep_id = cursor.fetchone()['id'] + + # Delete the others + delete_ids = [i for i in ids if i != keep_id] + if delete_ids: + cursor.execute(""" + DELETE FROM products WHERE id IN ({}) + """.format(','.join('?' * len(delete_ids))), delete_ids) + stats['products_removed'] += len(delete_ids) + stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids) + + # Find duplicates by similar name (same site, fuzzy match) + cursor.execute(""" + SELECT site, LOWER(TRIM(name)) as clean_name, COUNT(*) as count, GROUP_CONCAT(id) as ids + FROM products + GROUP BY site, clean_name + HAVING count > 1 + """) + name_dupes = cursor.fetchall() + + for row in name_dupes: + site = row['site'] + ids = [int(i) for i in row['ids'].split(',')] + stats['duplicates_found'] += 1 + + # Keep the one with the most recent last_seen + cursor.execute(""" + SELECT id FROM products + WHERE id IN ({}) + ORDER BY last_seen DESC, first_seen ASC + LIMIT 1 + """.format(','.join('?' * len(ids))), ids) + keep_id = cursor.fetchone()['id'] + + # Delete the others + delete_ids = [i for i in ids if i != keep_id] + if delete_ids: + cursor.execute(""" + DELETE FROM products WHERE id IN ({}) + """.format(','.join('?' * len(delete_ids))), delete_ids) + stats['products_removed'] += len(delete_ids) + stats['by_site'][site] = stats['by_site'].get(site, 0) + len(delete_ids) + + logger.info(f"Deduplication complete: {stats}") + return stats + + def deduplicate_events(self) -> Dict: + """ + Remove duplicate stock events (same product, same event type within 1 hour). + Keeps only the earliest event in each duplicate group. + Returns stats about what was removed. + """ + stats = {'duplicate_groups': 0, 'events_removed': 0} + + with self.get_connection() as conn: + cursor = conn.cursor() + + # Find events that have duplicates within 1 hour + # We'll keep the earliest one in each group + cursor.execute(""" + DELETE FROM stock_events + WHERE id NOT IN ( + SELECT MIN(id) + FROM stock_events + GROUP BY product_id, event_type, + strftime('%Y-%m-%d %H', recorded_at) + ) + """) + stats['events_removed'] = cursor.rowcount + + logger.info(f"Event deduplication complete: removed {stats['events_removed']} duplicate events") + return stats + # ==================== User Methods ==================== def create_user(self, name: str, zip_code: str = None, radius_miles: int = 25,