Cleanups for scrapers

scrapers/target.py

@@ -14,6 +14,50 @@ from src.browser import get_browser
 logger = logging.getLogger(__name__)
 
 
+def _find_image_in_json(data, depth=0) -> Optional[str]:
+    """Recursively search a JSON dict for a Target image URL"""
+    if depth > 6:
+        return None
+    if isinstance(data, str):
+        if 'scene7.com' in data and not data.endswith('/'):
+            return data
+        return None
+    if isinstance(data, dict):
+        # Check known Target image keys first
+        for key in ('primary_image_url', 'base_url', 'url', 'src'):
+            val = data.get(key)
+            if isinstance(val, str) and 'scene7.com' in val:
+                return val
+        # Recurse, prioritizing enrichment/images paths
+        for key in ('enrichment', 'images', 'image'):
+            if key in data:
+                result = _find_image_in_json(data[key], depth + 1)
+                if result:
+                    return result
+        for val in data.values():
+            result = _find_image_in_json(val, depth + 1)
+            if result:
+                return result
+    if isinstance(data, list) and data:
+        return _find_image_in_json(data[0], depth + 1)
+    return None
+
+
+def _extract_img_url(img_tag) -> Optional[str]:
+    """Extract real image URL from an img tag, skipping lazy-load placeholders"""
+    if not img_tag:
+        return None
+    for attr in ("src", "data-src", "data-lazy-src", "data-srcset", "srcset"):
+        val = img_tag.get(attr, "")
+        if not val:
+            continue
+        # srcset may contain multiple URLs - take the first
+        url = val.split()[0].rstrip(",")
+        if url and not url.startswith("data:") and len(url) > 20:
+            return url
+    return None
+
+
 class TargetScraper(BaseScraper):
     """Scraper for Target.com"""
 
@@ -178,9 +222,8 @@ class TargetScraper(BaseScraper):
             if fulfillment:
                 in_stock = fulfillment.get("is_out_of_stock_in_all_store_locations", True) is False
 
-            # Get image
-            images = data.get("images", [])
-            image_url = images[0].get("base_url") if images else None
+            # Get image - search all known Target JSON paths
+            image_url = _find_image_in_json(data)
 
             return Product(
                 name=name,
@@ -247,11 +290,16 @@ class TargetScraper(BaseScraper):
                         break
                     parent = parent.find_parent()
 
-            # Try to find image
-            image_url = None
-            img = link.select_one("img")
-            if img:
-                image_url = img.get("src") or img.get("data-src")
+            # Try to find image - check inside link first, then walk up to parents
+            image_url = _extract_img_url(link.select_one("img"))
+            if not image_url:
+                parent = link.find_parent()
+                for _ in range(4):
+                    if parent:
+                        image_url = _extract_img_url(parent.select_one("img"))
+                        if image_url:
+                            break
+                        parent = parent.find_parent()
 
             # Check stock (assume in stock unless we see otherwise)
             in_stock = True
@@ -307,8 +355,7 @@ class TargetScraper(BaseScraper):
             in_stock = not out_of_stock
 
             # Get image
-            img = card.select_one("img")
-            image_url = img.get("src") if img else None
+            image_url = _extract_img_url(card.select_one("img"))
 
             # Extract product ID from URL
             product_id = ""