feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
+74
-56
@@ -9,7 +9,7 @@ from typing import List, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import BaseScraper, Product
|
||||
from browser import get_browser
|
||||
from src.browser import get_browser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,87 +20,105 @@ class TargetScraper(BaseScraper):
|
||||
site_name = "target"
|
||||
base_url = "https://www.target.com"
|
||||
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||
"""
|
||||
Scrape a Target search/category page for all products
|
||||
|
||||
Args:
|
||||
url: Search/category page URL
|
||||
max_pages: Maximum number of pages to scrape (default 1)
|
||||
|
||||
Returns:
|
||||
List of Product objects
|
||||
"""
|
||||
browser = get_browser()
|
||||
products = []
|
||||
all_products = []
|
||||
|
||||
try:
|
||||
page, html = browser.get_page_content(
|
||||
url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
# Add sort by newest if not already in URL
|
||||
if "sortBy=" not in url:
|
||||
separator = "&" if "?" in url else "?"
|
||||
url = f"{url}{separator}sortBy=newest"
|
||||
|
||||
for page_num in range(1, max_pages + 1):
|
||||
products = []
|
||||
page_url = url if page_num == 1 else f"{url}&Nao={24 * (page_num - 1)}"
|
||||
|
||||
# Save debug screenshot
|
||||
try:
|
||||
page.screenshot(path="debug_target.png")
|
||||
logger.info("Saved debug screenshot to debug_target.png")
|
||||
except:
|
||||
pass
|
||||
page, html = browser.get_page_content(
|
||||
page_url,
|
||||
wait_for_selector=None,
|
||||
timeout=60000,
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Try to find product data in page scripts (Target uses React/hydration)
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
# Save debug screenshot
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
# Look for product data in the JSON
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
page.screenshot(path="debug_target.png")
|
||||
logger.info("Saved debug screenshot to debug_target.png")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Parse product links from the page
|
||||
product_links = soup.select("a[href*='/p/']")
|
||||
logger.info(f"Found {len(product_links)} product links")
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Group links by href and pick the best one (with aria-label or text)
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
# Try to find product data in page scripts (Target uses React/hydration)
|
||||
scripts = soup.find_all("script", type="application/json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string)
|
||||
# Look for product data in the JSON
|
||||
products.extend(self._extract_products_from_json(data))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
|
||||
logger.info(f"Found {len(href_to_links)} unique hrefs")
|
||||
# Parse product links from the page
|
||||
product_links = soup.select("a[href*='/p/']")
|
||||
logger.info(f"Found {len(product_links)} product links")
|
||||
|
||||
for href, links in href_to_links.items():
|
||||
# Find the best link (one with aria-label or text content)
|
||||
best_link = None
|
||||
for link in links:
|
||||
aria = link.get("aria-label", "")
|
||||
text = link.get_text(strip=True)
|
||||
if aria or (text and len(text) > 10):
|
||||
best_link = link
|
||||
break
|
||||
if not best_link:
|
||||
best_link = links[0] # Fallback to first link
|
||||
# Group links by href and pick the best one (with aria-label or text)
|
||||
href_to_links = {}
|
||||
for link in product_links:
|
||||
href = link.get("href", "")
|
||||
if not href:
|
||||
continue
|
||||
if href not in href_to_links:
|
||||
href_to_links[href] = []
|
||||
href_to_links[href].append(link)
|
||||
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
logger.info(f"Found {len(href_to_links)} unique hrefs")
|
||||
|
||||
page.close()
|
||||
for href, links in href_to_links.items():
|
||||
# Find the best link (one with aria-label or text content)
|
||||
best_link = None
|
||||
for link in links:
|
||||
aria = link.get("aria-label", "")
|
||||
text = link.get_text(strip=True)
|
||||
if aria or (text and len(text) > 10):
|
||||
best_link = link
|
||||
break
|
||||
if not best_link:
|
||||
best_link = links[0] # Fallback to first link
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Target category page: {e}")
|
||||
raise
|
||||
product = self._parse_product_link(best_link, soup)
|
||||
if product:
|
||||
products.append(product)
|
||||
|
||||
page.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping Target page {page_num}: {e}")
|
||||
if page_num == 1:
|
||||
raise
|
||||
break
|
||||
|
||||
all_products.extend(products)
|
||||
|
||||
# Stop if no products found on this page
|
||||
if not products:
|
||||
break
|
||||
|
||||
# Remove duplicates
|
||||
seen_urls = set()
|
||||
unique_products = []
|
||||
for p in products:
|
||||
for p in all_products:
|
||||
if p.url not in seen_urls:
|
||||
seen_urls.add(p.url)
|
||||
unique_products.append(p)
|
||||
|
||||
Reference in New Issue
Block a user