mmcghen/pokemon-stock-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement Walmart scraper and integrate with existing architecture

Browse Source 
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
Michael McGhen
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/pokemoncenter.py
+82 -24
View File
@@ -1,6 +1,6 @@
"""
PokemonCenter.com scraper
Handles bot protection with Playwright stealth
Handles bot protection with undetected-chromedriver stealth browser
"""
import re
@@ -9,44 +9,83 @@ from typing import List, Optional
from bs4 import BeautifulSoup
from .base import BaseScraper, Product
from browser import get_browser
import config
logger = logging.getLogger(__name__)
def get_stealth_or_regular_browser():
"""Get appropriate browser based on config"""
if config.USE_STEALTH_BROWSER:
from tools.stealth_browser import get_stealth_browser
return get_stealth_browser(
headless=config.STEALTH_HEADLESS,
session_name=config.STEALTH_SESSION_NAME
), True
else:
from src.browser import get_browser
return get_browser(), False
class PokemonCenterScraper(BaseScraper):
"""Scraper for PokemonCenter.com"""
site_name = "pokemoncenter"
base_url = "https://www.pokemoncenter.com"
def scrape_category_page(self, url: str) -> List[Product]:
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
"""
Scrape a PokemonCenter category page for all products
Args:
url: Category page URL
max_pages: Maximum pages to scrape (not yet implemented for PokemonCenter)
Returns:
List of Product objects
"""
browser = get_browser()
# TODO: Implement pagination for PokemonCenter when needed
browser, is_stealth = get_stealth_or_regular_browser()
products = []
try:
# Navigate and wait for page to load (don't wait for specific selector)
page, html = browser.get_page_content(
url,
wait_for_selector=None, # Let it use networkidle instead
timeout=60000,
)
if is_stealth:
# Use stealth browser (undetected-chromedriver)
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
# Save screenshot for debugging if needed
try:
page.screenshot(path="debug_screenshot.png")
logger.info("Saved debug screenshot to debug_screenshot.png")
except:
pass
# Check for CAPTCHA
if browser.check_for_captcha():
logger.warning("CAPTCHA detected on PokemonCenter!")
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
# Re-fetch page after CAPTCHA solve
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
else:
logger.error("Cannot solve CAPTCHA in headless mode")
return []
# Save screenshot for debugging
try:
browser.screenshot("debug_screenshot.png")
logger.info("Saved debug screenshot to debug_screenshot.png")
except:
pass
page = None # Stealth browser doesn't return page object
else:
# Use regular Playwright browser
page, html = browser.get_page_content(
url,
wait_for_selector=None, # Let it use networkidle instead
timeout=60000,
)
# Save screenshot for debugging if needed
try:
page.screenshot(path="debug_screenshot.png")
logger.info("Saved debug screenshot to debug_screenshot.png")
except:
pass
soup = BeautifulSoup(html, "html.parser")
@@ -96,7 +135,9 @@ class PokemonCenterScraper(BaseScraper):
if product:
products.append(product)
page.close()
# Close page only for Playwright browser
if page is not None:
page.close()
except Exception as e:
logger.error(f"Error scraping PokemonCenter category page: {e}")
@@ -224,14 +265,30 @@ class PokemonCenterScraper(BaseScraper):
Returns:
Tuple of (is_in_stock, price)
"""
browser = get_browser()
browser, is_stealth = get_stealth_or_regular_browser()
try:
page, html = browser.get_page_content(
product_url,
wait_for_selector="button, [data-testid]",
timeout=30000,
)
if is_stealth:
# Use stealth browser
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
# Check for CAPTCHA
if browser.check_for_captcha():
logger.warning("CAPTCHA detected on product page!")
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
else:
return False, None
page = None
else:
# Use regular Playwright browser
page, html = browser.get_page_content(
product_url,
wait_for_selector="button, [data-testid]",
timeout=30000,
)
soup = BeautifulSoup(html, "html.parser")
page_text = soup.get_text().lower()
@@ -257,7 +314,8 @@ class PokemonCenterScraper(BaseScraper):
if price_match:
price = price_match.group()
page.close()
if page is not None:
page.close()
return in_stock, price
except Exception as e: