""" Browser automation with Playwright and stealth patches Handles bot protection bypass for retail sites """ import logging from playwright.sync_api import sync_playwright, Browser, Page, BrowserContext from playwright_stealth import Stealth from config import HEADLESS, SLOW_MO, CHROME_USER_DATA_DIR, USE_REAL_CHROME logger = logging.getLogger(__name__) # Create stealth instance stealth = Stealth() class StealthBrowser: """Manages a stealth browser instance that bypasses bot detection""" def __init__(self): self.playwright = None self.browser: Browser = None self.context: BrowserContext = None self.using_persistent = False def start(self): """Initialize the browser""" logger.info("Starting stealth browser...") self.playwright = sync_playwright().start() # Check if we should use a persistent Chrome profile if CHROME_USER_DATA_DIR: logger.info(f"Using Chrome profile from: {CHROME_USER_DATA_DIR}") self._start_with_profile() else: self._start_fresh() logger.info("Browser started successfully") def _start_fresh(self): """Start with a fresh browser profile""" # Launch browser - use real Chrome if configured launch_options = { "headless": HEADLESS, "slow_mo": SLOW_MO, "args": [ "--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage", "--no-sandbox", ] } if USE_REAL_CHROME: launch_options["channel"] = "chrome" logger.info("Using real Chrome browser") self.browser = self.playwright.chromium.launch(**launch_options) # Create context with realistic viewport and user agent self.context = self.browser.new_context( viewport={"width": 1920, "height": 1080}, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", locale="en-US", timezone_id="America/New_York", ) # Apply stealth to context (affects all pages created from it) stealth.apply_stealth_sync(self.context) def _start_with_profile(self): """Start using an existing Chrome profile (better for bot detection)""" self.using_persistent = True # Launch persistent context with user's REAL Chrome (not Playwright's Chromium) # This is needed because Chrome encrypts credentials with its own binary self.context = self.playwright.chromium.launch_persistent_context( user_data_dir=CHROME_USER_DATA_DIR, channel="chrome", # Use real Chrome, not Playwright's Chromium headless=HEADLESS, slow_mo=SLOW_MO, viewport={"width": 1920, "height": 1080}, locale="en-US", timezone_id="America/New_York", args=[ "--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage", ] ) # Apply stealth stealth.apply_stealth_sync(self.context) def new_page(self) -> Page: """Create a new page with stealth patches applied""" if not self.context: raise RuntimeError("Browser not started. Call start() first.") page = self.context.new_page() return page def get_page_content(self, url: str, wait_for_selector: str = None, timeout: int = 30000) -> tuple[Page, str]: """ Navigate to URL and return page + HTML content Args: url: URL to navigate to wait_for_selector: CSS selector to wait for before returning timeout: Max time to wait in milliseconds Returns: Tuple of (page, html_content) """ page = self.new_page() try: logger.info(f"Navigating to: {url}") page.goto(url, wait_until="domcontentloaded", timeout=timeout) # Wait for specific element if provided if wait_for_selector: logger.debug(f"Waiting for selector: {wait_for_selector}") page.wait_for_selector(wait_for_selector, timeout=timeout) else: # Wait for DOM to be ready, then give extra time for JS rendering try: page.wait_for_load_state("networkidle", timeout=15000) except: # networkidle can timeout on dynamic sites, that's OK logger.debug("networkidle timeout, continuing anyway") # Let dynamic content render page.wait_for_timeout(5000) # Scroll down to trigger lazy loading page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)") page.wait_for_timeout(2000) page.evaluate("window.scrollTo(0, document.body.scrollHeight)") page.wait_for_timeout(2000) page.evaluate("window.scrollTo(0, 0)") page.wait_for_timeout(1000) content = page.content() logger.info(f"Page loaded, content length: {len(content)}") return page, content except Exception as e: logger.error(f"Error loading page {url}: {e}") page.close() raise def stop(self): """Clean up browser resources""" logger.info("Stopping browser...") if self.context: self.context.close() if self.browser and not self.using_persistent: # Only close browser if not using persistent context self.browser.close() if self.playwright: self.playwright.stop() logger.info("Browser stopped") # Global browser instance _browser: StealthBrowser = None def get_browser() -> StealthBrowser: """Get or create the global browser instance""" global _browser if _browser is None: _browser = StealthBrowser() _browser.start() return _browser def shutdown_browser(): """Shutdown the global browser instance""" global _browser if _browser: _browser.stop() _browser = None