Cleanups for scrapers

tools/stealth_browser.py

@@ -117,6 +117,10 @@ class StealthBrowser:
         options.add_argument("--no-sandbox")
         options.add_argument("--disable-infobars")
 
+        # Headless mode (must be set via options in uc >= 3.5)
+        if self.headless:
+            options.add_argument("--headless=new")
+
         # Window size (realistic resolution)
         options.add_argument("--window-size=1920,1080")
 
@@ -130,10 +134,6 @@ class StealthBrowser:
             if proxy_str:
                 options.add_argument(f"--proxy-server={proxy_str}")
 
-        # Headless mode (note: more detectable)
-        if self.headless:
-            options.add_argument("--headless=new")
-
         return options
 
     def _format_proxy(self, proxy: Dict) -> Optional[str]:
@@ -164,6 +164,7 @@ class StealthBrowser:
             # This prevents "ChromeDriver only supports Chrome version X" errors
             self.driver = uc.Chrome(
                 options=options,
+                headless=self.headless,
                 use_subprocess=True,
                 version_main=146,  # Match user's Chrome version
             )
@@ -207,7 +208,13 @@ class StealthBrowser:
             return True
         except Exception as e:
             error_msg = str(e).lower()
-            if "invalid session id" in error_msg or "session deleted" in error_msg or "no such session" in error_msg:
+            dead_session_errors = [
+                "invalid session id", "session deleted", "no such session",
+                "newconnectionerror", "connection refused", "winerror 10061",
+                "failed to establish a new connection", "max retries exceeded",
+                "browser has closed",
+            ]
+            if any(err in error_msg for err in dead_session_errors):
                 logger.warning(f"Browser session invalid: {e}")
                 return False
             # Other errors might be temporary
@@ -430,6 +437,64 @@ class StealthBrowser:
 
         return False
 
+    def resolve_captcha_interactively(self, url: str, timeout: int = 120) -> bool:
+        """
+        When running headless and a CAPTCHA is detected, temporarily pop a visible browser
+        window for the user to solve it, then return to headless mode with the updated cookies.
+        """
+        if not self.headless:
+            # Already visible - just wait in place
+            return self.wait_for_captcha_solve(timeout)
+
+        logger.info("CAPTCHA detected in headless mode - switching to visible browser for manual solve...")
+        print("\n" + "=" * 60)
+        print("CAPTCHA DETECTED - Opening visible browser window")
+        print("Please solve the challenge, then it will return to headless.")
+        print("=" * 60 + "\n")
+
+        # Save current cookies before restarting
+        self._save_cookies()
+
+        # Stop headless driver
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception:
+                pass
+            self.driver = None
+            self._setup_complete = False
+
+        # Restart visible
+        self.headless = False
+        solved = False
+        try:
+            self.start()
+            self.driver.get(url)
+            time.sleep(3)
+            solved = self.wait_for_captcha_solve(timeout)
+            if solved:
+                self._save_cookies()
+                logger.info("CAPTCHA solved - saving session and returning to headless mode")
+                print("\n>>> CAPTCHA solved! Returning to headless mode...\n")
+        except Exception as e:
+            logger.error(f"Error during interactive CAPTCHA solve: {e}")
+        finally:
+            # Always return to headless
+            if self.driver:
+                try:
+                    self.driver.quit()
+                except Exception:
+                    pass
+                self.driver = None
+                self._setup_complete = False
+            self.headless = True
+            try:
+                self.start()
+            except Exception as e:
+                logger.error(f"Failed to restart headless browser after CAPTCHA solve: {e}")
+
+        return solved
+
     def wait_for_captcha_solve(self, timeout: int = 120):
         """
         Wait for user to solve CAPTCHA manually.