Browser session recovery, price tracking, and dashboard improvements
- Add browser session validation and auto-restart for BestBuy/GameStop scrapers - Add price change detection and notifications - Remove PokemonCenter scraper (now uses Chrome Extension) - Dashboard UI improvements Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -771,6 +771,9 @@ async function loadControlPanel() {
|
|||||||
// Render scrapers grid
|
// Render scrapers grid
|
||||||
renderScrapersGrid(state.scrapers);
|
renderScrapersGrid(state.scrapers);
|
||||||
|
|
||||||
|
// Load Chrome Extension status
|
||||||
|
loadExtensionStatus();
|
||||||
|
|
||||||
// Setup event listeners for control panel buttons
|
// Setup event listeners for control panel buttons
|
||||||
setupControlPanelListeners();
|
setupControlPanelListeners();
|
||||||
|
|
||||||
@@ -783,10 +786,36 @@ async function loadControlPanel() {
|
|||||||
updateMonitorStatus(newState);
|
updateMonitorStatus(newState);
|
||||||
renderScrapersGrid(newState.scrapers);
|
renderScrapersGrid(newState.scrapers);
|
||||||
}
|
}
|
||||||
|
loadExtensionStatus();
|
||||||
}
|
}
|
||||||
}, 5000);
|
}, 5000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadExtensionStatus() {
|
||||||
|
const badge = document.getElementById('extensionBadge');
|
||||||
|
const productCount = document.getElementById('extProductCount');
|
||||||
|
const skuCount = document.getElementById('extSkuCount');
|
||||||
|
const lastSync = document.getElementById('extLastSync');
|
||||||
|
|
||||||
|
const stats = await api('/extension/stats');
|
||||||
|
|
||||||
|
if (stats && stats.last_sync) {
|
||||||
|
// Extension has synced data
|
||||||
|
badge.textContent = 'Connected';
|
||||||
|
badge.className = 'extension-badge connected';
|
||||||
|
productCount.textContent = stats.total_products || 0;
|
||||||
|
skuCount.textContent = stats.total_skus || 0;
|
||||||
|
lastSync.textContent = formatTime(new Date(stats.last_sync));
|
||||||
|
} else {
|
||||||
|
// No sync yet
|
||||||
|
badge.textContent = 'No Data';
|
||||||
|
badge.className = 'extension-badge disconnected';
|
||||||
|
productCount.textContent = '-';
|
||||||
|
skuCount.textContent = '-';
|
||||||
|
lastSync.textContent = 'Never';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function updateMonitorStatus(state) {
|
function updateMonitorStatus(state) {
|
||||||
const badge = document.getElementById('monitorBadge');
|
const badge = document.getElementById('monitorBadge');
|
||||||
const statusText = document.getElementById('monitorStatusText');
|
const statusText = document.getElementById('monitorStatusText');
|
||||||
|
|||||||
@@ -1103,6 +1103,57 @@ h3 {
|
|||||||
grid-column: span 2;
|
grid-column: span 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Chrome Extension Card */
|
||||||
|
.extension-card {
|
||||||
|
grid-column: span 2;
|
||||||
|
border-color: #ffcb05;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-badge {
|
||||||
|
padding: 4px 12px;
|
||||||
|
border-radius: 12px;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-badge.connected {
|
||||||
|
background: rgba(0, 255, 0, 0.2);
|
||||||
|
color: #00ff00;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-badge.disconnected {
|
||||||
|
background: rgba(255, 100, 100, 0.2);
|
||||||
|
color: #ff6464;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-stats {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, 1fr);
|
||||||
|
gap: 15px;
|
||||||
|
margin-top: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-stat {
|
||||||
|
background: var(--bg-primary);
|
||||||
|
padding: 12px;
|
||||||
|
border-radius: 8px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-stat .stat-label {
|
||||||
|
display: block;
|
||||||
|
font-size: 12px;
|
||||||
|
color: var(--text-secondary);
|
||||||
|
margin-bottom: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.extension-stat .stat-value {
|
||||||
|
display: block;
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--text-primary);
|
||||||
|
}
|
||||||
|
|
||||||
.scrapers-grid {
|
.scrapers-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
|
grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
|
||||||
|
|||||||
@@ -274,6 +274,37 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Chrome Extension Status (Pokemon Center) -->
|
||||||
|
<div class="control-card extension-card">
|
||||||
|
<div class="control-card-header">
|
||||||
|
<h3>Pokemon Center (Chrome Extension)</h3>
|
||||||
|
<span class="extension-badge" id="extensionBadge">Unknown</span>
|
||||||
|
</div>
|
||||||
|
<div class="control-card-body">
|
||||||
|
<p class="help-text">Pokemon Center is monitored via the Chrome Extension, not a Python scraper.</p>
|
||||||
|
<div id="extensionStatus">
|
||||||
|
<div class="extension-stats">
|
||||||
|
<div class="extension-stat">
|
||||||
|
<span class="stat-label">Products Tracked:</span>
|
||||||
|
<span class="stat-value" id="extProductCount">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="extension-stat">
|
||||||
|
<span class="stat-label">SKUs Detected:</span>
|
||||||
|
<span class="stat-value" id="extSkuCount">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="extension-stat">
|
||||||
|
<span class="stat-label">Last Sync:</span>
|
||||||
|
<span class="stat-value" id="extLastSync">Never</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p class="help-text" style="margin-top: 10px;">
|
||||||
|
<a href="chrome-extension://YOUR_EXTENSION_ID/popup.html" target="_blank">Open Extension</a> |
|
||||||
|
Install from <code>chrome-extension/</code> folder
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Scraper Controls -->
|
<!-- Scraper Controls -->
|
||||||
<div class="control-card scrapers-card">
|
<div class="control-card scrapers-card">
|
||||||
<div class="control-card-header">
|
<div class="control-card-header">
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import schedule
|
|||||||
from config import (
|
from config import (
|
||||||
CHECK_INTERVAL_SECONDS,
|
CHECK_INTERVAL_SECONDS,
|
||||||
SITES_ENABLED,
|
SITES_ENABLED,
|
||||||
POKEMON_CENTER_URLS,
|
|
||||||
TARGET_URLS,
|
TARGET_URLS,
|
||||||
GAMESTOP_URLS,
|
GAMESTOP_URLS,
|
||||||
BESTBUY_URLS,
|
BESTBUY_URLS,
|
||||||
@@ -23,13 +22,14 @@ from config import (
|
|||||||
KEYWORDS,
|
KEYWORDS,
|
||||||
LOG_LEVEL,
|
LOG_LEVEL,
|
||||||
NEW_DROP_NOTIFICATIONS,
|
NEW_DROP_NOTIFICATIONS,
|
||||||
|
PRICE_CHANGE_NOTIFICATIONS,
|
||||||
MAX_PAGES,
|
MAX_PAGES,
|
||||||
)
|
)
|
||||||
from src.browser import get_browser, shutdown_browser
|
from src.browser import get_browser, shutdown_browser
|
||||||
from src.product_tracker import ProductTracker
|
from src.product_tracker import ProductTracker
|
||||||
from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification
|
from src.discord_notifier import send_stock_alert, send_startup_notification, send_error_notification, send_price_change_alert
|
||||||
|
# Note: Pokemon Center uses Chrome Extension (chrome-extension/), not a Python scraper
|
||||||
from scrapers import (
|
from scrapers import (
|
||||||
PokemonCenterScraper,
|
|
||||||
TargetScraper,
|
TargetScraper,
|
||||||
GameStopScraper,
|
GameStopScraper,
|
||||||
BestBuyScraper,
|
BestBuyScraper,
|
||||||
@@ -50,9 +50,8 @@ logger = logging.getLogger(__name__)
|
|||||||
# Global tracker
|
# Global tracker
|
||||||
tracker = ProductTracker()
|
tracker = ProductTracker()
|
||||||
|
|
||||||
# Scrapers
|
# Scrapers (Pokemon Center uses Chrome Extension instead)
|
||||||
scrapers = {
|
scrapers = {
|
||||||
"pokemoncenter": PokemonCenterScraper(),
|
|
||||||
"target": TargetScraper(),
|
"target": TargetScraper(),
|
||||||
"gamestop": GameStopScraper(),
|
"gamestop": GameStopScraper(),
|
||||||
"bestbuy": BestBuyScraper(),
|
"bestbuy": BestBuyScraper(),
|
||||||
@@ -60,67 +59,14 @@ scrapers = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def check_pokemoncenter():
|
|
||||||
"""Check PokemonCenter for restocks and new drops"""
|
|
||||||
logger.info("Checking PokemonCenter...")
|
|
||||||
scraper = scrapers["pokemoncenter"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
for url in POKEMON_CENTER_URLS:
|
|
||||||
logger.info(f"Scraping: {url}")
|
|
||||||
products = scraper.scrape_category_page(url, max_pages=MAX_PAGES.get("pokemoncenter", 1))
|
|
||||||
|
|
||||||
# Apply keyword filter if enabled
|
|
||||||
if KEYWORD_FILTER_ENABLED and KEYWORDS:
|
|
||||||
products = scraper.filter_by_keywords(products, KEYWORDS)
|
|
||||||
logger.info(f"After keyword filter: {len(products)} products")
|
|
||||||
|
|
||||||
# Process and detect changes
|
|
||||||
new_products, restocked_products = tracker.process_products(products)
|
|
||||||
|
|
||||||
# Send notifications for new products (if enabled)
|
|
||||||
if NEW_DROP_NOTIFICATIONS:
|
|
||||||
for product in new_products:
|
|
||||||
if product.in_stock:
|
|
||||||
send_stock_alert(
|
|
||||||
product_name=product.name,
|
|
||||||
product_url=product.url,
|
|
||||||
price=product.price or "See link",
|
|
||||||
site="pokemoncenter",
|
|
||||||
alert_type="new_drop",
|
|
||||||
image_url=product.image_url,
|
|
||||||
)
|
|
||||||
logger.info(f"Sent notification for new product: {product.name}")
|
|
||||||
|
|
||||||
# Send notifications for restocks
|
|
||||||
for product in restocked_products:
|
|
||||||
send_stock_alert(
|
|
||||||
product_name=product.name,
|
|
||||||
product_url=product.url,
|
|
||||||
price=product.price or "See link",
|
|
||||||
site="pokemoncenter",
|
|
||||||
alert_type="restock",
|
|
||||||
image_url=product.image_url,
|
|
||||||
)
|
|
||||||
logger.info(f"Sent notification for restock: {product.name}")
|
|
||||||
|
|
||||||
# Log stats
|
|
||||||
stats = tracker.get_stats()
|
|
||||||
logger.info(
|
|
||||||
f"Stats: {stats['total_products']} total, "
|
|
||||||
f"{stats['in_stock']} in stock, "
|
|
||||||
f"{stats['out_of_stock']} out of stock"
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error checking PokemonCenter: {e}")
|
|
||||||
send_error_notification(f"Error checking PokemonCenter: {str(e)}", "PokemonCenter")
|
|
||||||
|
|
||||||
|
|
||||||
def check_target():
|
def check_target():
|
||||||
"""Check Target for restocks and new drops"""
|
"""Check Target for restocks and new drops"""
|
||||||
logger.info("Checking Target...")
|
logger.info("Checking Target...")
|
||||||
scraper = scrapers["target"]
|
scraper = scrapers["target"]
|
||||||
|
start_time = time.time()
|
||||||
|
total_products = 0
|
||||||
|
total_new = 0
|
||||||
|
total_restocks = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for url in TARGET_URLS:
|
for url in TARGET_URLS:
|
||||||
@@ -135,8 +81,12 @@ def check_target():
|
|||||||
products = scraper.filter_by_keywords(products, KEYWORDS)
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
||||||
logger.info(f"After keyword filter: {len(products)} products")
|
logger.info(f"After keyword filter: {len(products)} products")
|
||||||
|
|
||||||
|
total_products += len(products)
|
||||||
|
|
||||||
# Process and detect changes
|
# Process and detect changes
|
||||||
new_products, restocked_products = tracker.process_products(products)
|
new_products, restocked_products, price_changed_products = tracker.process_products(products)
|
||||||
|
total_new += len(new_products)
|
||||||
|
total_restocks += len(restocked_products)
|
||||||
|
|
||||||
# Send notifications for new products (if enabled)
|
# Send notifications for new products (if enabled)
|
||||||
if NEW_DROP_NOTIFICATIONS:
|
if NEW_DROP_NOTIFICATIONS:
|
||||||
@@ -164,6 +114,20 @@ def check_target():
|
|||||||
)
|
)
|
||||||
logger.info(f"Sent notification for restock: {product.name}")
|
logger.info(f"Sent notification for restock: {product.name}")
|
||||||
|
|
||||||
|
# Send notifications for price changes (if enabled)
|
||||||
|
if PRICE_CHANGE_NOTIFICATIONS:
|
||||||
|
for product, old_price, new_price in price_changed_products:
|
||||||
|
send_price_change_alert(
|
||||||
|
product_name=product.name,
|
||||||
|
product_url=product.url,
|
||||||
|
old_price=old_price,
|
||||||
|
new_price=new_price,
|
||||||
|
site="target",
|
||||||
|
in_stock=product.in_stock,
|
||||||
|
image_url=product.image_url,
|
||||||
|
)
|
||||||
|
logger.info(f"Sent notification for price change: {product.name} ({old_price} -> {new_price})")
|
||||||
|
|
||||||
# Log stats
|
# Log stats
|
||||||
stats = tracker.get_stats()
|
stats = tracker.get_stats()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -172,15 +136,26 @@ def check_target():
|
|||||||
f"{stats['out_of_stock']} out of stock"
|
f"{stats['out_of_stock']} out of stock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Log successful check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("target", total_products, total_new, total_restocks, duration_ms, success=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking Target: {e}")
|
logger.error(f"Error checking Target: {e}")
|
||||||
send_error_notification(f"Error checking Target: {str(e)}", "Target")
|
send_error_notification(f"Error checking Target: {str(e)}", "Target")
|
||||||
|
# Log failed check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("target", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
|
||||||
|
|
||||||
|
|
||||||
def check_gamestop():
|
def check_gamestop():
|
||||||
"""Check GameStop for restocks and new drops"""
|
"""Check GameStop for restocks and new drops"""
|
||||||
logger.info("Checking GameStop...")
|
logger.info("Checking GameStop...")
|
||||||
scraper = scrapers["gamestop"]
|
scraper = scrapers["gamestop"]
|
||||||
|
start_time = time.time()
|
||||||
|
total_products = 0
|
||||||
|
total_new = 0
|
||||||
|
total_restocks = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for url in GAMESTOP_URLS:
|
for url in GAMESTOP_URLS:
|
||||||
@@ -195,8 +170,12 @@ def check_gamestop():
|
|||||||
products = scraper.filter_by_keywords(products, KEYWORDS)
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
||||||
logger.info(f"After keyword filter: {len(products)} products")
|
logger.info(f"After keyword filter: {len(products)} products")
|
||||||
|
|
||||||
|
total_products += len(products)
|
||||||
|
|
||||||
# Process and detect changes
|
# Process and detect changes
|
||||||
new_products, restocked_products = tracker.process_products(products)
|
new_products, restocked_products, price_changed_products = tracker.process_products(products)
|
||||||
|
total_new += len(new_products)
|
||||||
|
total_restocks += len(restocked_products)
|
||||||
|
|
||||||
# Send notifications for new products (if enabled)
|
# Send notifications for new products (if enabled)
|
||||||
if NEW_DROP_NOTIFICATIONS:
|
if NEW_DROP_NOTIFICATIONS:
|
||||||
@@ -224,6 +203,20 @@ def check_gamestop():
|
|||||||
)
|
)
|
||||||
logger.info(f"Sent notification for restock: {product.name}")
|
logger.info(f"Sent notification for restock: {product.name}")
|
||||||
|
|
||||||
|
# Send notifications for price changes (if enabled)
|
||||||
|
if PRICE_CHANGE_NOTIFICATIONS:
|
||||||
|
for product, old_price, new_price in price_changed_products:
|
||||||
|
send_price_change_alert(
|
||||||
|
product_name=product.name,
|
||||||
|
product_url=product.url,
|
||||||
|
old_price=old_price,
|
||||||
|
new_price=new_price,
|
||||||
|
site="gamestop",
|
||||||
|
in_stock=product.in_stock,
|
||||||
|
image_url=product.image_url,
|
||||||
|
)
|
||||||
|
logger.info(f"Sent notification for price change: {product.name} ({old_price} -> {new_price})")
|
||||||
|
|
||||||
# Log stats
|
# Log stats
|
||||||
stats = tracker.get_stats()
|
stats = tracker.get_stats()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -232,15 +225,26 @@ def check_gamestop():
|
|||||||
f"{stats['out_of_stock']} out of stock"
|
f"{stats['out_of_stock']} out of stock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Log successful check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("gamestop", total_products, total_new, total_restocks, duration_ms, success=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking GameStop: {e}")
|
logger.error(f"Error checking GameStop: {e}")
|
||||||
send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
|
send_error_notification(f"Error checking GameStop: {str(e)}", "GameStop")
|
||||||
|
# Log failed check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("gamestop", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
|
||||||
|
|
||||||
|
|
||||||
def check_bestbuy():
|
def check_bestbuy():
|
||||||
"""Check Best Buy for restocks and new drops"""
|
"""Check Best Buy for restocks and new drops"""
|
||||||
logger.info("Checking Best Buy...")
|
logger.info("Checking Best Buy...")
|
||||||
scraper = scrapers["bestbuy"]
|
scraper = scrapers["bestbuy"]
|
||||||
|
start_time = time.time()
|
||||||
|
total_products = 0
|
||||||
|
total_new = 0
|
||||||
|
total_restocks = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for url in BESTBUY_URLS:
|
for url in BESTBUY_URLS:
|
||||||
@@ -255,8 +259,12 @@ def check_bestbuy():
|
|||||||
products = scraper.filter_by_keywords(products, KEYWORDS)
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
||||||
logger.info(f"After keyword filter: {len(products)} products")
|
logger.info(f"After keyword filter: {len(products)} products")
|
||||||
|
|
||||||
|
total_products += len(products)
|
||||||
|
|
||||||
# Process and detect changes
|
# Process and detect changes
|
||||||
new_products, restocked_products = tracker.process_products(products)
|
new_products, restocked_products, price_changed_products = tracker.process_products(products)
|
||||||
|
total_new += len(new_products)
|
||||||
|
total_restocks += len(restocked_products)
|
||||||
|
|
||||||
# Send notifications for new products (if enabled)
|
# Send notifications for new products (if enabled)
|
||||||
if NEW_DROP_NOTIFICATIONS:
|
if NEW_DROP_NOTIFICATIONS:
|
||||||
@@ -284,6 +292,20 @@ def check_bestbuy():
|
|||||||
)
|
)
|
||||||
logger.info(f"Sent notification for restock: {product.name}")
|
logger.info(f"Sent notification for restock: {product.name}")
|
||||||
|
|
||||||
|
# Send notifications for price changes (if enabled)
|
||||||
|
if PRICE_CHANGE_NOTIFICATIONS:
|
||||||
|
for product, old_price, new_price in price_changed_products:
|
||||||
|
send_price_change_alert(
|
||||||
|
product_name=product.name,
|
||||||
|
product_url=product.url,
|
||||||
|
old_price=old_price,
|
||||||
|
new_price=new_price,
|
||||||
|
site="bestbuy",
|
||||||
|
in_stock=product.in_stock,
|
||||||
|
image_url=product.image_url,
|
||||||
|
)
|
||||||
|
logger.info(f"Sent notification for price change: {product.name} ({old_price} -> {new_price})")
|
||||||
|
|
||||||
# Log stats
|
# Log stats
|
||||||
stats = tracker.get_stats()
|
stats = tracker.get_stats()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -292,15 +314,26 @@ def check_bestbuy():
|
|||||||
f"{stats['out_of_stock']} out of stock"
|
f"{stats['out_of_stock']} out of stock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Log successful check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("bestbuy", total_products, total_new, total_restocks, duration_ms, success=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking Best Buy: {e}")
|
logger.error(f"Error checking Best Buy: {e}")
|
||||||
send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
|
send_error_notification(f"Error checking Best Buy: {str(e)}", "Best Buy")
|
||||||
|
# Log failed check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("bestbuy", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
|
||||||
|
|
||||||
|
|
||||||
def check_walmart():
|
def check_walmart():
|
||||||
"""Check Walmart for restocks and new drops"""
|
"""Check Walmart for restocks and new drops"""
|
||||||
logger.info("Checking Walmart...")
|
logger.info("Checking Walmart...")
|
||||||
scraper = scrapers["walmart"]
|
scraper = scrapers["walmart"]
|
||||||
|
start_time = time.time()
|
||||||
|
total_products = 0
|
||||||
|
total_new = 0
|
||||||
|
total_restocks = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for url in WALMART_URLS:
|
for url in WALMART_URLS:
|
||||||
@@ -315,8 +348,12 @@ def check_walmart():
|
|||||||
products = scraper.filter_by_keywords(products, KEYWORDS)
|
products = scraper.filter_by_keywords(products, KEYWORDS)
|
||||||
logger.info(f"After keyword filter: {len(products)} products")
|
logger.info(f"After keyword filter: {len(products)} products")
|
||||||
|
|
||||||
|
total_products += len(products)
|
||||||
|
|
||||||
# Process and detect changes
|
# Process and detect changes
|
||||||
new_products, restocked_products = tracker.process_products(products)
|
new_products, restocked_products, price_changed_products = tracker.process_products(products)
|
||||||
|
total_new += len(new_products)
|
||||||
|
total_restocks += len(restocked_products)
|
||||||
|
|
||||||
# Send notifications for new products (if enabled)
|
# Send notifications for new products (if enabled)
|
||||||
if NEW_DROP_NOTIFICATIONS:
|
if NEW_DROP_NOTIFICATIONS:
|
||||||
@@ -344,6 +381,20 @@ def check_walmart():
|
|||||||
)
|
)
|
||||||
logger.info(f"Sent notification for restock: {product.name}")
|
logger.info(f"Sent notification for restock: {product.name}")
|
||||||
|
|
||||||
|
# Send notifications for price changes (if enabled)
|
||||||
|
if PRICE_CHANGE_NOTIFICATIONS:
|
||||||
|
for product, old_price, new_price in price_changed_products:
|
||||||
|
send_price_change_alert(
|
||||||
|
product_name=product.name,
|
||||||
|
product_url=product.url,
|
||||||
|
old_price=old_price,
|
||||||
|
new_price=new_price,
|
||||||
|
site="walmart",
|
||||||
|
in_stock=product.in_stock,
|
||||||
|
image_url=product.image_url,
|
||||||
|
)
|
||||||
|
logger.info(f"Sent notification for price change: {product.name} ({old_price} -> {new_price})")
|
||||||
|
|
||||||
# Log stats
|
# Log stats
|
||||||
stats = tracker.get_stats()
|
stats = tracker.get_stats()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -352,17 +403,23 @@ def check_walmart():
|
|||||||
f"{stats['out_of_stock']} out of stock"
|
f"{stats['out_of_stock']} out of stock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Log successful check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("walmart", total_products, total_new, total_restocks, duration_ms, success=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking Walmart: {e}")
|
logger.error(f"Error checking Walmart: {e}")
|
||||||
send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
|
send_error_notification(f"Error checking Walmart: {str(e)}", "Walmart")
|
||||||
|
# Log failed check to database
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
tracker.log_check("walmart", total_products, total_new, total_restocks, duration_ms, success=False, error_message=str(e))
|
||||||
|
|
||||||
|
|
||||||
def run_checks():
|
def run_checks():
|
||||||
"""Run all enabled site checks"""
|
"""Run all enabled site checks"""
|
||||||
logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
|
logger.info(f"Running checks at {datetime.now().strftime('%H:%M:%S')}")
|
||||||
|
|
||||||
if "pokemoncenter" in SITES_ENABLED:
|
# Note: Pokemon Center is monitored via Chrome Extension, not here
|
||||||
check_pokemoncenter()
|
|
||||||
|
|
||||||
if "target" in SITES_ENABLED:
|
if "target" in SITES_ENABLED:
|
||||||
check_target()
|
check_target()
|
||||||
|
|||||||
@@ -1,12 +1,11 @@
|
|||||||
# Scrapers package
|
# Scrapers package
|
||||||
from .pokemoncenter import PokemonCenterScraper
|
# Note: Pokemon Center uses Chrome Extension (chrome-extension/), not a Python scraper
|
||||||
from .target import TargetScraper
|
from .target import TargetScraper
|
||||||
from .gamestop import GameStopScraper, warmup_gamestop
|
from .gamestop import GameStopScraper, warmup_gamestop
|
||||||
from .bestbuy import BestBuyScraper
|
from .bestbuy import BestBuyScraper
|
||||||
from .walmart import WalmartScraper
|
from .walmart import WalmartScraper
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"PokemonCenterScraper",
|
|
||||||
"TargetScraper",
|
"TargetScraper",
|
||||||
"GameStopScraper",
|
"GameStopScraper",
|
||||||
"BestBuyScraper",
|
"BestBuyScraper",
|
||||||
|
|||||||
+97
-5
@@ -27,15 +27,55 @@ class BestBuyScraper(BaseScraper):
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._stealth_browser = None
|
self._stealth_browser = None
|
||||||
|
self._restart_attempts = 0
|
||||||
|
self._max_restart_attempts = 3
|
||||||
|
|
||||||
def _get_stealth_browser(self):
|
def _get_stealth_browser(self):
|
||||||
"""Get or create stealth browser for Best Buy"""
|
"""Get or create stealth browser for Best Buy, with auto-restart on session errors"""
|
||||||
if self._stealth_browser is None:
|
|
||||||
from tools.stealth_browser import StealthBrowser
|
from tools.stealth_browser import StealthBrowser
|
||||||
|
|
||||||
|
# Check if we need to create a new browser
|
||||||
|
if self._stealth_browser is None:
|
||||||
|
logger.info("Creating new Best Buy stealth browser...")
|
||||||
self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
|
self._stealth_browser = StealthBrowser(headless=False, session_name="bestbuy")
|
||||||
self._stealth_browser.start()
|
self._stealth_browser.start()
|
||||||
|
self._restart_attempts = 0
|
||||||
return self._stealth_browser
|
return self._stealth_browser
|
||||||
|
|
||||||
|
# Check if existing session is still valid
|
||||||
|
if not self._stealth_browser.is_session_valid():
|
||||||
|
if self._restart_attempts >= self._max_restart_attempts:
|
||||||
|
logger.error(f"Best Buy browser failed after {self._max_restart_attempts} restart attempts")
|
||||||
|
# Reset counter and try one more time after clearing
|
||||||
|
self._restart_attempts = 0
|
||||||
|
self._stealth_browser = None
|
||||||
|
return self._get_stealth_browser()
|
||||||
|
|
||||||
|
logger.warning("Best Buy browser session invalid, restarting...")
|
||||||
|
self._restart_attempts += 1
|
||||||
|
self._stealth_browser.restart()
|
||||||
|
|
||||||
|
return self._stealth_browser
|
||||||
|
|
||||||
|
def _handle_session_error(self, error: Exception) -> bool:
|
||||||
|
"""
|
||||||
|
Check if error is a session error and handle it.
|
||||||
|
Returns True if browser was restarted and operation should be retried.
|
||||||
|
"""
|
||||||
|
error_msg = str(error).lower()
|
||||||
|
session_errors = ["invalid session id", "session deleted", "no such session", "browser has closed"]
|
||||||
|
|
||||||
|
if any(err in error_msg for err in session_errors):
|
||||||
|
logger.warning(f"Session error detected: {error}")
|
||||||
|
if self._stealth_browser and self._restart_attempts < self._max_restart_attempts:
|
||||||
|
self._restart_attempts += 1
|
||||||
|
try:
|
||||||
|
self._stealth_browser.restart()
|
||||||
|
return True # Retry operation
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to restart browser: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||||
"""
|
"""
|
||||||
Scrape a Best Buy search/category page for all products
|
Scrape a Best Buy search/category page for all products
|
||||||
@@ -65,7 +105,11 @@ class BestBuyScraper(BaseScraper):
|
|||||||
for page_num in range(1, max_pages + 1):
|
for page_num in range(1, max_pages + 1):
|
||||||
products = []
|
products = []
|
||||||
page_url = url if page_num == 1 else f"{url}&cp={page_num}"
|
page_url = url if page_num == 1 else f"{url}&cp={page_num}"
|
||||||
|
retry_count = 0
|
||||||
|
max_retries = 2
|
||||||
|
html = None
|
||||||
|
|
||||||
|
while retry_count <= max_retries:
|
||||||
try:
|
try:
|
||||||
logger.info(f"Navigating to: {page_url}")
|
logger.info(f"Navigating to: {page_url}")
|
||||||
browser.driver.get(page_url)
|
browser.driver.get(page_url)
|
||||||
@@ -99,6 +143,25 @@ class BestBuyScraper(BaseScraper):
|
|||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
break # Success, exit retry loop
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
if self._handle_session_error(e) and retry_count < max_retries:
|
||||||
|
retry_count += 1
|
||||||
|
logger.info(f"Retrying after browser restart (attempt {retry_count}/{max_retries})")
|
||||||
|
browser = self._get_stealth_browser() # Get restarted browser
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
logger.error(f"Error navigating to {page_url}: {e}")
|
||||||
|
return all_products
|
||||||
|
else:
|
||||||
|
# while loop completed without break (all retries exhausted)
|
||||||
|
if html is None:
|
||||||
|
logger.error(f"Failed to load page after {max_retries} retries")
|
||||||
|
return all_products
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
|
||||||
# Try to find product data in JSON scripts
|
# Try to find product data in JSON scripts
|
||||||
@@ -272,11 +335,17 @@ class BestBuyScraper(BaseScraper):
|
|||||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
||||||
price = price_match.group() if price_match else None
|
price = price_match.group() if price_match else None
|
||||||
|
|
||||||
|
# Check stock status from element text
|
||||||
|
item_text = item.get_text().lower()
|
||||||
|
in_stock = True
|
||||||
|
if any(phrase in item_text for phrase in ["sold out", "out of stock", "unavailable", "coming soon"]):
|
||||||
|
in_stock = False
|
||||||
|
|
||||||
products.append(Product(
|
products.append(Product(
|
||||||
name=name,
|
name=name,
|
||||||
url=f"{self.base_url}/site/{sku_id}.p",
|
url=f"{self.base_url}/site/{sku_id}.p",
|
||||||
price=price,
|
price=price,
|
||||||
in_stock=True,
|
in_stock=in_stock,
|
||||||
image_url=None,
|
image_url=None,
|
||||||
site=self.site_name,
|
site=self.site_name,
|
||||||
product_id=sku_id,
|
product_id=sku_id,
|
||||||
@@ -359,12 +428,28 @@ class BestBuyScraper(BaseScraper):
|
|||||||
if image_matches:
|
if image_matches:
|
||||||
image_url = image_matches[0]
|
image_url = image_matches[0]
|
||||||
|
|
||||||
|
# Try to determine stock status from Apollo data context
|
||||||
|
in_stock = True # Default to True if no stock info found
|
||||||
|
if url_pos > 0:
|
||||||
|
# Look for availability info near this product's URL
|
||||||
|
context_start = max(0, url_pos - 3000)
|
||||||
|
context_end = min(len(script_content), url_pos + 500)
|
||||||
|
product_context = script_content[context_start:context_end]
|
||||||
|
|
||||||
|
# Check for explicit out of stock indicators
|
||||||
|
if '"isAvailable":false' in product_context or '"available":false' in product_context:
|
||||||
|
in_stock = False
|
||||||
|
elif '"soldOut":true' in product_context or '"outOfStock":true' in product_context:
|
||||||
|
in_stock = False
|
||||||
|
elif 'sold out' in product_context.lower() or 'out of stock' in product_context.lower():
|
||||||
|
in_stock = False
|
||||||
|
|
||||||
if name and len(name) > 5:
|
if name and len(name) > 5:
|
||||||
products.append(Product(
|
products.append(Product(
|
||||||
name=name,
|
name=name,
|
||||||
url=url,
|
url=url,
|
||||||
price=price,
|
price=price,
|
||||||
in_stock=True, # Assume in stock if listed
|
in_stock=in_stock,
|
||||||
image_url=image_url,
|
image_url=image_url,
|
||||||
site=self.site_name,
|
site=self.site_name,
|
||||||
product_id=sku_id,
|
product_id=sku_id,
|
||||||
@@ -512,7 +597,14 @@ class BestBuyScraper(BaseScraper):
|
|||||||
if "availability" in data:
|
if "availability" in data:
|
||||||
availability = data["availability"]
|
availability = data["availability"]
|
||||||
if isinstance(availability, dict):
|
if isinstance(availability, dict):
|
||||||
in_stock = availability.get("isAvailable", True) or availability.get("available", True)
|
# Check isAvailable first, then available - don't use 'or' which short-circuits incorrectly
|
||||||
|
is_available = availability.get("isAvailable")
|
||||||
|
available = availability.get("available")
|
||||||
|
if is_available is not None:
|
||||||
|
in_stock = bool(is_available)
|
||||||
|
elif available is not None:
|
||||||
|
in_stock = bool(available)
|
||||||
|
# else keep default True
|
||||||
elif isinstance(availability, bool):
|
elif isinstance(availability, bool):
|
||||||
in_stock = availability
|
in_stock = availability
|
||||||
elif isinstance(availability, str):
|
elif isinstance(availability, str):
|
||||||
|
|||||||
+61
-2
@@ -107,15 +107,55 @@ class GameStopScraper(BaseScraper):
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._stealth_browser = None
|
self._stealth_browser = None
|
||||||
|
self._restart_attempts = 0
|
||||||
|
self._max_restart_attempts = 3
|
||||||
|
|
||||||
def _get_stealth_browser(self):
|
def _get_stealth_browser(self):
|
||||||
"""Get or create stealth browser for GameStop"""
|
"""Get or create stealth browser for GameStop, with auto-restart on session errors"""
|
||||||
if self._stealth_browser is None:
|
|
||||||
from tools.stealth_browser import StealthBrowser
|
from tools.stealth_browser import StealthBrowser
|
||||||
|
|
||||||
|
# Check if we need to create a new browser
|
||||||
|
if self._stealth_browser is None:
|
||||||
|
logger.info("Creating new GameStop stealth browser...")
|
||||||
self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
|
self._stealth_browser = StealthBrowser(headless=False, session_name="gamestop")
|
||||||
self._stealth_browser.start()
|
self._stealth_browser.start()
|
||||||
|
self._restart_attempts = 0
|
||||||
return self._stealth_browser
|
return self._stealth_browser
|
||||||
|
|
||||||
|
# Check if existing session is still valid
|
||||||
|
if not self._stealth_browser.is_session_valid():
|
||||||
|
if self._restart_attempts >= self._max_restart_attempts:
|
||||||
|
logger.error(f"GameStop browser failed after {self._max_restart_attempts} restart attempts")
|
||||||
|
# Reset counter and try one more time after clearing
|
||||||
|
self._restart_attempts = 0
|
||||||
|
self._stealth_browser = None
|
||||||
|
return self._get_stealth_browser()
|
||||||
|
|
||||||
|
logger.warning("GameStop browser session invalid, restarting...")
|
||||||
|
self._restart_attempts += 1
|
||||||
|
self._stealth_browser.restart()
|
||||||
|
|
||||||
|
return self._stealth_browser
|
||||||
|
|
||||||
|
def _handle_session_error(self, error: Exception) -> bool:
|
||||||
|
"""
|
||||||
|
Check if error is a session error and handle it.
|
||||||
|
Returns True if browser was restarted and operation should be retried.
|
||||||
|
"""
|
||||||
|
error_msg = str(error).lower()
|
||||||
|
session_errors = ["invalid session id", "session deleted", "no such session", "browser has closed"]
|
||||||
|
|
||||||
|
if any(err in error_msg for err in session_errors):
|
||||||
|
logger.warning(f"Session error detected: {error}")
|
||||||
|
if self._stealth_browser and self._restart_attempts < self._max_restart_attempts:
|
||||||
|
self._restart_attempts += 1
|
||||||
|
try:
|
||||||
|
self._stealth_browser.restart()
|
||||||
|
return True # Retry operation
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to restart browser: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
||||||
"""
|
"""
|
||||||
Scrape a GameStop search/category page for all products
|
Scrape a GameStop search/category page for all products
|
||||||
@@ -145,14 +185,33 @@ class GameStopScraper(BaseScraper):
|
|||||||
for page_num in range(1, max_pages + 1):
|
for page_num in range(1, max_pages + 1):
|
||||||
products = []
|
products = []
|
||||||
page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}"
|
page_url = url if page_num == 1 else f"{url}&start={24 * (page_num - 1)}"
|
||||||
|
retry_count = 0
|
||||||
|
max_retries = 2
|
||||||
|
|
||||||
|
while retry_count <= max_retries:
|
||||||
try:
|
try:
|
||||||
logger.info(f"Navigating to: {page_url}")
|
logger.info(f"Navigating to: {page_url}")
|
||||||
browser.driver.get(page_url)
|
browser.driver.get(page_url)
|
||||||
time.sleep(5) # Wait for page load
|
time.sleep(5) # Wait for page load
|
||||||
|
|
||||||
html = browser.driver.page_source
|
html = browser.driver.page_source
|
||||||
|
break # Success, exit retry loop
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
if self._handle_session_error(e) and retry_count < max_retries:
|
||||||
|
retry_count += 1
|
||||||
|
logger.info(f"Retrying after browser restart (attempt {retry_count}/{max_retries})")
|
||||||
|
browser = self._get_stealth_browser() # Get restarted browser
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
logger.error(f"Error navigating to {page_url}: {e}")
|
||||||
|
return all_products
|
||||||
|
else:
|
||||||
|
# while loop completed without break (all retries exhausted)
|
||||||
|
logger.error(f"Failed to load page after {max_retries} retries")
|
||||||
|
return all_products
|
||||||
|
|
||||||
|
try:
|
||||||
# Check for Cloudflare challenge
|
# Check for Cloudflare challenge
|
||||||
html_lower = html.lower()
|
html_lower = html.lower()
|
||||||
title = browser.driver.title.lower()
|
title = browser.driver.title.lower()
|
||||||
|
|||||||
@@ -1,334 +0,0 @@
|
|||||||
"""
|
|
||||||
PokemonCenter.com scraper
|
|
||||||
Handles bot protection with undetected-chromedriver stealth browser
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
|
||||||
import logging
|
|
||||||
from typing import List, Optional
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
from .base import BaseScraper, Product
|
|
||||||
import config
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def get_stealth_or_regular_browser():
|
|
||||||
"""Get appropriate browser based on config"""
|
|
||||||
if config.USE_STEALTH_BROWSER:
|
|
||||||
from tools.stealth_browser import get_stealth_browser
|
|
||||||
return get_stealth_browser(
|
|
||||||
headless=config.STEALTH_HEADLESS,
|
|
||||||
session_name=config.STEALTH_SESSION_NAME
|
|
||||||
), True
|
|
||||||
else:
|
|
||||||
from src.browser import get_browser
|
|
||||||
return get_browser(), False
|
|
||||||
|
|
||||||
|
|
||||||
class PokemonCenterScraper(BaseScraper):
|
|
||||||
"""Scraper for PokemonCenter.com"""
|
|
||||||
|
|
||||||
site_name = "pokemoncenter"
|
|
||||||
base_url = "https://www.pokemoncenter.com"
|
|
||||||
|
|
||||||
def scrape_category_page(self, url: str, max_pages: int = 1) -> List[Product]:
|
|
||||||
"""
|
|
||||||
Scrape a PokemonCenter category page for all products
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: Category page URL
|
|
||||||
max_pages: Maximum pages to scrape (not yet implemented for PokemonCenter)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of Product objects
|
|
||||||
"""
|
|
||||||
# TODO: Implement pagination for PokemonCenter when needed
|
|
||||||
browser, is_stealth = get_stealth_or_regular_browser()
|
|
||||||
products = []
|
|
||||||
|
|
||||||
try:
|
|
||||||
if is_stealth:
|
|
||||||
# Use stealth browser (undetected-chromedriver)
|
|
||||||
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
|
|
||||||
|
|
||||||
# Check for CAPTCHA
|
|
||||||
if browser.check_for_captcha():
|
|
||||||
logger.warning("CAPTCHA detected on PokemonCenter!")
|
|
||||||
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
|
|
||||||
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
|
|
||||||
# Re-fetch page after CAPTCHA solve
|
|
||||||
html = browser.get_page(url, wait_time=config.HUMAN_DELAY_MAX)
|
|
||||||
else:
|
|
||||||
logger.error("Cannot solve CAPTCHA in headless mode")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Save screenshot for debugging
|
|
||||||
try:
|
|
||||||
browser.screenshot("debug_screenshot.png")
|
|
||||||
logger.info("Saved debug screenshot to debug_screenshot.png")
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
page = None # Stealth browser doesn't return page object
|
|
||||||
else:
|
|
||||||
# Use regular Playwright browser
|
|
||||||
page, html = browser.get_page_content(
|
|
||||||
url,
|
|
||||||
wait_for_selector=None, # Let it use networkidle instead
|
|
||||||
timeout=60000,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Save screenshot for debugging if needed
|
|
||||||
try:
|
|
||||||
page.screenshot(path="debug_screenshot.png")
|
|
||||||
logger.info("Saved debug screenshot to debug_screenshot.png")
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
|
||||||
|
|
||||||
# Try multiple selectors for product cards
|
|
||||||
# PokemonCenter may use different structures
|
|
||||||
product_cards = (
|
|
||||||
soup.select("[data-testid='product-card']")
|
|
||||||
or soup.select(".product-card")
|
|
||||||
or soup.select(".product-tile")
|
|
||||||
or soup.select("article[data-product-id]")
|
|
||||||
or soup.select(".product-grid-item")
|
|
||||||
or soup.select("[class*='ProductCard']")
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info(f"Found {len(product_cards)} product cards")
|
|
||||||
|
|
||||||
# If no product cards found, try a more generic approach
|
|
||||||
if not product_cards:
|
|
||||||
# Look for any links that look like product pages
|
|
||||||
product_links = soup.select("a[href*='/product/']")
|
|
||||||
logger.info(f"Fallback: Found {len(product_links)} product links")
|
|
||||||
|
|
||||||
for link in product_links:
|
|
||||||
href = link.get("href", "")
|
|
||||||
if href and "/product/" in href:
|
|
||||||
full_url = href if href.startswith("http") else f"{self.base_url}{href}"
|
|
||||||
|
|
||||||
# Try to get product name from link text or nearby elements
|
|
||||||
name = link.get_text(strip=True)
|
|
||||||
if not name or len(name) < 3:
|
|
||||||
name = link.get("title", "") or link.get("aria-label", "") or "Unknown Product"
|
|
||||||
|
|
||||||
product = Product(
|
|
||||||
name=name,
|
|
||||||
url=full_url,
|
|
||||||
price=None,
|
|
||||||
in_stock=True, # Assume in stock if listed, will verify later
|
|
||||||
image_url=None,
|
|
||||||
site=self.site_name,
|
|
||||||
product_id=self._extract_product_id(full_url),
|
|
||||||
)
|
|
||||||
products.append(product)
|
|
||||||
|
|
||||||
else:
|
|
||||||
for card in product_cards:
|
|
||||||
product = self._parse_product_card(card)
|
|
||||||
if product:
|
|
||||||
products.append(product)
|
|
||||||
|
|
||||||
# Close page only for Playwright browser
|
|
||||||
if page is not None:
|
|
||||||
page.close()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error scraping PokemonCenter category page: {e}")
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Remove duplicates
|
|
||||||
seen_urls = set()
|
|
||||||
unique_products = []
|
|
||||||
for p in products:
|
|
||||||
if p.url not in seen_urls:
|
|
||||||
seen_urls.add(p.url)
|
|
||||||
unique_products.append(p)
|
|
||||||
|
|
||||||
logger.info(f"Scraped {len(unique_products)} unique products from PokemonCenter")
|
|
||||||
return unique_products
|
|
||||||
|
|
||||||
def _parse_product_card(self, card) -> Optional[Product]:
|
|
||||||
"""Parse a product card element into a Product object"""
|
|
||||||
try:
|
|
||||||
# Try to find product link
|
|
||||||
link = card.select_one("a[href*='/product/']") or card.select_one("a")
|
|
||||||
if not link:
|
|
||||||
return None
|
|
||||||
|
|
||||||
href = link.get("href", "")
|
|
||||||
if not href:
|
|
||||||
return None
|
|
||||||
|
|
||||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
|
||||||
|
|
||||||
# Get product name
|
|
||||||
name_elem = (
|
|
||||||
card.select_one("[data-testid='product-name']")
|
|
||||||
or card.select_one(".product-name")
|
|
||||||
or card.select_one("h2")
|
|
||||||
or card.select_one("h3")
|
|
||||||
or card.select_one("[class*='title']")
|
|
||||||
or card.select_one("[class*='name']")
|
|
||||||
)
|
|
||||||
name = name_elem.get_text(strip=True) if name_elem else link.get_text(strip=True)
|
|
||||||
|
|
||||||
if not name or len(name) < 3:
|
|
||||||
name = link.get("title", "") or link.get("aria-label", "") or "Unknown Product"
|
|
||||||
|
|
||||||
# Get price
|
|
||||||
price_elem = (
|
|
||||||
card.select_one("[data-testid='product-price']")
|
|
||||||
or card.select_one(".product-price")
|
|
||||||
or card.select_one("[class*='price']")
|
|
||||||
or card.select_one("span:contains('$')")
|
|
||||||
)
|
|
||||||
price = None
|
|
||||||
if price_elem:
|
|
||||||
price_text = price_elem.get_text(strip=True)
|
|
||||||
# Extract price with regex
|
|
||||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_text)
|
|
||||||
if price_match:
|
|
||||||
price = price_match.group()
|
|
||||||
|
|
||||||
# Check stock status
|
|
||||||
in_stock = self._check_card_stock_status(card)
|
|
||||||
|
|
||||||
# Get image URL
|
|
||||||
img = card.select_one("img")
|
|
||||||
image_url = None
|
|
||||||
if img:
|
|
||||||
image_url = img.get("src") or img.get("data-src")
|
|
||||||
if image_url and not image_url.startswith("http"):
|
|
||||||
image_url = f"{self.base_url}{image_url}"
|
|
||||||
|
|
||||||
return Product(
|
|
||||||
name=name,
|
|
||||||
url=url,
|
|
||||||
price=price,
|
|
||||||
in_stock=in_stock,
|
|
||||||
image_url=image_url,
|
|
||||||
site=self.site_name,
|
|
||||||
product_id=self._extract_product_id(url),
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"Error parsing product card: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _check_card_stock_status(self, card) -> bool:
|
|
||||||
"""Check if a product card indicates in-stock status"""
|
|
||||||
card_text = card.get_text(lower=True) if hasattr(card, "get_text") else str(card).lower()
|
|
||||||
|
|
||||||
# Out of stock indicators
|
|
||||||
out_of_stock_phrases = [
|
|
||||||
"sold out",
|
|
||||||
"out of stock",
|
|
||||||
"unavailable",
|
|
||||||
"coming soon",
|
|
||||||
"notify me",
|
|
||||||
]
|
|
||||||
|
|
||||||
for phrase in out_of_stock_phrases:
|
|
||||||
if phrase in card_text:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# In stock indicators
|
|
||||||
in_stock_phrases = [
|
|
||||||
"add to cart",
|
|
||||||
"add to bag",
|
|
||||||
"buy now",
|
|
||||||
"in stock",
|
|
||||||
"available",
|
|
||||||
]
|
|
||||||
|
|
||||||
for phrase in in_stock_phrases:
|
|
||||||
if phrase in card_text:
|
|
||||||
return True
|
|
||||||
|
|
||||||
# If we can't determine, assume it might be in stock (will verify on product page)
|
|
||||||
return True
|
|
||||||
|
|
||||||
def check_product_stock(self, product_url: str) -> tuple[bool, Optional[str]]:
|
|
||||||
"""
|
|
||||||
Check if a specific product is in stock by visiting its page
|
|
||||||
|
|
||||||
Args:
|
|
||||||
product_url: URL of the product page
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (is_in_stock, price)
|
|
||||||
"""
|
|
||||||
browser, is_stealth = get_stealth_or_regular_browser()
|
|
||||||
|
|
||||||
try:
|
|
||||||
if is_stealth:
|
|
||||||
# Use stealth browser
|
|
||||||
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
|
|
||||||
|
|
||||||
# Check for CAPTCHA
|
|
||||||
if browser.check_for_captcha():
|
|
||||||
logger.warning("CAPTCHA detected on product page!")
|
|
||||||
if config.PAUSE_ON_CAPTCHA and not config.STEALTH_HEADLESS:
|
|
||||||
browser.wait_for_captcha_solve(config.CAPTCHA_WAIT_TIMEOUT)
|
|
||||||
html = browser.get_page(product_url, wait_time=config.HUMAN_DELAY_MAX)
|
|
||||||
else:
|
|
||||||
return False, None
|
|
||||||
|
|
||||||
page = None
|
|
||||||
else:
|
|
||||||
# Use regular Playwright browser
|
|
||||||
page, html = browser.get_page_content(
|
|
||||||
product_url,
|
|
||||||
wait_for_selector="button, [data-testid]",
|
|
||||||
timeout=30000,
|
|
||||||
)
|
|
||||||
|
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
|
||||||
page_text = soup.get_text().lower()
|
|
||||||
|
|
||||||
# Check for out of stock indicators
|
|
||||||
out_of_stock = any(
|
|
||||||
phrase in page_text
|
|
||||||
for phrase in ["sold out", "out of stock", "unavailable", "notify me when available"]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check for add to cart button
|
|
||||||
add_to_cart = soup.select_one(
|
|
||||||
"button:contains('Add to Cart'), button:contains('Add to Bag'), [data-testid='add-to-cart']"
|
|
||||||
)
|
|
||||||
|
|
||||||
in_stock = not out_of_stock and add_to_cart is not None
|
|
||||||
|
|
||||||
# Get price
|
|
||||||
price = None
|
|
||||||
price_elem = soup.select_one("[data-testid='product-price'], .product-price, [class*='price']")
|
|
||||||
if price_elem:
|
|
||||||
price_match = re.search(r"\$[\d,]+\.?\d*", price_elem.get_text())
|
|
||||||
if price_match:
|
|
||||||
price = price_match.group()
|
|
||||||
|
|
||||||
if page is not None:
|
|
||||||
page.close()
|
|
||||||
return in_stock, price
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error checking product stock: {e}")
|
|
||||||
return False, None
|
|
||||||
|
|
||||||
def _extract_product_id(self, url: str) -> str:
|
|
||||||
"""Extract product ID from URL"""
|
|
||||||
# PokemonCenter URLs typically look like:
|
|
||||||
# https://www.pokemoncenter.com/product/123456/product-name
|
|
||||||
match = re.search(r"/product/(\d+)", url)
|
|
||||||
if match:
|
|
||||||
return match.group(1)
|
|
||||||
|
|
||||||
# Fallback: use URL path as ID
|
|
||||||
return url.split("/")[-1]
|
|
||||||
+51
-14
@@ -8,8 +8,9 @@ import logging
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Set, Optional, Tuple
|
from typing import Dict, List, Set, Optional, Tuple
|
||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import config
|
||||||
from scrapers.base import Product
|
from scrapers.base import Product
|
||||||
from .database import get_database
|
from .database import get_database
|
||||||
from .favorites import get_favorites_manager
|
from .favorites import get_favorites_manager
|
||||||
@@ -53,7 +54,7 @@ class ProductTracker:
|
|||||||
except IOError as e:
|
except IOError as e:
|
||||||
logger.error(f"Error saving products file: {e}")
|
logger.error(f"Error saving products file: {e}")
|
||||||
|
|
||||||
def process_products(self, products: List[Product]) -> Tuple[List[Product], List[Product]]:
|
def process_products(self, products: List[Product]) -> Tuple[List[Product], List[Product], List[Tuple[Product, str, str]]]:
|
||||||
"""
|
"""
|
||||||
Process a list of scraped products and detect changes.
|
Process a list of scraped products and detect changes.
|
||||||
Also logs events to database for stats tracking.
|
Also logs events to database for stats tracking.
|
||||||
@@ -62,10 +63,12 @@ class ProductTracker:
|
|||||||
products: List of products from scraper
|
products: List of products from scraper
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (new_products, restocked_products)
|
Tuple of (new_products, restocked_products, price_changed_products)
|
||||||
|
price_changed_products is a list of tuples: (product, old_price, new_price)
|
||||||
"""
|
"""
|
||||||
new_products = []
|
new_products = []
|
||||||
restocked_products = []
|
restocked_products = []
|
||||||
|
price_changed_products = []
|
||||||
|
|
||||||
for product in products:
|
for product in products:
|
||||||
url = product.url
|
url = product.url
|
||||||
@@ -120,22 +123,49 @@ class ProductTracker:
|
|||||||
existing["image_url"] = product.image_url or existing.get("image_url")
|
existing["image_url"] = product.image_url or existing.get("image_url")
|
||||||
existing["db_id"] = db_product_id
|
existing["db_id"] = db_product_id
|
||||||
|
|
||||||
# Check for price change
|
# Check for price change (only if both prices exist and are different)
|
||||||
if product.price and product.price != old_price:
|
if product.price and old_price and product.price != old_price:
|
||||||
|
# Record price change for notification
|
||||||
|
price_changed_products.append((product, old_price, product.price))
|
||||||
|
existing["price"] = product.price
|
||||||
|
self.db.record_price(db_product_id, product.price)
|
||||||
|
self.db.record_stock_event(db_product_id, "price_change")
|
||||||
|
logger.info(f"PRICE CHANGE: {product.name}: {old_price} -> {product.price}")
|
||||||
|
elif product.price and not old_price:
|
||||||
|
# First time we're seeing a price - just update, don't notify
|
||||||
existing["price"] = product.price
|
existing["price"] = product.price
|
||||||
self.db.record_price(db_product_id, product.price)
|
self.db.record_price(db_product_id, product.price)
|
||||||
logger.debug(f"Price change for {product.name}: {old_price} -> {product.price}")
|
|
||||||
|
|
||||||
# Check for restock
|
# Check for restock
|
||||||
if product.in_stock and not was_in_stock:
|
if product.in_stock and not was_in_stock:
|
||||||
# RESTOCK!
|
# Check restock cooldown to prevent duplicate notifications
|
||||||
|
last_restock_notified = existing.get("last_restock_notified")
|
||||||
|
cooldown_hours = getattr(config, 'RESTOCK_COOLDOWN_HOURS', 6)
|
||||||
|
cooldown_expired = True
|
||||||
|
|
||||||
|
if last_restock_notified:
|
||||||
|
try:
|
||||||
|
last_notified_time = datetime.fromisoformat(last_restock_notified)
|
||||||
|
cooldown_threshold = datetime.now() - timedelta(hours=cooldown_hours)
|
||||||
|
cooldown_expired = last_notified_time < cooldown_threshold
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
cooldown_expired = True # Invalid timestamp, allow notification
|
||||||
|
|
||||||
|
if cooldown_expired:
|
||||||
|
# RESTOCK! (outside cooldown period)
|
||||||
restocked_products.append(product)
|
restocked_products.append(product)
|
||||||
existing["last_in_stock"] = now
|
existing["last_restock_notified"] = now
|
||||||
logger.info(f"RESTOCK: {product.name}")
|
logger.info(f"RESTOCK: {product.name}")
|
||||||
|
|
||||||
# Log restock event to database
|
# Log restock event to database
|
||||||
self.db.record_stock_event(db_product_id, "restock")
|
self.db.record_stock_event(db_product_id, "restock")
|
||||||
self.db.record_stock_event(db_product_id, "in_stock")
|
self.db.record_stock_event(db_product_id, "in_stock")
|
||||||
|
else:
|
||||||
|
# Within cooldown period - skip notification but still update stock status
|
||||||
|
logger.debug(f"Restock detected for {product.name} but within {cooldown_hours}h cooldown (last notified: {last_restock_notified})")
|
||||||
|
self.db.record_stock_event(db_product_id, "in_stock")
|
||||||
|
|
||||||
|
existing["last_in_stock"] = now
|
||||||
|
|
||||||
# Check for out of stock
|
# Check for out of stock
|
||||||
elif not product.in_stock and was_in_stock:
|
elif not product.in_stock and was_in_stock:
|
||||||
@@ -147,17 +177,18 @@ class ProductTracker:
|
|||||||
self.products[url] = existing
|
self.products[url] = existing
|
||||||
|
|
||||||
self.save()
|
self.save()
|
||||||
return new_products, restocked_products
|
return new_products, restocked_products, price_changed_products
|
||||||
|
|
||||||
def process_products_with_priority(self, products: List[Product]) -> Tuple[List[Tuple[Product, dict]], List[Tuple[Product, dict]]]:
|
def process_products_with_priority(self, products: List[Product]) -> Tuple[List[Tuple[Product, dict]], List[Tuple[Product, dict]], List[Tuple[Product, str, str, dict]]]:
|
||||||
"""
|
"""
|
||||||
Process products and return with priority/favorite info.
|
Process products and return with priority/favorite info.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (new_products_with_priority, restocked_products_with_priority)
|
Tuple of (new_products_with_priority, restocked_products_with_priority, price_changed_with_priority)
|
||||||
Each item is a tuple of (Product, notification_settings)
|
new/restocked items are tuples of (Product, notification_settings)
|
||||||
|
price_changed items are tuples of (Product, old_price, new_price, notification_settings)
|
||||||
"""
|
"""
|
||||||
new_products, restocked_products = self.process_products(products)
|
new_products, restocked_products, price_changed_products = self.process_products(products)
|
||||||
|
|
||||||
def add_priority(product: Product) -> Tuple[Product, dict]:
|
def add_priority(product: Product) -> Tuple[Product, dict]:
|
||||||
# Get product from DB for category
|
# Get product from DB for category
|
||||||
@@ -174,7 +205,13 @@ class ProductTracker:
|
|||||||
new_with_priority = [add_priority(p) for p in new_products]
|
new_with_priority = [add_priority(p) for p in new_products]
|
||||||
restocked_with_priority = [add_priority(p) for p in restocked_products]
|
restocked_with_priority = [add_priority(p) for p in restocked_products]
|
||||||
|
|
||||||
return new_with_priority, restocked_with_priority
|
# Price changes include old/new price info
|
||||||
|
price_changed_with_priority = []
|
||||||
|
for product, old_price, new_price in price_changed_products:
|
||||||
|
_, settings = add_priority(product)
|
||||||
|
price_changed_with_priority.append((product, old_price, new_price, settings))
|
||||||
|
|
||||||
|
return new_with_priority, restocked_with_priority, price_changed_with_priority
|
||||||
|
|
||||||
def log_check(self, site: str, products_found: int, new_count: int,
|
def log_check(self, site: str, products_found: int, new_count: int,
|
||||||
restock_count: int, duration_ms: int, success: bool = True,
|
restock_count: int, duration_ms: int, success: bool = True,
|
||||||
|
|||||||
@@ -15,9 +15,9 @@ from pathlib import Path
|
|||||||
STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json"
|
STATE_FILE = Path(__file__).parent.parent / "data" / "scraper_state.json"
|
||||||
|
|
||||||
# Default state
|
# Default state
|
||||||
|
# Note: Pokemon Center uses Chrome Extension, not a Python scraper
|
||||||
DEFAULT_STATE = {
|
DEFAULT_STATE = {
|
||||||
"scrapers": {
|
"scrapers": {
|
||||||
"pokemoncenter": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
|
||||||
"target": {"enabled": True, "running": False, "last_run": None, "last_error": None},
|
"target": {"enabled": True, "running": False, "last_run": None, "last_error": None},
|
||||||
"gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
"gamestop": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||||
"walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
"walmart": {"enabled": False, "running": False, "last_run": None, "last_error": None},
|
||||||
|
|||||||
@@ -193,6 +193,47 @@ class StealthBrowser:
|
|||||||
self.driver = None
|
self.driver = None
|
||||||
self._setup_complete = False
|
self._setup_complete = False
|
||||||
|
|
||||||
|
def is_session_valid(self) -> bool:
|
||||||
|
"""
|
||||||
|
Check if the browser session is still valid.
|
||||||
|
Returns False if the session has crashed or been closed.
|
||||||
|
"""
|
||||||
|
if not self.driver:
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Try to get current URL - this will fail if session is invalid
|
||||||
|
_ = self.driver.current_url
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
error_msg = str(e).lower()
|
||||||
|
if "invalid session id" in error_msg or "session deleted" in error_msg or "no such session" in error_msg:
|
||||||
|
logger.warning(f"Browser session invalid: {e}")
|
||||||
|
return False
|
||||||
|
# Other errors might be temporary
|
||||||
|
return True
|
||||||
|
|
||||||
|
def restart(self):
|
||||||
|
"""
|
||||||
|
Restart the browser, cleaning up the old session.
|
||||||
|
Useful when the session has become invalid.
|
||||||
|
"""
|
||||||
|
logger.info("Restarting stealth browser...")
|
||||||
|
|
||||||
|
# Force cleanup of old driver without trying to save cookies (session is dead)
|
||||||
|
if self.driver:
|
||||||
|
try:
|
||||||
|
self.driver.quit()
|
||||||
|
except Exception:
|
||||||
|
pass # Ignore errors - session is already dead
|
||||||
|
finally:
|
||||||
|
self.driver = None
|
||||||
|
self._setup_complete = False
|
||||||
|
|
||||||
|
# Start fresh
|
||||||
|
self.start()
|
||||||
|
logger.info("Stealth browser restarted successfully")
|
||||||
|
|
||||||
def _get_cookie_file(self) -> Path:
|
def _get_cookie_file(self) -> Path:
|
||||||
"""Get path to cookie file for this session"""
|
"""Get path to cookie file for this session"""
|
||||||
return SESSION_DIR / f"{self.session_name}_cookies.pkl"
|
return SESSION_DIR / f"{self.session_name}_cookies.pkl"
|
||||||
|
|||||||
Reference in New Issue
Block a user