feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Test modules
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
Test Pokemon Center API endpoints directly
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
|
||||
# Headers extracted from HAR capture
|
||||
HEADERS = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Version": "1",
|
||||
"Content-Type": "application/json",
|
||||
"X-Store-Locale": "en-us",
|
||||
"X-Store-Scope": "pokemon",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.pokemoncenter.com/category/tcg-cards",
|
||||
}
|
||||
|
||||
# Test endpoints
|
||||
ENDPOINTS = [
|
||||
# Category listing
|
||||
("GET", "https://www.pokemoncenter.com/site/resourceapi/category/new-releases"),
|
||||
|
||||
# Product details (example SKU)
|
||||
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/699-17113"),
|
||||
|
||||
# Product status
|
||||
("GET", "https://www.pokemoncenter.com/tpci-ecommweb-api/product/status/qgqvbkjwhe4s2mjxgeytg="),
|
||||
]
|
||||
|
||||
print("=" * 70)
|
||||
print("Pokemon Center API Test")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
for method, url in ENDPOINTS:
|
||||
print(f"[{method}] {url[:80]}...")
|
||||
|
||||
try:
|
||||
if method == "GET":
|
||||
response = requests.get(url, headers=HEADERS, timeout=10)
|
||||
else:
|
||||
response = requests.post(url, headers=HEADERS, timeout=10)
|
||||
|
||||
print(f" Status: {response.status_code}")
|
||||
print(f" Content-Type: {response.headers.get('Content-Type', 'N/A')}")
|
||||
|
||||
if response.status_code == 200:
|
||||
try:
|
||||
data = response.json()
|
||||
print(f" Response keys: {list(data.keys())[:5]}...")
|
||||
|
||||
# Show a preview
|
||||
preview = json.dumps(data, indent=2)[:500]
|
||||
print(f" Preview:\n{preview}")
|
||||
except:
|
||||
print(f" Raw: {response.text[:200]}")
|
||||
else:
|
||||
print(f" Response: {response.text[:200]}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("If APIs return 200, we can monitor without a browser!")
|
||||
print("=" * 70)
|
||||
@@ -0,0 +1,407 @@
|
||||
"""
|
||||
Unit tests for Dashboard REST API endpoints.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
# Add project root to path
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from flask import Flask
|
||||
from src.database import Database
|
||||
|
||||
# Import api module directly to avoid circular imports via dashboard/__init__.py
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"dashboard_api",
|
||||
os.path.join(project_root, "dashboard", "api.py")
|
||||
)
|
||||
dashboard_api = importlib.util.module_from_spec(spec)
|
||||
sys.modules["dashboard_api"] = dashboard_api
|
||||
spec.loader.exec_module(dashboard_api)
|
||||
api_bp = dashboard_api.api_bp
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""Create test Flask app"""
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.register_blueprint(api_bp, url_prefix='/api')
|
||||
|
||||
# Use temp database
|
||||
with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
|
||||
db_path = f.name
|
||||
|
||||
# Monkey-patch the database
|
||||
import src.database as db_module
|
||||
db_module._db = Database(db_path)
|
||||
|
||||
# Also reset the favorites manager so it picks up the new database
|
||||
import src.favorites as fav_module
|
||||
fav_module._favorites = None # Reset so it gets recreated with new db
|
||||
|
||||
yield app
|
||||
|
||||
# Cleanup
|
||||
os.unlink(db_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
"""Create test client"""
|
||||
return app.test_client()
|
||||
|
||||
|
||||
class TestStatsEndpoints:
|
||||
"""Tests for stats and health endpoints"""
|
||||
|
||||
def test_get_stats(self, client):
|
||||
"""Test /api/stats endpoint"""
|
||||
response = client.get('/api/stats')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'total_products' in data
|
||||
assert 'in_stock_count' in data
|
||||
assert 'new_drops_today' in data
|
||||
|
||||
def test_get_health(self, client):
|
||||
"""Test /api/health endpoint"""
|
||||
response = client.get('/api/health')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert data['status'] == 'ok'
|
||||
|
||||
|
||||
class TestProductsEndpoints:
|
||||
"""Tests for product endpoints"""
|
||||
|
||||
def test_get_products_empty(self, client):
|
||||
"""Test getting products when empty"""
|
||||
response = client.get('/api/products')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_products_with_filters(self, client):
|
||||
"""Test product filtering"""
|
||||
response = client.get('/api/products?site=pokemoncenter&in_stock=true')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_get_product_not_found(self, client):
|
||||
"""Test getting non-existent product"""
|
||||
response = client.get('/api/products/99999')
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestEventsEndpoints:
|
||||
"""Tests for events endpoints"""
|
||||
|
||||
def test_get_events(self, client):
|
||||
"""Test /api/events endpoint"""
|
||||
response = client.get('/api/events')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'events' in data
|
||||
|
||||
def test_get_events_with_type_filter(self, client):
|
||||
"""Test filtering events by type"""
|
||||
response = client.get('/api/events?type=restock&type=new_drop')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestAnalyticsEndpoints:
|
||||
"""Tests for analytics endpoints"""
|
||||
|
||||
def test_get_drop_timing(self, client):
|
||||
"""Test /api/analytics/drops endpoint"""
|
||||
response = client.get('/api/analytics/drops')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'drop_timing' in data
|
||||
|
||||
def test_get_stock_duration(self, client):
|
||||
"""Test /api/analytics/stock endpoint"""
|
||||
response = client.get('/api/analytics/stock')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'stock_duration' in data
|
||||
|
||||
def test_get_selling_rates(self, client):
|
||||
"""Test /api/analytics/selling-rates endpoint"""
|
||||
response = client.get('/api/analytics/selling-rates')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
|
||||
def test_get_site_stats(self, client):
|
||||
"""Test /api/analytics/sites endpoint"""
|
||||
response = client.get('/api/analytics/sites')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'sites' in data
|
||||
|
||||
|
||||
class TestExtensionSyncEndpoint:
|
||||
"""Tests for Chrome extension sync endpoint"""
|
||||
|
||||
def test_sync_empty_data(self, client):
|
||||
"""Test syncing with no data returns error"""
|
||||
response = client.post('/api/extension/sync',
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_sync_skus(self, client):
|
||||
"""Test syncing SKUs from extension"""
|
||||
data = {
|
||||
'skus': ['699-17113', '191-85953', '100-12345'],
|
||||
'products': [],
|
||||
'events': []
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['total_skus'] == 3
|
||||
assert result['new_skus_added'] == 3
|
||||
|
||||
def test_sync_products(self, client):
|
||||
"""Test syncing products from extension"""
|
||||
data = {
|
||||
'skus': [],
|
||||
'products': [
|
||||
{
|
||||
'url': 'https://www.pokemoncenter.com/product/699-17113',
|
||||
'name': 'Pokemon ETB Prismatic Evolutions',
|
||||
'price': '$49.99',
|
||||
'inStock': True,
|
||||
'imageUrl': 'https://example.com/image.jpg',
|
||||
'productId': '699-17113'
|
||||
}
|
||||
],
|
||||
'events': []
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['total_products'] >= 1
|
||||
|
||||
def test_sync_events(self, client):
|
||||
"""Test syncing events (restocks, drops) from extension"""
|
||||
# First sync a product
|
||||
product_data = {
|
||||
'skus': [],
|
||||
'products': [
|
||||
{
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'price': '$29.99',
|
||||
'inStock': True
|
||||
}
|
||||
],
|
||||
'events': []
|
||||
}
|
||||
client.post('/api/extension/sync',
|
||||
data=json.dumps(product_data),
|
||||
content_type='application/json')
|
||||
|
||||
# Now sync events
|
||||
event_data = {
|
||||
'skus': [],
|
||||
'products': [],
|
||||
'events': [
|
||||
{
|
||||
'type': 'restock',
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'price': '$29.99',
|
||||
'timestamp': '2024-01-15T10:00:00Z'
|
||||
},
|
||||
{
|
||||
'type': 'out_of_stock',
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'timestamp': '2024-01-15T10:30:00Z'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(event_data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['events_processed'] == 2
|
||||
|
||||
def test_sync_price_change_event(self, client):
|
||||
"""Test syncing price change events"""
|
||||
# First create the product
|
||||
client.post('/api/extension/sync',
|
||||
data=json.dumps({
|
||||
'products': [{
|
||||
'url': 'https://www.pokemoncenter.com/product/price-test',
|
||||
'name': 'Price Test Product',
|
||||
'price': '$39.99',
|
||||
'inStock': True
|
||||
}]
|
||||
}),
|
||||
content_type='application/json')
|
||||
|
||||
# Now send price change event
|
||||
event_data = {
|
||||
'events': [
|
||||
{
|
||||
'type': 'price_change',
|
||||
'url': 'https://www.pokemoncenter.com/product/price-test',
|
||||
'oldPrice': '$39.99',
|
||||
'newPrice': '$34.99',
|
||||
'timestamp': '2024-01-15T12:00:00Z'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(event_data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
assert result['events_processed'] == 1
|
||||
|
||||
|
||||
class TestExtensionDataEndpoints:
|
||||
"""Tests for extension data retrieval endpoints"""
|
||||
|
||||
def test_get_extension_skus(self, client):
|
||||
"""Test getting extension SKUs"""
|
||||
response = client.get('/api/extension/skus')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'skus' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_extension_products(self, client):
|
||||
"""Test getting extension products"""
|
||||
response = client.get('/api/extension/products')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_extension_stats(self, client):
|
||||
"""Test getting extension stats"""
|
||||
response = client.get('/api/extension/stats')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'total_skus' in data
|
||||
assert 'total_products' in data
|
||||
assert 'source' in data
|
||||
assert data['source'] == 'chrome_extension'
|
||||
|
||||
def test_clear_extension_data(self, client):
|
||||
"""Test clearing extension data"""
|
||||
response = client.post('/api/extension/clear')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert data['success'] is True
|
||||
|
||||
|
||||
class TestFavoritesEndpoints:
|
||||
"""Tests for favorites CRUD endpoints"""
|
||||
|
||||
def test_add_favorite(self, client):
|
||||
"""Test adding a favorite"""
|
||||
data = {
|
||||
'type': 'product',
|
||||
'value': 'https://example.com/fav-product',
|
||||
'display_name': 'My Favorite ETB',
|
||||
'priority': 'high'
|
||||
}
|
||||
|
||||
response = client.post('/api/favorites',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 201
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert 'id' in result
|
||||
|
||||
def test_get_favorites(self, client):
|
||||
"""Test getting all favorites"""
|
||||
response = client.get('/api/favorites')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'favorites' in data
|
||||
|
||||
def test_delete_favorite(self, client):
|
||||
"""Test deleting a favorite"""
|
||||
# First add one
|
||||
add_response = client.post('/api/favorites',
|
||||
data=json.dumps({
|
||||
'type': 'category',
|
||||
'value': 'ETB'
|
||||
}),
|
||||
content_type='application/json')
|
||||
|
||||
fav_id = json.loads(add_response.data)['id']
|
||||
|
||||
# Now delete it
|
||||
response = client.delete(f'/api/favorites/{fav_id}')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
Unit tests for database operations.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from src.database import Database
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db():
|
||||
"""Create a temporary database for testing"""
|
||||
with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
|
||||
db_path = f.name
|
||||
|
||||
database = Database(db_path)
|
||||
yield database
|
||||
|
||||
# Cleanup
|
||||
os.unlink(db_path)
|
||||
|
||||
|
||||
class TestProductOperations:
|
||||
"""Tests for product CRUD operations"""
|
||||
|
||||
def test_create_product(self, db):
|
||||
"""Test creating a new product"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/product/123",
|
||||
name="Test Pokemon Card",
|
||||
site="pokemoncenter",
|
||||
product_id="123",
|
||||
price="$19.99",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
assert product_id is not None
|
||||
assert product_id > 0
|
||||
|
||||
def test_get_product_by_url(self, db):
|
||||
"""Test retrieving product by URL"""
|
||||
url = "https://example.com/product/456"
|
||||
db.get_or_create_product(
|
||||
url=url,
|
||||
name="Pikachu ETB",
|
||||
site="target",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
product = db.get_product_by_url(url)
|
||||
|
||||
assert product is not None
|
||||
assert product['name'] == "Pikachu ETB"
|
||||
assert product['site'] == "target"
|
||||
|
||||
def test_update_product_price(self, db):
|
||||
"""Test updating product price"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/product/789",
|
||||
name="Booster Box",
|
||||
site="bestbuy",
|
||||
price="$99.99",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
# Update price
|
||||
db.update_product_price(product_id, "$89.99")
|
||||
|
||||
product = db.get_product(product_id)
|
||||
assert product['current_price'] == "$89.99"
|
||||
|
||||
# Check price history was recorded
|
||||
history = db.get_price_history(product_id, days=1)
|
||||
assert len(history) >= 1
|
||||
|
||||
def test_update_product_stock(self, db):
|
||||
"""Test updating product stock status"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/product/stock-test",
|
||||
name="Limited Edition",
|
||||
site="pokemoncenter",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
# Mark out of stock
|
||||
db.update_product_stock(product_id, False)
|
||||
|
||||
product = db.get_product(product_id)
|
||||
assert product['in_stock'] == 0 # SQLite stores as 0/1
|
||||
|
||||
def test_category_detection(self, db):
|
||||
"""Test automatic category detection from product name"""
|
||||
test_cases = [
|
||||
("Pokemon Scarlet Elite Trainer Box", "ETB"),
|
||||
("Charizard Booster Bundle", "Booster Bundle"),
|
||||
("Display Booster Box 36 Packs", "Booster Box"),
|
||||
("Sleeved Booster Pack", "Booster Pack"),
|
||||
("Premium Collection Box", "Collection Box"),
|
||||
("Pokemon Trading Card Tin", "Tin"),
|
||||
("Card Binder Album", "Accessories"),
|
||||
("Random Pokemon Item", "Other"),
|
||||
]
|
||||
|
||||
for name, expected_category in test_cases:
|
||||
product_id = db.get_or_create_product(
|
||||
url=f"https://example.com/{name.replace(' ', '-')}",
|
||||
name=name,
|
||||
site="test",
|
||||
in_stock=True
|
||||
)
|
||||
product = db.get_product(product_id)
|
||||
assert product['category'] == expected_category, f"Failed for: {name}"
|
||||
|
||||
|
||||
class TestStockEvents:
|
||||
"""Tests for stock event tracking"""
|
||||
|
||||
def test_record_stock_event(self, db):
|
||||
"""Test recording stock events"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/event-test",
|
||||
name="Event Test Product",
|
||||
site="test",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
db.record_stock_event(product_id, "new_drop")
|
||||
db.record_stock_event(product_id, "out_of_stock")
|
||||
db.record_stock_event(product_id, "restock")
|
||||
|
||||
events = db.get_recent_events(limit=10)
|
||||
|
||||
# Should have 3 events
|
||||
product_events = [e for e in events if e['product_id'] == product_id]
|
||||
assert len(product_events) == 3
|
||||
|
||||
def test_events_today_count(self, db):
|
||||
"""Test counting events from today"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/today-test",
|
||||
name="Today Test",
|
||||
site="test",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
db.record_stock_event(product_id, "new_drop")
|
||||
db.record_stock_event(product_id, "restock")
|
||||
|
||||
new_drops = db.get_events_today("new_drop")
|
||||
restocks = db.get_events_today("restock")
|
||||
|
||||
assert new_drops >= 1
|
||||
assert restocks >= 1
|
||||
|
||||
def test_selling_rate_calculation(self, db):
|
||||
"""Test that selling rate can be calculated from events"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/sellrate-test",
|
||||
name="Fast Seller",
|
||||
site="pokemoncenter",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
# Simulate: drop -> sold out
|
||||
db.record_stock_event(product_id, "new_drop")
|
||||
db.record_stock_event(product_id, "out_of_stock")
|
||||
|
||||
# Get stock duration stats
|
||||
stats = db.get_stock_duration_stats()
|
||||
# Should return at least an empty list (stats depend on timing)
|
||||
assert isinstance(stats, list)
|
||||
|
||||
|
||||
class TestPriceHistory:
|
||||
"""Tests for price history tracking"""
|
||||
|
||||
def test_record_price_change(self, db):
|
||||
"""Test that price changes are recorded"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/price-test",
|
||||
name="Price Test",
|
||||
site="test",
|
||||
price="$49.99",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
# Record initial price
|
||||
db.record_price(product_id, "$49.99")
|
||||
|
||||
# Change price
|
||||
db.record_price(product_id, "$39.99")
|
||||
|
||||
history = db.get_price_history(product_id, days=1)
|
||||
|
||||
# Should have at least 2 price points
|
||||
assert len(history) >= 1
|
||||
|
||||
def test_duplicate_price_not_recorded(self, db):
|
||||
"""Test that same price is not recorded multiple times"""
|
||||
product_id = db.get_or_create_product(
|
||||
url="https://example.com/dup-price-test",
|
||||
name="Dup Price Test",
|
||||
site="test",
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
# Record same price multiple times
|
||||
db.record_price(product_id, "$29.99")
|
||||
db.record_price(product_id, "$29.99")
|
||||
db.record_price(product_id, "$29.99")
|
||||
|
||||
history = db.get_price_history(product_id, days=1)
|
||||
|
||||
# Should only have 1 entry
|
||||
assert len(history) == 1
|
||||
|
||||
|
||||
class TestDashboardStats:
|
||||
"""Tests for dashboard statistics"""
|
||||
|
||||
def test_get_dashboard_stats(self, db):
|
||||
"""Test getting dashboard stats summary"""
|
||||
# Add some test data
|
||||
for i in range(5):
|
||||
product_id = db.get_or_create_product(
|
||||
url=f"https://example.com/stats-{i}",
|
||||
name=f"Stats Product {i}",
|
||||
site="pokemoncenter",
|
||||
in_stock=(i % 2 == 0)
|
||||
)
|
||||
db.record_stock_event(product_id, "new_drop")
|
||||
|
||||
stats = db.get_dashboard_stats()
|
||||
|
||||
assert 'total_products' in stats
|
||||
assert 'in_stock_count' in stats
|
||||
assert 'new_drops_today' in stats
|
||||
assert stats['total_products'] >= 5
|
||||
|
||||
def test_get_site_stats(self, db):
|
||||
"""Test per-site statistics"""
|
||||
sites = ['target', 'bestbuy', 'pokemoncenter']
|
||||
|
||||
for site in sites:
|
||||
db.get_or_create_product(
|
||||
url=f"https://{site}.com/test",
|
||||
name=f"{site} Product",
|
||||
site=site,
|
||||
in_stock=True
|
||||
)
|
||||
|
||||
stats = db.get_site_stats()
|
||||
|
||||
assert len(stats) == 3
|
||||
site_names = [s['site'] for s in stats]
|
||||
assert 'target' in site_names
|
||||
assert 'bestbuy' in site_names
|
||||
|
||||
|
||||
class TestFavorites:
|
||||
"""Tests for favorites functionality"""
|
||||
|
||||
def test_add_favorite(self, db):
|
||||
"""Test adding a favorite"""
|
||||
fav_id = db.add_favorite(
|
||||
fav_type="product",
|
||||
value="https://example.com/fav-product",
|
||||
display_name="My Favorite Card",
|
||||
priority="high"
|
||||
)
|
||||
|
||||
assert fav_id is not None
|
||||
|
||||
favorite = db.get_favorite(fav_id)
|
||||
assert favorite['display_name'] == "My Favorite Card"
|
||||
|
||||
def test_check_is_favorite(self, db):
|
||||
"""Test checking if product is favorited"""
|
||||
url = "https://example.com/check-fav"
|
||||
|
||||
# Not a favorite yet
|
||||
result = db.check_is_favorite(url=url)
|
||||
assert result is None
|
||||
|
||||
# Add as favorite
|
||||
db.add_favorite(fav_type="product", value=url)
|
||||
|
||||
# Now should be found
|
||||
result = db.check_is_favorite(url=url)
|
||||
assert result is not None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Unit tests for Pokemon product validation filter.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from scrapers.base import BaseScraper, Product
|
||||
|
||||
|
||||
def make_product(name: str, url: str = "", site: str = "test") -> Product:
|
||||
"""Helper to create Product with required fields"""
|
||||
return Product(name=name, url=url, price=None, in_stock=True, site=site)
|
||||
|
||||
|
||||
class MockScraper(BaseScraper):
|
||||
"""Mock scraper for testing base class methods"""
|
||||
|
||||
site_name = "test"
|
||||
|
||||
def scrape_category_page(self, url: str):
|
||||
return []
|
||||
|
||||
def check_product_stock(self, product):
|
||||
return True, "$19.99"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scraper():
|
||||
return MockScraper()
|
||||
|
||||
|
||||
class TestPokemonProductFilter:
|
||||
"""Tests for is_pokemon_product validation"""
|
||||
|
||||
def test_valid_pokemon_products(self, scraper):
|
||||
"""Test that valid Pokemon products are accepted"""
|
||||
valid_products = [
|
||||
make_product("Pokemon Scarlet & Violet Elite Trainer Box"),
|
||||
make_product("Pikachu V Collection Box"),
|
||||
make_product("TCG Booster Pack Prismatic Evolutions"),
|
||||
make_product("Charizard Premium Collection"),
|
||||
make_product("Pokemon ETB Paldean Fates"),
|
||||
make_product("Pokémon Trading Card Game Tin"), # Unicode é
|
||||
make_product("Mewtwo VMAX Box Set"),
|
||||
make_product("Eevee Heroes Booster Box"),
|
||||
]
|
||||
|
||||
for product in valid_products:
|
||||
assert scraper.is_pokemon_product(product), f"Should accept: {product.name}"
|
||||
|
||||
def test_invalid_non_pokemon_products(self, scraper):
|
||||
"""Test that non-Pokemon products are rejected"""
|
||||
invalid_products = [
|
||||
make_product("Ice Cube Tray Silicone Mold"),
|
||||
make_product("Barbie Dream House Playset"),
|
||||
make_product("Hot Wheels Track Builder"),
|
||||
make_product("LEGO Star Wars Set"),
|
||||
make_product("Kitchen Appliance Blender"),
|
||||
make_product("Transformers Action Figure"),
|
||||
make_product("Room Essentials Bedding Set"),
|
||||
make_product("Threshold Furniture Table"),
|
||||
make_product("Random Gaming Accessory"),
|
||||
]
|
||||
|
||||
for product in invalid_products:
|
||||
assert not scraper.is_pokemon_product(product), f"Should reject: {product.name}"
|
||||
|
||||
def test_exclusion_takes_priority(self, scraper):
|
||||
"""Test that exclusion terms override Pokemon terms"""
|
||||
# This has "pokemon" but also "ice cube" - should be rejected
|
||||
product = make_product("Pokemon Ice Cube Tray Silicone")
|
||||
assert not scraper.is_pokemon_product(product)
|
||||
|
||||
def test_case_insensitive(self, scraper):
|
||||
"""Test that matching is case insensitive"""
|
||||
products = [
|
||||
make_product("POKEMON ELITE TRAINER BOX"),
|
||||
make_product("pokemon scarlet booster"),
|
||||
make_product("PoKeMoN ChArIzArD"),
|
||||
]
|
||||
|
||||
for product in products:
|
||||
assert scraper.is_pokemon_product(product), f"Should accept: {product.name}"
|
||||
|
||||
|
||||
class TestFilterPokemonProducts:
|
||||
"""Tests for filter_pokemon_products batch filtering"""
|
||||
|
||||
def test_filter_mixed_products(self, scraper):
|
||||
"""Test filtering a mix of valid and invalid products"""
|
||||
products = [
|
||||
make_product("Pokemon Booster Pack", url="1"),
|
||||
make_product("Ice Cube Tray", url="2"),
|
||||
make_product("Charizard Collection", url="3"),
|
||||
make_product("Barbie Doll", url="4"),
|
||||
make_product("Pikachu Tin", url="5"),
|
||||
]
|
||||
|
||||
filtered = scraper.filter_pokemon_products(products)
|
||||
|
||||
assert len(filtered) == 3
|
||||
urls = [p.url for p in filtered]
|
||||
assert "1" in urls
|
||||
assert "3" in urls
|
||||
assert "5" in urls
|
||||
assert "2" not in urls
|
||||
assert "4" not in urls
|
||||
|
||||
def test_filter_empty_list(self, scraper):
|
||||
"""Test filtering empty list returns empty"""
|
||||
filtered = scraper.filter_pokemon_products([])
|
||||
assert filtered == []
|
||||
|
||||
def test_filter_all_valid(self, scraper):
|
||||
"""Test filtering when all products are valid"""
|
||||
products = [
|
||||
make_product("Pokemon ETB", url="1"),
|
||||
make_product("Pokemon Booster", url="2"),
|
||||
]
|
||||
|
||||
filtered = scraper.filter_pokemon_products(products)
|
||||
assert len(filtered) == 2
|
||||
|
||||
def test_filter_all_invalid(self, scraper):
|
||||
"""Test filtering when all products are invalid"""
|
||||
products = [
|
||||
make_product("Random Item", url="1"),
|
||||
make_product("Another Thing", url="2"),
|
||||
]
|
||||
|
||||
filtered = scraper.filter_pokemon_products(products)
|
||||
assert len(filtered) == 0
|
||||
|
||||
|
||||
class TestProductDataclass:
|
||||
"""Tests for Product dataclass"""
|
||||
|
||||
def test_product_creation(self):
|
||||
"""Test creating a Product with all fields"""
|
||||
product = Product(
|
||||
name="Test Product",
|
||||
url="https://example.com/test",
|
||||
price="$29.99",
|
||||
in_stock=True,
|
||||
site="pokemoncenter",
|
||||
image_url="https://example.com/image.jpg",
|
||||
product_id="123"
|
||||
)
|
||||
|
||||
assert product.name == "Test Product"
|
||||
assert product.price == "$29.99"
|
||||
assert product.in_stock is True
|
||||
|
||||
def test_product_minimal(self):
|
||||
"""Test Product with only required fields"""
|
||||
product = Product(
|
||||
name="Minimal Product",
|
||||
url="https://example.com",
|
||||
price=None,
|
||||
in_stock=False
|
||||
)
|
||||
|
||||
assert product.name == "Minimal Product"
|
||||
assert product.price is None
|
||||
assert product.in_stock is False
|
||||
assert product.image_url is None
|
||||
assert product.product_id is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Test script for the stealth browser module
|
||||
Run this to verify undetected-chromedriver is working
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
print("=" * 60)
|
||||
print("Stealth Browser Test")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Check dependencies
|
||||
print("1. Checking dependencies...")
|
||||
try:
|
||||
import undetected_chromedriver as uc
|
||||
print(" [OK] undetected-chromedriver installed")
|
||||
except ImportError:
|
||||
print(" [ERROR] undetected-chromedriver not installed!")
|
||||
print(" Run: pip install undetected-chromedriver")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
print(" [OK] selenium installed")
|
||||
except ImportError:
|
||||
print(" [ERROR] selenium not installed!")
|
||||
print(" Run: pip install selenium")
|
||||
sys.exit(1)
|
||||
|
||||
# Import our stealth browser
|
||||
print()
|
||||
print("2. Importing stealth browser module...")
|
||||
try:
|
||||
from stealth_browser import StealthBrowser, get_stealth_browser
|
||||
print(" [OK] stealth_browser module loaded")
|
||||
except Exception as e:
|
||||
print(f" [ERROR] Failed to import: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Test browser launch
|
||||
print()
|
||||
print("3. Launching stealth browser...")
|
||||
print(" (A Chrome window should open)")
|
||||
print()
|
||||
|
||||
browser = None
|
||||
try:
|
||||
browser = StealthBrowser(
|
||||
headless=False,
|
||||
session_name="test_session"
|
||||
)
|
||||
browser.start()
|
||||
print(" [OK] Browser started successfully!")
|
||||
|
||||
# Test navigation
|
||||
print()
|
||||
print("4. Testing navigation to Pokemon Center...")
|
||||
print(" (Watch the browser window)")
|
||||
|
||||
url = "https://www.pokemoncenter.com/category/tcg-cards"
|
||||
html = browser.get_page(url, wait_time=5)
|
||||
|
||||
print(f" [OK] Page loaded - {len(html)} bytes")
|
||||
|
||||
# Check for CAPTCHA
|
||||
print()
|
||||
print("5. Checking for bot detection...")
|
||||
if browser.check_for_captcha():
|
||||
print(" [!] CAPTCHA/Challenge detected!")
|
||||
print(" The browser window is open - solve it manually if needed")
|
||||
print(" Waiting up to 60 seconds...")
|
||||
browser.wait_for_captcha_solve(timeout=60)
|
||||
else:
|
||||
print(" [OK] No CAPTCHA detected! Stealth mode working.")
|
||||
|
||||
# Show some page info
|
||||
print()
|
||||
print("6. Page analysis...")
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
title = soup.title.string if soup.title else "No title"
|
||||
print(f" Page title: {title}")
|
||||
|
||||
# Count potential product elements
|
||||
products = soup.select("[data-testid='product-card'], .product-card, a[href*='/product/']")
|
||||
print(f" Potential product elements: {len(products)}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("TEST COMPLETE")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("The browser window will stay open for 10 seconds so you can inspect it.")
|
||||
print("Press Ctrl+C to close early.")
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n Interrupted by user")
|
||||
except Exception as e:
|
||||
print(f" [ERROR] {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if browser:
|
||||
print()
|
||||
print("Closing browser...")
|
||||
browser.stop()
|
||||
print("Done!")
|
||||
Reference in New Issue
Block a user