feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,407 @@
|
||||
"""
|
||||
Unit tests for Dashboard REST API endpoints.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
# Add project root to path
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from flask import Flask
|
||||
from src.database import Database
|
||||
|
||||
# Import api module directly to avoid circular imports via dashboard/__init__.py
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"dashboard_api",
|
||||
os.path.join(project_root, "dashboard", "api.py")
|
||||
)
|
||||
dashboard_api = importlib.util.module_from_spec(spec)
|
||||
sys.modules["dashboard_api"] = dashboard_api
|
||||
spec.loader.exec_module(dashboard_api)
|
||||
api_bp = dashboard_api.api_bp
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""Create test Flask app"""
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.register_blueprint(api_bp, url_prefix='/api')
|
||||
|
||||
# Use temp database
|
||||
with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
|
||||
db_path = f.name
|
||||
|
||||
# Monkey-patch the database
|
||||
import src.database as db_module
|
||||
db_module._db = Database(db_path)
|
||||
|
||||
# Also reset the favorites manager so it picks up the new database
|
||||
import src.favorites as fav_module
|
||||
fav_module._favorites = None # Reset so it gets recreated with new db
|
||||
|
||||
yield app
|
||||
|
||||
# Cleanup
|
||||
os.unlink(db_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
"""Create test client"""
|
||||
return app.test_client()
|
||||
|
||||
|
||||
class TestStatsEndpoints:
|
||||
"""Tests for stats and health endpoints"""
|
||||
|
||||
def test_get_stats(self, client):
|
||||
"""Test /api/stats endpoint"""
|
||||
response = client.get('/api/stats')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'total_products' in data
|
||||
assert 'in_stock_count' in data
|
||||
assert 'new_drops_today' in data
|
||||
|
||||
def test_get_health(self, client):
|
||||
"""Test /api/health endpoint"""
|
||||
response = client.get('/api/health')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert data['status'] == 'ok'
|
||||
|
||||
|
||||
class TestProductsEndpoints:
|
||||
"""Tests for product endpoints"""
|
||||
|
||||
def test_get_products_empty(self, client):
|
||||
"""Test getting products when empty"""
|
||||
response = client.get('/api/products')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_products_with_filters(self, client):
|
||||
"""Test product filtering"""
|
||||
response = client.get('/api/products?site=pokemoncenter&in_stock=true')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_get_product_not_found(self, client):
|
||||
"""Test getting non-existent product"""
|
||||
response = client.get('/api/products/99999')
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestEventsEndpoints:
|
||||
"""Tests for events endpoints"""
|
||||
|
||||
def test_get_events(self, client):
|
||||
"""Test /api/events endpoint"""
|
||||
response = client.get('/api/events')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'events' in data
|
||||
|
||||
def test_get_events_with_type_filter(self, client):
|
||||
"""Test filtering events by type"""
|
||||
response = client.get('/api/events?type=restock&type=new_drop')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
class TestAnalyticsEndpoints:
|
||||
"""Tests for analytics endpoints"""
|
||||
|
||||
def test_get_drop_timing(self, client):
|
||||
"""Test /api/analytics/drops endpoint"""
|
||||
response = client.get('/api/analytics/drops')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'drop_timing' in data
|
||||
|
||||
def test_get_stock_duration(self, client):
|
||||
"""Test /api/analytics/stock endpoint"""
|
||||
response = client.get('/api/analytics/stock')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'stock_duration' in data
|
||||
|
||||
def test_get_selling_rates(self, client):
|
||||
"""Test /api/analytics/selling-rates endpoint"""
|
||||
response = client.get('/api/analytics/selling-rates')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
|
||||
def test_get_site_stats(self, client):
|
||||
"""Test /api/analytics/sites endpoint"""
|
||||
response = client.get('/api/analytics/sites')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'sites' in data
|
||||
|
||||
|
||||
class TestExtensionSyncEndpoint:
|
||||
"""Tests for Chrome extension sync endpoint"""
|
||||
|
||||
def test_sync_empty_data(self, client):
|
||||
"""Test syncing with no data returns error"""
|
||||
response = client.post('/api/extension/sync',
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_sync_skus(self, client):
|
||||
"""Test syncing SKUs from extension"""
|
||||
data = {
|
||||
'skus': ['699-17113', '191-85953', '100-12345'],
|
||||
'products': [],
|
||||
'events': []
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['total_skus'] == 3
|
||||
assert result['new_skus_added'] == 3
|
||||
|
||||
def test_sync_products(self, client):
|
||||
"""Test syncing products from extension"""
|
||||
data = {
|
||||
'skus': [],
|
||||
'products': [
|
||||
{
|
||||
'url': 'https://www.pokemoncenter.com/product/699-17113',
|
||||
'name': 'Pokemon ETB Prismatic Evolutions',
|
||||
'price': '$49.99',
|
||||
'inStock': True,
|
||||
'imageUrl': 'https://example.com/image.jpg',
|
||||
'productId': '699-17113'
|
||||
}
|
||||
],
|
||||
'events': []
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['total_products'] >= 1
|
||||
|
||||
def test_sync_events(self, client):
|
||||
"""Test syncing events (restocks, drops) from extension"""
|
||||
# First sync a product
|
||||
product_data = {
|
||||
'skus': [],
|
||||
'products': [
|
||||
{
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'price': '$29.99',
|
||||
'inStock': True
|
||||
}
|
||||
],
|
||||
'events': []
|
||||
}
|
||||
client.post('/api/extension/sync',
|
||||
data=json.dumps(product_data),
|
||||
content_type='application/json')
|
||||
|
||||
# Now sync events
|
||||
event_data = {
|
||||
'skus': [],
|
||||
'products': [],
|
||||
'events': [
|
||||
{
|
||||
'type': 'restock',
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'price': '$29.99',
|
||||
'timestamp': '2024-01-15T10:00:00Z'
|
||||
},
|
||||
{
|
||||
'type': 'out_of_stock',
|
||||
'url': 'https://www.pokemoncenter.com/product/test-123',
|
||||
'name': 'Test Product',
|
||||
'timestamp': '2024-01-15T10:30:00Z'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(event_data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['events_processed'] == 2
|
||||
|
||||
def test_sync_price_change_event(self, client):
|
||||
"""Test syncing price change events"""
|
||||
# First create the product
|
||||
client.post('/api/extension/sync',
|
||||
data=json.dumps({
|
||||
'products': [{
|
||||
'url': 'https://www.pokemoncenter.com/product/price-test',
|
||||
'name': 'Price Test Product',
|
||||
'price': '$39.99',
|
||||
'inStock': True
|
||||
}]
|
||||
}),
|
||||
content_type='application/json')
|
||||
|
||||
# Now send price change event
|
||||
event_data = {
|
||||
'events': [
|
||||
{
|
||||
'type': 'price_change',
|
||||
'url': 'https://www.pokemoncenter.com/product/price-test',
|
||||
'oldPrice': '$39.99',
|
||||
'newPrice': '$34.99',
|
||||
'timestamp': '2024-01-15T12:00:00Z'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = client.post('/api/extension/sync',
|
||||
data=json.dumps(event_data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 200
|
||||
result = json.loads(response.data)
|
||||
assert result['events_processed'] == 1
|
||||
|
||||
|
||||
class TestExtensionDataEndpoints:
|
||||
"""Tests for extension data retrieval endpoints"""
|
||||
|
||||
def test_get_extension_skus(self, client):
|
||||
"""Test getting extension SKUs"""
|
||||
response = client.get('/api/extension/skus')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'skus' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_extension_products(self, client):
|
||||
"""Test getting extension products"""
|
||||
response = client.get('/api/extension/products')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'products' in data
|
||||
assert 'total' in data
|
||||
|
||||
def test_get_extension_stats(self, client):
|
||||
"""Test getting extension stats"""
|
||||
response = client.get('/api/extension/stats')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'total_skus' in data
|
||||
assert 'total_products' in data
|
||||
assert 'source' in data
|
||||
assert data['source'] == 'chrome_extension'
|
||||
|
||||
def test_clear_extension_data(self, client):
|
||||
"""Test clearing extension data"""
|
||||
response = client.post('/api/extension/clear')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert data['success'] is True
|
||||
|
||||
|
||||
class TestFavoritesEndpoints:
|
||||
"""Tests for favorites CRUD endpoints"""
|
||||
|
||||
def test_add_favorite(self, client):
|
||||
"""Test adding a favorite"""
|
||||
data = {
|
||||
'type': 'product',
|
||||
'value': 'https://example.com/fav-product',
|
||||
'display_name': 'My Favorite ETB',
|
||||
'priority': 'high'
|
||||
}
|
||||
|
||||
response = client.post('/api/favorites',
|
||||
data=json.dumps(data),
|
||||
content_type='application/json')
|
||||
|
||||
assert response.status_code == 201
|
||||
result = json.loads(response.data)
|
||||
|
||||
assert result['success'] is True
|
||||
assert 'id' in result
|
||||
|
||||
def test_get_favorites(self, client):
|
||||
"""Test getting all favorites"""
|
||||
response = client.get('/api/favorites')
|
||||
|
||||
assert response.status_code == 200
|
||||
data = json.loads(response.data)
|
||||
|
||||
assert 'favorites' in data
|
||||
|
||||
def test_delete_favorite(self, client):
|
||||
"""Test deleting a favorite"""
|
||||
# First add one
|
||||
add_response = client.post('/api/favorites',
|
||||
data=json.dumps({
|
||||
'type': 'category',
|
||||
'value': 'ETB'
|
||||
}),
|
||||
content_type='application/json')
|
||||
|
||||
fav_id = json.loads(add_response.data)['id']
|
||||
|
||||
# Now delete it
|
||||
response = client.delete(f'/api/favorites/{fav_id}')
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user