mmcghen
8d382e723f
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
1083 lines
33 KiB
Python
1083 lines
33 KiB
Python
"""
|
||
REST API endpoints for Pokemon Stock Monitor dashboard.
|
||
"""
|
||
|
||
import sys
|
||
import re
|
||
from pathlib import Path
|
||
|
||
# Add parent directory to path for imports
|
||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||
|
||
from flask import Blueprint, jsonify, request
|
||
from src.database import get_database
|
||
from src.favorites import get_favorites_manager
|
||
from src.scraper_state import scraper_state
|
||
|
||
api_bp = Blueprint('api', __name__)
|
||
|
||
|
||
def clean_product_name(name: str) -> str:
|
||
"""Fix common encoding issues in product names"""
|
||
if not name:
|
||
return name
|
||
|
||
# Fix Pokémon encoding issues
|
||
name = re.sub(r'Pok[éÃ\u00c3\u00a9]+mon', 'Pokémon', name, flags=re.IGNORECASE)
|
||
name = name.replace('Pokémon', 'Pokémon')
|
||
name = name.replace('Pokﾃｩmon', 'Pokémon')
|
||
|
||
# Fix dashes
|
||
name = name.replace('â€"', '–')
|
||
name = name.replace('â€"', '-')
|
||
|
||
# Fix quotes and apostrophes
|
||
name = name.replace('’', "'")
|
||
name = name.replace('‘', "'")
|
||
name = name.replace('“', '"')
|
||
name = name.replace(''', "'")
|
||
name = name.replace(''', "'")
|
||
|
||
# Fix accented characters
|
||
name = name.replace('é', 'é')
|
||
name = name.replace('è', 'è')
|
||
name = name.replace('Ã ', 'à')
|
||
|
||
# Fix trademark
|
||
name = name.replace('â„¢', '™')
|
||
name = name.replace('®', '®')
|
||
name = name.replace('Â', '')
|
||
|
||
return name.strip()
|
||
|
||
|
||
def clean_product_dict(product: dict) -> dict:
|
||
"""Clean encoding in a product dictionary"""
|
||
if product and 'name' in product:
|
||
product['name'] = clean_product_name(product['name'])
|
||
return product
|
||
|
||
|
||
# ==================== Stats Endpoints ====================
|
||
|
||
@api_bp.route('/stats')
|
||
def get_stats():
|
||
"""Get dashboard stats summary"""
|
||
db = get_database()
|
||
stats = db.get_dashboard_stats()
|
||
return jsonify(stats)
|
||
|
||
|
||
@api_bp.route('/health')
|
||
def get_health():
|
||
"""Get monitor health status"""
|
||
db = get_database()
|
||
last_check = db.get_last_check()
|
||
return jsonify({
|
||
'status': 'ok',
|
||
'last_check': last_check
|
||
})
|
||
|
||
|
||
# ==================== Products Endpoints ====================
|
||
|
||
@api_bp.route('/products')
|
||
def get_products():
|
||
"""Get products with optional filters"""
|
||
db = get_database()
|
||
|
||
# Parse query params
|
||
site = request.args.get('site')
|
||
category = request.args.get('category')
|
||
in_stock = request.args.get('in_stock')
|
||
favorites_only = request.args.get('favorites_only', 'false').lower() == 'true'
|
||
event_type = request.args.get('event_type') # 'new_drop' or 'restock'
|
||
period = request.args.get('period') # 'today' or 'week'
|
||
limit = int(request.args.get('limit', 100))
|
||
offset = int(request.args.get('offset', 0))
|
||
|
||
# Convert in_stock to bool if provided
|
||
if in_stock is not None:
|
||
in_stock = in_stock.lower() == 'true'
|
||
|
||
products = db.get_products(
|
||
site=site,
|
||
category=category,
|
||
in_stock=in_stock,
|
||
favorites_only=favorites_only,
|
||
event_type=event_type,
|
||
period=period,
|
||
limit=limit,
|
||
offset=offset
|
||
)
|
||
|
||
# Clean names and add favorite status to each product
|
||
favorites = get_favorites_manager()
|
||
for product in products:
|
||
clean_product_dict(product)
|
||
fav = favorites.check_product_priority(
|
||
url=product['url'],
|
||
name=product['name'],
|
||
category=product.get('category')
|
||
)
|
||
product['is_favorite'] = fav is not None
|
||
product['favorite_id'] = fav['id'] if fav else None
|
||
|
||
return jsonify({
|
||
'products': products,
|
||
'total': db.get_product_count(site=site, in_stock=in_stock)
|
||
})
|
||
|
||
|
||
@api_bp.route('/products/<int:product_id>')
|
||
def get_product(product_id):
|
||
"""Get single product details"""
|
||
db = get_database()
|
||
product = db.get_product(product_id)
|
||
|
||
if not product:
|
||
return jsonify({'error': 'Product not found'}), 404
|
||
|
||
clean_product_dict(product)
|
||
return jsonify(product)
|
||
|
||
|
||
@api_bp.route('/products/<int:product_id>/history')
|
||
def get_product_history(product_id):
|
||
"""Get price and stock history for a product"""
|
||
db = get_database()
|
||
days = int(request.args.get('days', 30))
|
||
|
||
price_history = db.get_price_history(product_id, days=days)
|
||
|
||
# Get stock events for this product
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
cursor.execute("""
|
||
SELECT event_type, recorded_at FROM stock_events
|
||
WHERE product_id = ? AND recorded_at >= datetime('now', ?)
|
||
ORDER BY recorded_at ASC
|
||
""", (product_id, f"-{days} days"))
|
||
stock_events = [dict(row) for row in cursor.fetchall()]
|
||
|
||
return jsonify({
|
||
'price_history': price_history,
|
||
'stock_events': stock_events
|
||
})
|
||
|
||
|
||
@api_bp.route('/products/<int:product_id>/timeline')
|
||
def get_product_timeline(product_id):
|
||
"""Get complete timeline of all events for a product"""
|
||
db = get_database()
|
||
|
||
product = db.get_product(product_id)
|
||
if not product:
|
||
return jsonify({'error': 'Product not found'}), 404
|
||
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
|
||
# Get all stock events
|
||
cursor.execute("""
|
||
SELECT 'stock' as source, event_type as type, recorded_at as timestamp
|
||
FROM stock_events WHERE product_id = ?
|
||
UNION ALL
|
||
SELECT 'price' as source, 'price_change' as type, recorded_at as timestamp
|
||
FROM price_history WHERE product_id = ?
|
||
ORDER BY timestamp DESC
|
||
""", (product_id, product_id))
|
||
events = [dict(row) for row in cursor.fetchall()]
|
||
|
||
# Get price at each point
|
||
cursor.execute("""
|
||
SELECT price, recorded_at FROM price_history
|
||
WHERE product_id = ? ORDER BY recorded_at ASC
|
||
""", (product_id,))
|
||
prices = [dict(row) for row in cursor.fetchall()]
|
||
|
||
# Calculate stats
|
||
stats = {
|
||
'total_restocks': sum(1 for e in events if e['type'] == 'restock'),
|
||
'total_sellouts': sum(1 for e in events if e['type'] == 'out_of_stock'),
|
||
'price_changes': sum(1 for e in events if e['type'] == 'price_change'),
|
||
'first_seen': product.get('first_seen'),
|
||
'last_seen': product.get('last_seen')
|
||
}
|
||
|
||
return jsonify({
|
||
'product': product,
|
||
'timeline': events,
|
||
'price_history': prices,
|
||
'stats': stats
|
||
})
|
||
|
||
|
||
# ==================== Events Endpoints ====================
|
||
|
||
@api_bp.route('/events')
|
||
def get_events():
|
||
"""Get recent events feed"""
|
||
db = get_database()
|
||
limit = int(request.args.get('limit', 20))
|
||
event_types = request.args.getlist('type')
|
||
|
||
events = db.get_recent_events(
|
||
limit=limit,
|
||
event_types=event_types if event_types else None
|
||
)
|
||
|
||
# Clean product names in events
|
||
for event in events:
|
||
clean_product_dict(event)
|
||
|
||
return jsonify({'events': events})
|
||
|
||
|
||
# ==================== Analytics Endpoints ====================
|
||
|
||
@api_bp.route('/analytics/drops')
|
||
def get_drop_timing():
|
||
"""Get drop timing analytics"""
|
||
db = get_database()
|
||
days = int(request.args.get('days', 30))
|
||
stats = db.get_drop_timing_stats(days=days)
|
||
return jsonify({'drop_timing': stats})
|
||
|
||
|
||
@api_bp.route('/analytics/stock')
|
||
def get_stock_duration():
|
||
"""Get stock duration analytics"""
|
||
db = get_database()
|
||
stats = db.get_stock_duration_stats()
|
||
return jsonify({'stock_duration': stats})
|
||
|
||
|
||
@api_bp.route('/analytics/sites')
|
||
def get_site_stats():
|
||
"""Get per-site statistics"""
|
||
db = get_database()
|
||
stats = db.get_site_stats()
|
||
return jsonify({'sites': stats})
|
||
|
||
|
||
@api_bp.route('/analytics/checks')
|
||
def get_check_history():
|
||
"""Get check history"""
|
||
db = get_database()
|
||
site = request.args.get('site')
|
||
limit = int(request.args.get('limit', 100))
|
||
|
||
history = db.get_check_history(site=site, limit=limit)
|
||
return jsonify({'checks': history})
|
||
|
||
|
||
@api_bp.route('/analytics/selling-rates')
|
||
def get_selling_rates():
|
||
"""Get detailed selling rate analytics per product"""
|
||
db = get_database()
|
||
limit = int(request.args.get('limit', 50))
|
||
category = request.args.get('category')
|
||
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
# Get products with their sell-through times
|
||
query = """
|
||
SELECT
|
||
p.id, p.name, p.url, p.site, p.category, p.image_url,
|
||
COUNT(CASE WHEN e.event_type = 'restock' THEN 1 END) as restock_count,
|
||
COUNT(CASE WHEN e.event_type = 'out_of_stock' THEN 1 END) as sellout_count,
|
||
COUNT(CASE WHEN e.event_type = 'new_drop' THEN 1 END) as drop_count
|
||
FROM products p
|
||
LEFT JOIN stock_events e ON p.id = e.product_id
|
||
WHERE 1=1
|
||
"""
|
||
params = []
|
||
|
||
if category:
|
||
query += " AND p.category = ?"
|
||
params.append(category)
|
||
|
||
query += """
|
||
GROUP BY p.id
|
||
HAVING sellout_count > 0 OR restock_count > 0
|
||
ORDER BY (restock_count + sellout_count) DESC
|
||
LIMIT ?
|
||
"""
|
||
params.append(limit)
|
||
|
||
cursor.execute(query, params)
|
||
products = [dict(row) for row in cursor.fetchall()]
|
||
|
||
# For each product, calculate average time to sell out
|
||
for product in products:
|
||
cursor.execute("""
|
||
SELECT
|
||
in_event.recorded_at as in_stock_time,
|
||
out_event.recorded_at as out_stock_time,
|
||
CAST((julianday(out_event.recorded_at) - julianday(in_event.recorded_at)) * 24 * 60 AS INTEGER) as minutes_in_stock
|
||
FROM stock_events in_event
|
||
JOIN stock_events out_event ON in_event.product_id = out_event.product_id
|
||
AND out_event.event_type = 'out_of_stock'
|
||
AND out_event.recorded_at > in_event.recorded_at
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM stock_events middle
|
||
WHERE middle.product_id = in_event.product_id
|
||
AND middle.event_type IN ('restock', 'new_drop', 'out_of_stock')
|
||
AND middle.recorded_at > in_event.recorded_at
|
||
AND middle.recorded_at < out_event.recorded_at
|
||
)
|
||
WHERE in_event.product_id = ?
|
||
AND in_event.event_type IN ('restock', 'new_drop')
|
||
ORDER BY in_event.recorded_at DESC
|
||
LIMIT 10
|
||
""", (product['id'],))
|
||
|
||
sellthrough_times = [dict(row) for row in cursor.fetchall()]
|
||
product['sellthrough_history'] = sellthrough_times
|
||
|
||
if sellthrough_times:
|
||
avg_minutes = sum(s['minutes_in_stock'] for s in sellthrough_times) / len(sellthrough_times)
|
||
product['avg_minutes_to_sellout'] = round(avg_minutes, 1)
|
||
else:
|
||
product['avg_minutes_to_sellout'] = None
|
||
|
||
return jsonify({'products': products})
|
||
|
||
|
||
# ==================== Favorites Endpoints ====================
|
||
|
||
@api_bp.route('/favorites', methods=['GET'])
|
||
def get_favorites():
|
||
"""Get all favorites"""
|
||
favorites = get_favorites_manager()
|
||
fav_type = request.args.get('type')
|
||
|
||
if fav_type:
|
||
favs = favorites.db.get_favorites(fav_type=fav_type)
|
||
else:
|
||
favs = favorites.get_all_favorites()
|
||
|
||
return jsonify({'favorites': favs})
|
||
|
||
|
||
@api_bp.route('/favorites', methods=['POST'])
|
||
def add_favorite():
|
||
"""Add a new favorite"""
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
fav_type = data.get('type')
|
||
value = data.get('value')
|
||
|
||
if not fav_type or not value:
|
||
return jsonify({'error': 'type and value are required'}), 400
|
||
|
||
favorites = get_favorites_manager()
|
||
|
||
if fav_type == 'product':
|
||
fav_id = favorites.add_product_favorite(
|
||
url=value,
|
||
display_name=data.get('display_name'),
|
||
priority=data.get('priority', 'high'),
|
||
notify_discord=data.get('notify_discord', True),
|
||
notify_sound=data.get('notify_sound', False),
|
||
custom_webhook=data.get('custom_webhook')
|
||
)
|
||
elif fav_type == 'category':
|
||
fav_id = favorites.add_category_favorite(
|
||
category=value,
|
||
display_name=data.get('display_name'),
|
||
priority=data.get('priority', 'high'),
|
||
notify_discord=data.get('notify_discord', True),
|
||
notify_sound=data.get('notify_sound', False),
|
||
custom_webhook=data.get('custom_webhook')
|
||
)
|
||
else:
|
||
return jsonify({'error': 'Invalid type. Must be "product" or "category"'}), 400
|
||
|
||
return jsonify({'id': fav_id, 'success': True}), 201
|
||
|
||
|
||
@api_bp.route('/favorites/<int:favorite_id>', methods=['PUT'])
|
||
def update_favorite(favorite_id):
|
||
"""Update a favorite"""
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
favorites = get_favorites_manager()
|
||
favorites.update_favorite(favorite_id, **data)
|
||
|
||
return jsonify({'success': True})
|
||
|
||
|
||
@api_bp.route('/favorites/<int:favorite_id>', methods=['DELETE'])
|
||
def delete_favorite(favorite_id):
|
||
"""Delete a favorite"""
|
||
favorites = get_favorites_manager()
|
||
favorites.remove_favorite(favorite_id)
|
||
return jsonify({'success': True})
|
||
|
||
|
||
@api_bp.route('/favorites/suggestions')
|
||
def get_favorite_suggestions():
|
||
"""Get suggested categories for favorites"""
|
||
favorites = get_favorites_manager()
|
||
suggestions = favorites.get_suggested_categories()
|
||
return jsonify({'suggestions': suggestions})
|
||
|
||
|
||
# ==================== Scraper Control Endpoints ====================
|
||
|
||
@api_bp.route('/scrapers/state')
|
||
def get_scraper_state():
|
||
"""Get current scraper state"""
|
||
state = scraper_state.get_state()
|
||
return jsonify(state)
|
||
|
||
|
||
@api_bp.route('/scrapers/<scraper>/toggle', methods=['POST'])
|
||
def toggle_scraper(scraper):
|
||
"""Enable or disable a scraper"""
|
||
data = request.get_json() or {}
|
||
enabled = data.get('enabled')
|
||
|
||
if enabled is None:
|
||
# Toggle current state
|
||
current = scraper_state.get_state()
|
||
if scraper not in current['scrapers']:
|
||
return jsonify({'error': 'Unknown scraper'}), 404
|
||
enabled = not current['scrapers'][scraper]['enabled']
|
||
|
||
success = scraper_state.set_scraper_enabled(scraper, enabled)
|
||
|
||
if success:
|
||
return jsonify({'success': True, 'enabled': enabled})
|
||
return jsonify({'error': 'Failed to update scraper state'}), 400
|
||
|
||
|
||
@api_bp.route('/scrapers/interval', methods=['POST'])
|
||
def set_check_interval():
|
||
"""Set the check interval"""
|
||
data = request.get_json() or {}
|
||
interval = data.get('interval')
|
||
|
||
if not interval or not isinstance(interval, int):
|
||
return jsonify({'error': 'interval (int) required'}), 400
|
||
|
||
success = scraper_state.set_check_interval(interval)
|
||
|
||
if success:
|
||
return jsonify({'success': True, 'interval': interval})
|
||
return jsonify({'error': 'Interval must be at least 10 seconds'}), 400
|
||
|
||
|
||
@api_bp.route('/scrapers/start', methods=['POST'])
|
||
def start_monitor():
|
||
"""Start the stock monitor"""
|
||
result = scraper_state.start_monitor()
|
||
if result['success']:
|
||
return jsonify(result)
|
||
return jsonify(result), 400
|
||
|
||
|
||
@api_bp.route('/scrapers/stop', methods=['POST'])
|
||
def stop_monitor():
|
||
"""Stop the stock monitor"""
|
||
result = scraper_state.stop_monitor()
|
||
if result['success']:
|
||
return jsonify(result)
|
||
return jsonify(result), 400
|
||
|
||
|
||
# ==================== News Endpoints ====================
|
||
|
||
@api_bp.route('/news')
|
||
def get_news():
|
||
"""Get news articles with optional filters"""
|
||
db = get_database()
|
||
|
||
# Parse query params
|
||
source = request.args.get('source')
|
||
sentiment = request.args.get('sentiment')
|
||
drop_related = request.args.get('drop_related')
|
||
limit = int(request.args.get('limit', 50))
|
||
offset = int(request.args.get('offset', 0))
|
||
|
||
# Convert drop_related to bool if provided
|
||
if drop_related is not None:
|
||
drop_related = drop_related.lower() == 'true'
|
||
|
||
articles = db.get_news_articles(
|
||
source=source,
|
||
sentiment=sentiment,
|
||
drop_related=drop_related,
|
||
limit=limit,
|
||
offset=offset
|
||
)
|
||
|
||
return jsonify({
|
||
'articles': articles,
|
||
'total': len(articles) # For now, just return count of fetched
|
||
})
|
||
|
||
|
||
@api_bp.route('/news/stats')
|
||
def get_news_stats():
|
||
"""Get news statistics"""
|
||
db = get_database()
|
||
stats = db.get_news_stats()
|
||
return jsonify(stats)
|
||
|
||
|
||
@api_bp.route('/news/correlation')
|
||
def get_news_correlation():
|
||
"""Get news articles correlated with recent drops"""
|
||
db = get_database()
|
||
hours = int(request.args.get('hours', 24))
|
||
articles = db.get_correlated_news(hours=hours)
|
||
return jsonify({'articles': articles})
|
||
|
||
|
||
@api_bp.route('/news/refresh', methods=['POST'])
|
||
def refresh_news():
|
||
"""Manually trigger news fetch from all sources"""
|
||
from src.news.pokemon_fetcher import fetch_pokemon_news
|
||
from src.news.sentiment import analyze_article
|
||
|
||
db = get_database()
|
||
fetched = 0
|
||
errors = []
|
||
|
||
# Fetch Pokemon.com news
|
||
try:
|
||
articles = fetch_pokemon_news(limit=20)
|
||
for article in articles:
|
||
db.add_news_article(
|
||
source=article.source,
|
||
source_account=article.source_account,
|
||
external_id=article.external_id,
|
||
title=article.title,
|
||
content=article.content,
|
||
url=article.url,
|
||
author=article.author,
|
||
image_url=article.image_url,
|
||
published_at=article.published_at.isoformat() if article.published_at else None,
|
||
sentiment_score=article.sentiment_score,
|
||
sentiment_label=article.sentiment_label,
|
||
keywords=article.keywords,
|
||
is_drop_related=article.is_drop_related,
|
||
is_restock_related=article.is_restock_related
|
||
)
|
||
fetched += 1
|
||
except Exception as e:
|
||
errors.append(f"Pokemon.com: {str(e)}")
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'fetched': fetched,
|
||
'errors': errors
|
||
})
|
||
|
||
|
||
@api_bp.route('/news/manual', methods=['POST'])
|
||
def add_manual_news():
|
||
"""Add manually pasted news (Discord messages, etc.)"""
|
||
from src.news.sentiment import analyze_article
|
||
from datetime import datetime
|
||
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
content = data.get('content')
|
||
if not content:
|
||
return jsonify({'error': 'content is required'}), 400
|
||
|
||
source_account = data.get('source_account', 'Manual Entry')
|
||
url = data.get('url')
|
||
title = data.get('title')
|
||
|
||
# Analyze sentiment
|
||
analysis = analyze_article(content, title)
|
||
|
||
db = get_database()
|
||
article_id = db.add_news_article(
|
||
source='discord_manual',
|
||
source_account=source_account,
|
||
external_id=f"manual_{datetime.now().timestamp()}",
|
||
title=title,
|
||
content=content,
|
||
url=url,
|
||
author=source_account,
|
||
published_at=datetime.now().isoformat(),
|
||
sentiment_score=analysis['sentiment_score'],
|
||
sentiment_label=analysis['sentiment_label'],
|
||
keywords=analysis['keywords'],
|
||
is_drop_related=analysis['is_drop_related'],
|
||
is_restock_related=analysis['is_restock_related']
|
||
)
|
||
|
||
return jsonify({'id': article_id, 'success': True, 'analysis': analysis}), 201
|
||
|
||
|
||
@api_bp.route('/news/<int:article_id>', methods=['DELETE'])
|
||
def delete_news_article(article_id):
|
||
"""Delete a news article"""
|
||
db = get_database()
|
||
|
||
# Check if article exists
|
||
article = db.get_news_article(article_id)
|
||
if not article:
|
||
return jsonify({'error': 'Article not found'}), 404
|
||
|
||
# Delete from database
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
cursor.execute("DELETE FROM news_articles WHERE id = ?", (article_id,))
|
||
|
||
return jsonify({'success': True})
|
||
|
||
|
||
# ==================== Chrome Extension Integration ====================
|
||
|
||
# In-memory storage for extension data (persisted to file)
|
||
import json
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
EXTENSION_DATA_FILE = Path(__file__).parent.parent / "data" / "extension_data.json"
|
||
|
||
def load_extension_data():
|
||
"""Load extension data from file"""
|
||
if EXTENSION_DATA_FILE.exists():
|
||
try:
|
||
with open(EXTENSION_DATA_FILE, 'r') as f:
|
||
return json.load(f)
|
||
except:
|
||
pass
|
||
return {
|
||
'skus': [],
|
||
'products': [],
|
||
'api_stats': {},
|
||
'last_sync': None
|
||
}
|
||
|
||
def save_extension_data(data):
|
||
"""Save extension data to file"""
|
||
EXTENSION_DATA_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(EXTENSION_DATA_FILE, 'w') as f:
|
||
json.dump(data, f, indent=2)
|
||
|
||
extension_data = load_extension_data()
|
||
|
||
|
||
@api_bp.route('/extension/sync', methods=['POST'])
|
||
def sync_extension_data():
|
||
"""
|
||
Receive data from Chrome extension.
|
||
The extension should POST its known SKUs, products, events, and stats.
|
||
"""
|
||
global extension_data
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
db = get_database()
|
||
new_skus = []
|
||
events_processed = 0
|
||
|
||
# Merge SKUs (add new ones)
|
||
# NOTE: We only store SKUs for reference - we don't create products from bare SKUs
|
||
# because Pokemon Center URLs require a product slug that we don't have.
|
||
# Products are only created when we receive full product data with proper URLs.
|
||
if 'skus' in data:
|
||
existing_skus = set(extension_data.get('skus', []))
|
||
new_skus = [s for s in data['skus'] if s not in existing_skus]
|
||
extension_data['skus'] = list(existing_skus | set(data['skus']))
|
||
|
||
# Merge products - update existing or add new
|
||
if 'products' in data:
|
||
existing_urls = {p['url']: p for p in extension_data.get('products', [])}
|
||
for product in data['products']:
|
||
url = product.get('url')
|
||
if not url:
|
||
continue
|
||
|
||
# Update extension data store
|
||
if url in existing_urls:
|
||
# Update existing product data
|
||
existing_urls[url].update(product)
|
||
else:
|
||
extension_data['products'].append(product)
|
||
existing_urls[url] = product
|
||
|
||
# Sync to main database
|
||
product_id = db.get_or_create_product(
|
||
url=url,
|
||
name=product.get('name', 'Unknown'),
|
||
site=product.get('site', 'pokemoncenter'),
|
||
product_id=product.get('sku') or product.get('productId'),
|
||
image_url=product.get('imageUrl'),
|
||
price=product.get('price'),
|
||
in_stock=product.get('inStock', True)
|
||
)
|
||
|
||
# Update price if changed
|
||
if product.get('price') and product_id:
|
||
db.update_product_price(product_id, product['price'])
|
||
|
||
extension_data['products'] = list(existing_urls.values())
|
||
|
||
# Process events (restocks, new drops, price changes)
|
||
if 'events' in data:
|
||
for event in data['events']:
|
||
event_type = event.get('type')
|
||
url = event.get('url')
|
||
|
||
if not event_type or not url:
|
||
continue
|
||
|
||
product = db.get_product_by_url(url)
|
||
if not product:
|
||
# Create product if it doesn't exist
|
||
product_id = db.get_or_create_product(
|
||
url=url,
|
||
name=event.get('name', 'Unknown'),
|
||
site='pokemoncenter',
|
||
in_stock=event.get('inStock', True),
|
||
price=event.get('price')
|
||
)
|
||
else:
|
||
product_id = product['id']
|
||
|
||
if product_id:
|
||
if event_type == 'restock':
|
||
db.record_stock_event(product_id, 'restock')
|
||
# Update stock status
|
||
db.update_product_stock(product_id, True)
|
||
events_processed += 1
|
||
|
||
elif event_type == 'new_drop':
|
||
db.record_stock_event(product_id, 'new_drop')
|
||
events_processed += 1
|
||
|
||
elif event_type == 'out_of_stock':
|
||
db.record_stock_event(product_id, 'out_of_stock')
|
||
# Update stock status
|
||
db.update_product_stock(product_id, False)
|
||
events_processed += 1
|
||
|
||
elif event_type == 'price_change':
|
||
new_price = event.get('newPrice')
|
||
if new_price:
|
||
db.update_product_price(product_id, new_price)
|
||
events_processed += 1
|
||
|
||
# Update API stats
|
||
if 'apiStats' in data:
|
||
extension_data['api_stats'] = data['apiStats']
|
||
|
||
extension_data['last_sync'] = datetime.now().isoformat()
|
||
save_extension_data(extension_data)
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'total_skus': len(extension_data['skus']),
|
||
'total_products': len(extension_data['products']),
|
||
'new_skus_added': len(new_skus),
|
||
'events_processed': events_processed
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/skus', methods=['GET'])
|
||
def get_extension_skus():
|
||
"""Get all SKUs tracked by the extension"""
|
||
return jsonify({
|
||
'skus': extension_data.get('skus', []),
|
||
'total': len(extension_data.get('skus', []))
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/skus', methods=['POST'])
|
||
def add_extension_skus():
|
||
"""Add new SKUs from extension"""
|
||
global extension_data
|
||
data = request.get_json()
|
||
|
||
if not data or 'skus' not in data:
|
||
return jsonify({'error': 'skus array required'}), 400
|
||
|
||
existing = set(extension_data.get('skus', []))
|
||
new_skus = [s for s in data['skus'] if s not in existing]
|
||
|
||
extension_data['skus'] = list(existing | set(data['skus']))
|
||
extension_data['last_sync'] = datetime.now().isoformat()
|
||
save_extension_data(extension_data)
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'added': len(new_skus),
|
||
'total': len(extension_data['skus'])
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/products', methods=['GET'])
|
||
def get_extension_products():
|
||
"""Get all products tracked by the extension"""
|
||
return jsonify({
|
||
'products': extension_data.get('products', []),
|
||
'total': len(extension_data.get('products', []))
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/stats')
|
||
def get_extension_stats():
|
||
"""Get extension statistics for dashboard"""
|
||
return jsonify({
|
||
'total_skus': len(extension_data.get('skus', [])),
|
||
'total_products': len(extension_data.get('products', [])),
|
||
'api_stats': extension_data.get('api_stats', {}),
|
||
'last_sync': extension_data.get('last_sync'),
|
||
'source': 'chrome_extension'
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/broken-urls', methods=['GET'])
|
||
def get_broken_pokemoncenter_urls():
|
||
"""
|
||
Find Pokemon Center products with broken URLs.
|
||
Broken URLs are those without a product slug (just /product/{sku}).
|
||
"""
|
||
db = get_database()
|
||
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
# Find pokemoncenter products where URL ends with just the SKU (no slug)
|
||
# Valid URLs look like: /product/10-10022-102/product-name-here
|
||
# Invalid URLs look like: /product/10-10050 (no trailing slug)
|
||
cursor.execute("""
|
||
SELECT id, url, name, product_id, first_seen
|
||
FROM products
|
||
WHERE site = 'pokemoncenter'
|
||
AND url LIKE '%/product/%'
|
||
AND url NOT LIKE '%/product/%/%'
|
||
""")
|
||
broken = [dict(row) for row in cursor.fetchall()]
|
||
|
||
return jsonify({
|
||
'broken_urls': broken,
|
||
'count': len(broken)
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/broken-urls/cleanup', methods=['POST'])
|
||
def cleanup_broken_pokemoncenter_urls():
|
||
"""
|
||
Delete Pokemon Center products with broken URLs.
|
||
"""
|
||
db = get_database()
|
||
|
||
with db.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
# Delete products with broken URLs
|
||
cursor.execute("""
|
||
DELETE FROM products
|
||
WHERE site = 'pokemoncenter'
|
||
AND url LIKE '%/product/%'
|
||
AND url NOT LIKE '%/product/%/%'
|
||
""")
|
||
deleted = cursor.rowcount
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'deleted': deleted
|
||
})
|
||
|
||
|
||
@api_bp.route('/extension/clear', methods=['POST'])
|
||
def clear_extension_data():
|
||
"""Clear all extension data"""
|
||
global extension_data
|
||
extension_data = {
|
||
'skus': [],
|
||
'products': [],
|
||
'api_stats': {},
|
||
'last_sync': None
|
||
}
|
||
save_extension_data(extension_data)
|
||
return jsonify({'success': True})
|
||
|
||
|
||
# ==================== Check Trigger ====================
|
||
|
||
@api_bp.route('/check', methods=['POST'])
|
||
def trigger_check():
|
||
"""Trigger a manual stock check (placeholder - needs integration)"""
|
||
# This would need to communicate with the main monitor process
|
||
# For now, just return a message
|
||
return jsonify({
|
||
'message': 'Manual check triggered. Check monitor logs for results.',
|
||
'success': True
|
||
})
|
||
|
||
|
||
# ==================== User Management Endpoints ====================
|
||
|
||
@api_bp.route('/users', methods=['GET'])
|
||
def get_users():
|
||
"""Get all users"""
|
||
db = get_database()
|
||
users = db.get_all_users()
|
||
return jsonify({'users': users})
|
||
|
||
|
||
@api_bp.route('/users', methods=['POST'])
|
||
def create_user():
|
||
"""Create a new user"""
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
name = data.get('name')
|
||
if not name:
|
||
return jsonify({'error': 'name is required'}), 400
|
||
|
||
db = get_database()
|
||
|
||
# Check if user already exists
|
||
existing = db.get_user_by_name(name)
|
||
if existing:
|
||
return jsonify({'error': 'User with this name already exists'}), 400
|
||
|
||
user_id = db.create_user(
|
||
name=name,
|
||
zip_code=data.get('zip_code'),
|
||
radius_miles=data.get('radius_miles', 25),
|
||
discord_webhook=data.get('discord_webhook'),
|
||
notify_enabled=data.get('notify_enabled', True)
|
||
)
|
||
|
||
return jsonify({'id': user_id, 'success': True}), 201
|
||
|
||
|
||
@api_bp.route('/users/<int:user_id>', methods=['GET'])
|
||
def get_user(user_id):
|
||
"""Get a specific user"""
|
||
db = get_database()
|
||
user = db.get_user(user_id)
|
||
|
||
if not user:
|
||
return jsonify({'error': 'User not found'}), 404
|
||
|
||
return jsonify(user)
|
||
|
||
|
||
@api_bp.route('/users/<int:user_id>', methods=['PUT'])
|
||
def update_user(user_id):
|
||
"""Update a user"""
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
db = get_database()
|
||
|
||
# Check user exists
|
||
user = db.get_user(user_id)
|
||
if not user:
|
||
return jsonify({'error': 'User not found'}), 404
|
||
|
||
# If changing name, check for conflicts
|
||
if 'name' in data and data['name'] != user['name']:
|
||
existing = db.get_user_by_name(data['name'])
|
||
if existing:
|
||
return jsonify({'error': 'User with this name already exists'}), 400
|
||
|
||
success = db.update_user(user_id, **data)
|
||
return jsonify({'success': success})
|
||
|
||
|
||
@api_bp.route('/users/<int:user_id>', methods=['DELETE'])
|
||
def delete_user(user_id):
|
||
"""Delete a user"""
|
||
db = get_database()
|
||
|
||
user = db.get_user(user_id)
|
||
if not user:
|
||
return jsonify({'error': 'User not found'}), 404
|
||
|
||
success = db.delete_user(user_id)
|
||
return jsonify({'success': success})
|
||
|
||
|
||
@api_bp.route('/users/<int:user_id>/location', methods=['PUT'])
|
||
def update_user_location(user_id):
|
||
"""Update user's location settings"""
|
||
data = request.get_json()
|
||
|
||
if not data:
|
||
return jsonify({'error': 'No data provided'}), 400
|
||
|
||
zip_code = data.get('zip_code')
|
||
if not zip_code:
|
||
return jsonify({'error': 'zip_code is required'}), 400
|
||
|
||
db = get_database()
|
||
|
||
user = db.get_user(user_id)
|
||
if not user:
|
||
return jsonify({'error': 'User not found'}), 404
|
||
|
||
radius_miles = data.get('radius_miles', 25)
|
||
success = db.update_user_location(user_id, zip_code, radius_miles)
|
||
|
||
return jsonify({'success': success})
|
||
|
||
|
||
@api_bp.route('/users/<int:user_id>/stores', methods=['GET'])
|
||
def get_user_stores(user_id):
|
||
"""Get nearby stores for a user based on their location"""
|
||
from src.store_locator import find_all_nearby_stores
|
||
|
||
db = get_database()
|
||
user = db.get_user(user_id)
|
||
|
||
if not user:
|
||
return jsonify({'error': 'User not found'}), 404
|
||
|
||
if not user.get('zip_code'):
|
||
return jsonify({'error': 'User has no location set'}), 400
|
||
|
||
stores = find_all_nearby_stores(user['zip_code'], user.get('radius_miles', 25))
|
||
|
||
# Convert Store objects to dictionaries
|
||
result = {}
|
||
for retailer, store_list in stores.items():
|
||
result[retailer] = [
|
||
{
|
||
'name': s.name,
|
||
'address': s.address,
|
||
'city': s.city,
|
||
'state': s.state,
|
||
'zip_code': s.zip_code,
|
||
'distance_miles': s.distance_miles,
|
||
'phone': s.phone,
|
||
'store_id': s.store_id
|
||
}
|
||
for s in store_list
|
||
]
|
||
|
||
return jsonify({
|
||
'user': user['name'],
|
||
'location': {'zip_code': user['zip_code'], 'radius_miles': user.get('radius_miles', 25)},
|
||
'stores': result
|
||
})
|