feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
# Dashboard module
|
||||
from .app import run_dashboard
|
||||
|
||||
__all__ = ['run_dashboard']
|
||||
+1082
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
Flask dashboard for Pokemon Stock Monitor.
|
||||
Provides web UI for stats visualization and favorites management.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from flask import Flask, render_template, jsonify, request
|
||||
from flask_cors import CORS
|
||||
|
||||
from src.database import get_database
|
||||
from src.favorites import get_favorites_manager
|
||||
from .api import api_bp
|
||||
|
||||
# Create Flask app
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
# Register API blueprint
|
||||
app.register_blueprint(api_bp, url_prefix='/api')
|
||||
|
||||
|
||||
@app.route('/')
|
||||
def dashboard():
|
||||
"""Main dashboard page"""
|
||||
return render_template('index.html')
|
||||
|
||||
|
||||
@app.route('/products')
|
||||
def products_page():
|
||||
"""Products listing page"""
|
||||
return render_template('index.html', page='products')
|
||||
|
||||
|
||||
@app.route('/news')
|
||||
def news_page():
|
||||
"""News aggregation page"""
|
||||
return render_template('index.html', page='news')
|
||||
|
||||
|
||||
@app.route('/analytics')
|
||||
def analytics_page():
|
||||
"""Analytics page"""
|
||||
return render_template('index.html', page='analytics')
|
||||
|
||||
|
||||
@app.route('/favorites')
|
||||
def favorites_page():
|
||||
"""Favorites management page"""
|
||||
return render_template('index.html', page='favorites')
|
||||
|
||||
|
||||
@app.route('/settings')
|
||||
def settings_page():
|
||||
"""Settings page"""
|
||||
return render_template('index.html', page='settings')
|
||||
|
||||
|
||||
@app.route('/users')
|
||||
def users_page():
|
||||
"""Users management page"""
|
||||
return render_template('index.html', page='users')
|
||||
|
||||
|
||||
def run_dashboard(host: str = '0.0.0.0', port: int = 5000, debug: bool = False):
|
||||
"""Run the dashboard server"""
|
||||
print(f"Starting Pokemon Stock Monitor Dashboard on http://{host}:{port}")
|
||||
app.run(host=host, port=port, debug=debug, threaded=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_dashboard(debug=True)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,371 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Pokemon Stock Monitor</title>
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="app">
|
||||
<!-- Sidebar Navigation -->
|
||||
<nav class="sidebar">
|
||||
<div class="logo">
|
||||
<span class="logo-icon">⚡</span>
|
||||
<span class="logo-text">Pokemon Monitor</span>
|
||||
</div>
|
||||
<ul class="nav-links">
|
||||
<li><a href="#" data-page="dashboard" class="active">Dashboard</a></li>
|
||||
<li><a href="#" data-page="products">Products</a></li>
|
||||
<li><a href="#" data-page="news">News</a></li>
|
||||
<li><a href="#" data-page="analytics">Analytics</a></li>
|
||||
<li><a href="#" data-page="favorites">Favorites</a></li>
|
||||
<li><a href="#" data-page="control">Control Panel</a></li>
|
||||
<li><a href="#" data-page="users">Users</a></li>
|
||||
<li><a href="#" data-page="settings">Settings</a></li>
|
||||
</ul>
|
||||
<div class="sidebar-footer">
|
||||
<div class="status-indicator">
|
||||
<span class="status-dot" id="monitorStatus"></span>
|
||||
<span id="statusText">Loading...</span>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<!-- Main Content -->
|
||||
<main class="content">
|
||||
<!-- Dashboard Page -->
|
||||
<section id="page-dashboard" class="page active">
|
||||
<h1>Dashboard</h1>
|
||||
|
||||
<!-- Store Cards -->
|
||||
<div class="store-cards" id="storeCards">
|
||||
<!-- Populated by JavaScript -->
|
||||
</div>
|
||||
|
||||
<div class="stats-grid">
|
||||
<div class="stat-card clickable" onclick="goToFilteredProducts('new_drop', 'today')">
|
||||
<div class="stat-value" id="newDropsToday">-</div>
|
||||
<div class="stat-label">New Drops Today</div>
|
||||
</div>
|
||||
<div class="stat-card clickable" onclick="goToFilteredProducts('restock', 'today')">
|
||||
<div class="stat-value" id="restocksToday">-</div>
|
||||
<div class="stat-label">Restocks Today</div>
|
||||
</div>
|
||||
<div class="stat-card clickable" onclick="goToFilteredProducts('new_drop', 'week')">
|
||||
<div class="stat-value" id="newDropsWeek">-</div>
|
||||
<div class="stat-label">New Drops (7 days)</div>
|
||||
</div>
|
||||
<div class="stat-card clickable" onclick="goToFilteredProducts('restock', 'week')">
|
||||
<div class="stat-value" id="restocksWeek">-</div>
|
||||
<div class="stat-label">Restocks (7 days)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>Recent Activity</h2>
|
||||
<div class="activity-feed" id="activityFeed">
|
||||
<div class="loading">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Products Page -->
|
||||
<section id="page-products" class="page">
|
||||
<h1>Products</h1>
|
||||
|
||||
<div id="eventFilterBadge" class="event-filter-badge" style="display: none;"></div>
|
||||
|
||||
<div class="filters">
|
||||
<select id="filterSite">
|
||||
<option value="">All Sites</option>
|
||||
<option value="pokemoncenter">PokemonCenter</option>
|
||||
<option value="target">Target</option>
|
||||
</select>
|
||||
<select id="filterCategory">
|
||||
<option value="">All Categories</option>
|
||||
<option value="ETB">ETB</option>
|
||||
<option value="Booster Bundle">Booster Bundle</option>
|
||||
<option value="Booster Box">Booster Box</option>
|
||||
<option value="Booster Pack">Booster Pack</option>
|
||||
<option value="Collection Box">Collection Box</option>
|
||||
<option value="Tin">Tin</option>
|
||||
</select>
|
||||
<select id="filterStock">
|
||||
<option value="">All Stock Status</option>
|
||||
<option value="true">In Stock</option>
|
||||
<option value="false">Out of Stock</option>
|
||||
</select>
|
||||
<label class="checkbox-label">
|
||||
<input type="checkbox" id="filterFavorites">
|
||||
Favorites Only
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="products-grid" id="productsGrid">
|
||||
<div class="loading">Loading products...</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Analytics Page -->
|
||||
<section id="page-analytics" class="page">
|
||||
<h1>Analytics</h1>
|
||||
|
||||
<div class="charts-grid">
|
||||
<div class="chart-card">
|
||||
<h3>Drop Timing (by hour)</h3>
|
||||
<canvas id="dropTimingChart"></canvas>
|
||||
</div>
|
||||
<div class="chart-card">
|
||||
<h3>Stock Duration by Category</h3>
|
||||
<canvas id="stockDurationChart"></canvas>
|
||||
</div>
|
||||
<div class="chart-card">
|
||||
<h3>Products by Site</h3>
|
||||
<canvas id="siteChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- News Page -->
|
||||
<section id="page-news" class="page">
|
||||
<h1>News</h1>
|
||||
|
||||
<!-- News Stats Overview -->
|
||||
<div class="news-stats-bar" id="newsStatsBar">
|
||||
<div class="sentiment-overview">
|
||||
<span class="sentiment-label">Sentiment:</span>
|
||||
<span class="sentiment-bar">
|
||||
<span class="sentiment-positive" id="sentimentPositiveBar"></span>
|
||||
<span class="sentiment-neutral" id="sentimentNeutralBar"></span>
|
||||
<span class="sentiment-negative" id="sentimentNegativeBar"></span>
|
||||
</span>
|
||||
<span class="sentiment-text" id="sentimentText">Loading...</span>
|
||||
</div>
|
||||
<div class="news-count">
|
||||
<span id="todayArticles">0</span> articles today
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- News Filters -->
|
||||
<div class="filters news-filters">
|
||||
<select id="filterNewsSource">
|
||||
<option value="">All Sources</option>
|
||||
<option value="twitter">Twitter</option>
|
||||
<option value="pokemon_official">Pokemon.com</option>
|
||||
<option value="discord_manual">Discord (Manual)</option>
|
||||
</select>
|
||||
<select id="filterNewsSentiment">
|
||||
<option value="">All Sentiment</option>
|
||||
<option value="positive">Positive</option>
|
||||
<option value="neutral">Neutral</option>
|
||||
<option value="negative">Negative</option>
|
||||
</select>
|
||||
<label class="checkbox-label">
|
||||
<input type="checkbox" id="filterDropRelated">
|
||||
Drop/Restock Related Only
|
||||
</label>
|
||||
<button id="refreshNewsBtn" class="btn-secondary">
|
||||
<span class="btn-icon">↻</span> Refresh News
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Manual News Input -->
|
||||
<div class="manual-news-section">
|
||||
<details>
|
||||
<summary>Add Manual News (Discord messages, etc.)</summary>
|
||||
<div class="manual-news-form">
|
||||
<div class="form-row">
|
||||
<select id="manualNewsSource">
|
||||
<option value="Poke Pullz Discord">Poke Pullz Discord</option>
|
||||
<option value="PokePings Discord">PokePings Discord</option>
|
||||
<option value="Other">Other</option>
|
||||
</select>
|
||||
<input type="text" id="manualNewsTitle" placeholder="Title (optional)">
|
||||
</div>
|
||||
<textarea id="manualNewsContent" placeholder="Paste the message content here..." rows="3"></textarea>
|
||||
<div class="form-row">
|
||||
<input type="text" id="manualNewsUrl" placeholder="URL (optional)">
|
||||
<button id="addManualNewsBtn" class="btn-primary">Add News</button>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<!-- News Feed -->
|
||||
<div class="news-feed" id="newsFeed">
|
||||
<div class="loading">Loading news...</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Favorites Page -->
|
||||
<section id="page-favorites" class="page">
|
||||
<h1>Favorites</h1>
|
||||
|
||||
<div class="favorites-section">
|
||||
<div class="add-favorite-form">
|
||||
<h3>Add New Favorite</h3>
|
||||
<div class="form-row">
|
||||
<select id="favType">
|
||||
<option value="category">Category/Set</option>
|
||||
<option value="product">Product URL</option>
|
||||
</select>
|
||||
<input type="text" id="favValue" placeholder="e.g., ETB, Chaos Rising, or product URL">
|
||||
<select id="favPriority">
|
||||
<option value="high">High Priority</option>
|
||||
<option value="medium">Medium Priority</option>
|
||||
<option value="low">Low Priority</option>
|
||||
</select>
|
||||
<button id="addFavoriteBtn" class="btn-primary">Add Favorite</button>
|
||||
</div>
|
||||
<div class="suggestions" id="suggestions"></div>
|
||||
</div>
|
||||
|
||||
<h3>Your Favorites</h3>
|
||||
<div class="favorites-list" id="favoritesList">
|
||||
<div class="loading">Loading favorites...</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Control Panel Page -->
|
||||
<section id="page-control" class="page">
|
||||
<h1>Control Panel</h1>
|
||||
|
||||
<div class="control-panel">
|
||||
<!-- Monitor Status Card -->
|
||||
<div class="control-card monitor-status-card">
|
||||
<div class="control-card-header">
|
||||
<h3>Monitor Status</h3>
|
||||
<span class="monitor-badge" id="monitorBadge">Stopped</span>
|
||||
</div>
|
||||
<div class="control-card-body">
|
||||
<p id="monitorStatusText">Monitor is not running</p>
|
||||
<p class="last-check-text">Last check: <span id="controlLastCheck">Never</span></p>
|
||||
</div>
|
||||
<div class="control-card-actions">
|
||||
<button id="startMonitorBtn" class="btn-primary btn-large">
|
||||
<span class="btn-icon">▶</span> Start Monitor
|
||||
</button>
|
||||
<button id="stopMonitorBtn" class="btn-danger btn-large" disabled>
|
||||
<span class="btn-icon">■</span> Stop Monitor
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Check Interval -->
|
||||
<div class="control-card">
|
||||
<div class="control-card-header">
|
||||
<h3>Check Interval</h3>
|
||||
</div>
|
||||
<div class="control-card-body">
|
||||
<div class="interval-input">
|
||||
<input type="number" id="checkIntervalInput" min="10" max="600" value="60">
|
||||
<span>seconds</span>
|
||||
<button id="saveIntervalBtn" class="btn-secondary">Save</button>
|
||||
</div>
|
||||
<p class="help-text">How often to check for new products (minimum 10 seconds)</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scraper Controls -->
|
||||
<div class="control-card scrapers-card">
|
||||
<div class="control-card-header">
|
||||
<h3>Scrapers</h3>
|
||||
</div>
|
||||
<div class="control-card-body">
|
||||
<div class="scrapers-grid" id="scrapersGrid">
|
||||
<!-- Populated by JavaScript -->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Quick Actions -->
|
||||
<div class="control-card">
|
||||
<div class="control-card-header">
|
||||
<h3>Quick Actions</h3>
|
||||
</div>
|
||||
<div class="control-card-body">
|
||||
<button id="runCheckNowBtn" class="btn-secondary">
|
||||
<span class="btn-icon">↻</span> Run Check Now
|
||||
</button>
|
||||
<button id="viewLogsBtn" class="btn-secondary">
|
||||
<span class="btn-icon">📄</span> View Logs
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Users Page -->
|
||||
<section id="page-users" class="page">
|
||||
<h1>Users</h1>
|
||||
<p class="help-text">Manage users and their store locations for local stock checking</p>
|
||||
|
||||
<div class="users-section">
|
||||
<div class="add-user-form">
|
||||
<h3>Add New User</h3>
|
||||
<div class="form-group">
|
||||
<label for="newUserName">Name</label>
|
||||
<input type="text" id="newUserName" placeholder="e.g., John">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="newUserZip">Zip Code</label>
|
||||
<input type="text" id="newUserZip" placeholder="e.g., 90210" maxlength="10">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="newUserRadius">Search Radius (miles)</label>
|
||||
<input type="number" id="newUserRadius" value="25" min="5" max="100">
|
||||
</div>
|
||||
<button id="addUserBtn" class="btn-primary">Add User</button>
|
||||
</div>
|
||||
|
||||
<h3>Current Users</h3>
|
||||
<div id="usersList" class="users-list">
|
||||
<!-- Populated by JavaScript -->
|
||||
<p class="loading">Loading users...</p>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Settings Page -->
|
||||
<section id="page-settings" class="page">
|
||||
<h1>Settings</h1>
|
||||
|
||||
<div class="settings-section">
|
||||
<h3>Discord Webhook</h3>
|
||||
<p class="help-text">Configure your Discord webhook URL in config.py</p>
|
||||
|
||||
<h3>Monitor Status</h3>
|
||||
<div id="monitorInfo">
|
||||
<p>Last check: <span id="lastCheckTime">-</span></p>
|
||||
<p>Next check: <span id="nextCheckTime">-</span></p>
|
||||
</div>
|
||||
|
||||
<h3>Data Management</h3>
|
||||
<button id="migrateBtn" class="btn-secondary">Migrate products.json to Database</button>
|
||||
<button id="exportBtn" class="btn-secondary">Export Data</button>
|
||||
|
||||
<h3>Pokemon Center Cleanup</h3>
|
||||
<p class="help-text">Remove Pokemon Center products with broken URLs (SKUs without product slugs)</p>
|
||||
<button id="checkBrokenUrlsBtn" class="btn-secondary">Check Broken URLs</button>
|
||||
<button id="cleanupBrokenUrlsBtn" class="btn-danger">Cleanup Broken URLs</button>
|
||||
<p id="brokenUrlsResult" class="help-text"></p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<!-- Add Favorite Modal -->
|
||||
<div class="modal" id="productModal">
|
||||
<div class="modal-content">
|
||||
<span class="modal-close">×</span>
|
||||
<h2 id="modalProductName">Product Details</h2>
|
||||
<div id="modalProductDetails"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="{{ url_for('static', filename='app.js') }}"></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user