feat: Implement Walmart scraper and integrate with existing architecture

- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details.
- Introduced a stealth browser module to handle bot protection and improve scraping reliability.
- Created a SQLite database for tracking product history, price changes, stock events, and user favorites.
- Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores.
- Implemented a favorites system to manage priority products and categories with custom notification settings.
- Added news aggregation module to fetch and analyze Pokemon TCG news from various sources.
- Created tools for API discovery and monitoring, including a backend monitor for detecting new products.
- Added unit tests for database operations, product filtering, and API endpoints to ensure functionality.
- Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
2026-03-27 23:08:09 -04:00
parent cddae24e34
commit 8d382e723f
64 changed files with 15433 additions and 443 deletions
+44
View File
@@ -35,6 +35,50 @@ class BaseScraper(ABC):
site_name: str = "unknown"
# Terms that indicate a Pokemon product
POKEMON_TERMS = [
"pokemon", "pokémon", "poke", "tcg",
"pikachu", "charizard", "mewtwo", "eevee", "snorlax",
"booster", "elite trainer", "etb",
"scarlet", "violet", "prismatic", "evolutions",
]
# Terms that indicate NOT a Pokemon product (false positives from search)
EXCLUDE_TERMS = [
"ice cube", "oven", "barbie", "hot wheels", "lego",
"furniture", "appliance", "kitchen", "bedding",
"glitter girls", "masters of the universe", "transformers",
"room essentials", "threshold",
]
def is_pokemon_product(self, product: Product) -> bool:
"""
Check if a product is actually a Pokemon product.
Filters out false positives from search results.
"""
name_lower = product.name.lower()
# Check for exclusion terms first
for term in self.EXCLUDE_TERMS:
if term in name_lower:
return False
# Check for Pokemon terms
for term in self.POKEMON_TERMS:
if term in name_lower:
return True
# If no Pokemon terms found, reject it
return False
def filter_pokemon_products(self, products: List[Product]) -> List[Product]:
"""Filter to only include valid Pokemon products"""
filtered = [p for p in products if self.is_pokemon_product(p)]
rejected = len(products) - len(filtered)
if rejected > 0:
logger.info(f"Filtered out {rejected} non-Pokemon products")
return filtered
@abstractmethod
def scrape_category_page(self, url: str) -> List[Product]:
"""