feat: Implement Walmart scraper and integrate with existing architecture
- Added Walmart scraper to scrape product data from Walmart.com, including category pages and product details. - Introduced a stealth browser module to handle bot protection and improve scraping reliability. - Created a SQLite database for tracking product history, price changes, stock events, and user favorites. - Developed a Discord bot for user interaction, allowing location setting and stock checking at local stores. - Implemented a favorites system to manage priority products and categories with custom notification settings. - Added news aggregation module to fetch and analyze Pokemon TCG news from various sources. - Created tools for API discovery and monitoring, including a backend monitor for detecting new products. - Added unit tests for database operations, product filtering, and API endpoints to ensure functionality. - Enhanced existing modules with improved error handling and logging for better maintainability.
This commit is contained in:
@@ -35,6 +35,50 @@ class BaseScraper(ABC):
|
||||
|
||||
site_name: str = "unknown"
|
||||
|
||||
# Terms that indicate a Pokemon product
|
||||
POKEMON_TERMS = [
|
||||
"pokemon", "pokémon", "poke", "tcg",
|
||||
"pikachu", "charizard", "mewtwo", "eevee", "snorlax",
|
||||
"booster", "elite trainer", "etb",
|
||||
"scarlet", "violet", "prismatic", "evolutions",
|
||||
]
|
||||
|
||||
# Terms that indicate NOT a Pokemon product (false positives from search)
|
||||
EXCLUDE_TERMS = [
|
||||
"ice cube", "oven", "barbie", "hot wheels", "lego",
|
||||
"furniture", "appliance", "kitchen", "bedding",
|
||||
"glitter girls", "masters of the universe", "transformers",
|
||||
"room essentials", "threshold",
|
||||
]
|
||||
|
||||
def is_pokemon_product(self, product: Product) -> bool:
|
||||
"""
|
||||
Check if a product is actually a Pokemon product.
|
||||
Filters out false positives from search results.
|
||||
"""
|
||||
name_lower = product.name.lower()
|
||||
|
||||
# Check for exclusion terms first
|
||||
for term in self.EXCLUDE_TERMS:
|
||||
if term in name_lower:
|
||||
return False
|
||||
|
||||
# Check for Pokemon terms
|
||||
for term in self.POKEMON_TERMS:
|
||||
if term in name_lower:
|
||||
return True
|
||||
|
||||
# If no Pokemon terms found, reject it
|
||||
return False
|
||||
|
||||
def filter_pokemon_products(self, products: List[Product]) -> List[Product]:
|
||||
"""Filter to only include valid Pokemon products"""
|
||||
filtered = [p for p in products if self.is_pokemon_product(p)]
|
||||
rejected = len(products) - len(filtered)
|
||||
if rejected > 0:
|
||||
logger.info(f"Filtered out {rejected} non-Pokemon products")
|
||||
return filtered
|
||||
|
||||
@abstractmethod
|
||||
def scrape_category_page(self, url: str) -> List[Product]:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user