#!/usr/bin/env python3 """ Downloader Monitoring Module Tracks download success/failure and sends alerts when downloaders are consistently failing """ import sqlite3 from datetime import datetime, timedelta from pathlib import Path from typing import Optional, Dict, List from modules.universal_logger import get_logger class DownloaderMonitor: """Monitor downloader health and send alerts on persistent failures""" def __init__(self, unified_db=None, settings_manager=None): """ Initialize monitor Args: unified_db: UnifiedDatabase instance settings_manager: SettingsManager instance for config """ self.db = unified_db self.settings_manager = settings_manager self.logger = get_logger('DownloaderMonitor') # Default config self.config = { 'enabled': True, 'failure_window_hours': 3, 'min_consecutive_failures': 2, 'pushover': { 'enabled': True, 'priority': 1 # High priority }, 'downloaders': { 'fastdl': True, 'imginn': True, 'toolzu': True, 'instagram': True, 'snapchat': True, 'tiktok': True, 'forums': True } } # Load config from settings manager if self.settings_manager: try: monitoring_config = self.settings_manager.get('monitoring', {}) if monitoring_config: self.config.update(monitoring_config) except Exception as e: self.logger.warning(f"Could not load monitoring config: {e}") def log_download_attempt(self, downloader: str, username: str, success: bool, file_count: int = 0, error_message: str = None): """ Log a download attempt Args: downloader: Downloader name (fastdl, imginn, toolzu, etc.) username: Username being downloaded success: Whether download succeeded file_count: Number of files downloaded error_message: Error message if failed """ if not self.config.get('enabled', True): return # Check if this downloader is being monitored if not self.config.get('downloaders', {}).get(downloader, True): return try: with self.db.get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT INTO download_monitor (downloader, username, timestamp, success, file_count, error_message, alert_sent) VALUES (?, ?, ?, ?, ?, ?, 0) """, ( downloader, username, datetime.now().isoformat(), 1 if success else 0, file_count, error_message )) conn.commit() self.logger.debug(f"Logged {downloader}/{username}: {'success' if success else 'failure'} ({file_count} files)") # Check if we should send an alert if not success: self._check_and_alert(downloader, username) except Exception as e: self.logger.error(f"Failed to log download attempt: {e}") def _check_and_alert(self, downloader: str, username: str): """ Check if downloader has been failing consistently and send alert Args: downloader: Downloader name username: Username """ try: window_hours = self.config.get('failure_window_hours', 3) min_failures = self.config.get('min_consecutive_failures', 2) cutoff_time = datetime.now() - timedelta(hours=window_hours) with self.db.get_connection() as conn: cursor = conn.cursor() # Get recent attempts within the window cursor.execute(""" SELECT timestamp, success, file_count, error_message, alert_sent FROM download_monitor WHERE downloader = ? AND username = ? AND timestamp > ? ORDER BY timestamp DESC LIMIT 10 """, (downloader, username, cutoff_time.isoformat())) attempts = cursor.fetchall() if not attempts: return # Count consecutive failures from most recent consecutive_failures = 0 latest_error = None last_success_time = None for attempt in attempts: if attempt['success'] == 0: consecutive_failures += 1 if latest_error is None and attempt['error_message']: latest_error = attempt['error_message'] else: last_success_time = attempt['timestamp'] break # Check if we should alert if consecutive_failures >= min_failures: # Check if we already sent an alert recently cursor.execute(""" SELECT COUNT(*) FROM download_monitor WHERE downloader = ? AND username = ? AND alert_sent = 1 AND timestamp > ? """, (downloader, username, cutoff_time.isoformat())) result = cursor.fetchone() alert_count = result[0] if result else 0 if alert_count == 0: # Send alert self._send_alert( downloader, username, consecutive_failures, last_success_time, latest_error ) # Mark most recent failure as alerted cursor.execute(""" UPDATE download_monitor SET alert_sent = 1 WHERE id = ( SELECT id FROM download_monitor WHERE downloader = ? AND username = ? ORDER BY timestamp DESC LIMIT 1 ) """, (downloader, username)) conn.commit() except Exception as e: self.logger.error(f"Failed to check for alerts: {e}") def _send_alert(self, downloader: str, username: str, failure_count: int, last_success_time: str, error_message: str): """ Send Pushover alert for persistent failures Args: downloader: Downloader name username: Username failure_count: Number of consecutive failures last_success_time: Timestamp of last success (ISO format) error_message: Latest error message """ if not self.config.get('pushover', {}).get('enabled', True): return try: from modules.pushover_notifier import PushoverNotifier # Get pushover config from settings pushover_config = {} if self.settings_manager: pushover_config = self.settings_manager.get('pushover', {}) if not pushover_config.get('enabled'): return notifier = PushoverNotifier( api_token=pushover_config.get('api_token'), user_key=pushover_config.get('user_key') ) # Calculate time since last success time_since_success = "Never" if last_success_time: try: last_success = datetime.fromisoformat(last_success_time) delta = datetime.now() - last_success hours = int(delta.total_seconds() / 3600) if hours < 24: time_since_success = f"{hours} hours ago" else: days = hours // 24 time_since_success = f"{days} days ago" except (ValueError, TypeError) as e: self.logger.warning(f"Failed to parse last_success_time '{last_success_time}': {e}") time_since_success = "Unknown (parse error)" # Format downloader name nicely downloader_display = downloader.replace('_', ' ').title() # Build message title = f"🚨 {downloader_display} Failing" message = f"""Downloader has been failing for {self.config.get('failure_window_hours', 3)}+ hours Username: {username} Consecutive Failures: {failure_count} Last Success: {time_since_success} Latest Error: {error_message or 'Unknown'} Check logs for details.""" # Send notification with high priority notifier.send_notification( title=title, message=message, priority=self.config.get('pushover', {}).get('priority', 1) ) self.logger.warning(f"Sent alert for {downloader}/{username} ({failure_count} failures)") except Exception as e: self.logger.error(f"Failed to send alert: {e}") def get_downloader_status(self, downloader: str = None, hours: int = 24) -> List[Dict]: """ Get recent status for downloader(s) Args: downloader: Specific downloader (None = all) hours: How many hours to look back Returns: List of status dicts with stats per downloader """ try: cutoff = datetime.now() - timedelta(hours=hours) with self.db.get_connection() as conn: cursor = conn.cursor() if downloader: cursor.execute(""" SELECT downloader, COUNT(*) as total_attempts, SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful, SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed, SUM(file_count) as total_files, MAX(CASE WHEN success = 1 THEN timestamp END) as last_success, MAX(timestamp) as last_attempt FROM download_monitor WHERE downloader = ? AND timestamp > ? GROUP BY downloader """, (downloader, cutoff.isoformat())) else: cursor.execute(""" SELECT downloader, COUNT(*) as total_attempts, SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful, SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed, SUM(file_count) as total_files, MAX(CASE WHEN success = 1 THEN timestamp END) as last_success, MAX(timestamp) as last_attempt FROM download_monitor WHERE timestamp > ? GROUP BY downloader ORDER BY downloader """, (cutoff.isoformat(),)) results = [] for row in cursor.fetchall(): results.append({ 'downloader': row['downloader'], 'total_attempts': row['total_attempts'], 'successful': row['successful'] or 0, 'failed': row['failed'] or 0, 'total_files': row['total_files'] or 0, 'success_rate': round((row['successful'] or 0) / row['total_attempts'] * 100, 1) if row['total_attempts'] > 0 else 0, 'last_success': row['last_success'], 'last_attempt': row['last_attempt'] }) return results except Exception as e: self.logger.error(f"Failed to get downloader status: {e}") return [] def clear_old_logs(self, days: int = 30): """ Clear monitoring logs older than specified days Args: days: How many days to keep """ try: cutoff = datetime.now() - timedelta(days=days) with self.db.get_connection() as conn: cursor = conn.cursor() cursor.execute(""" DELETE FROM download_monitor WHERE timestamp < ? """, (cutoff.isoformat(),)) deleted = cursor.rowcount conn.commit() self.logger.info(f"Cleared {deleted} old monitoring logs (older than {days} days)") except Exception as e: self.logger.error(f"Failed to clear old logs: {e}") # Singleton instance with thread-safe initialization _monitor_instance = None _monitor_instance_lock = __import__('threading').Lock() def get_monitor(unified_db=None, settings_manager=None): """Get or create monitor singleton (thread-safe)""" global _monitor_instance if _monitor_instance is None: with _monitor_instance_lock: # Double-check inside lock to prevent race condition if _monitor_instance is None: # Auto-initialize database if not provided if unified_db is None: from modules.unified_database import UnifiedDatabase unified_db = UnifiedDatabase() # Auto-initialize settings manager if not provided if settings_manager is None: from modules.settings_manager import SettingsManager settings_manager = SettingsManager('/opt/media-downloader/database/media_downloader.db') _monitor_instance = DownloaderMonitor(unified_db, settings_manager) return _monitor_instance