Todd
375
modules/downloader_monitor.py
Normal file
375
modules/downloader_monitor.py
Normal file
@@ -0,0 +1,375 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Downloader Monitoring Module
|
||||
Tracks download success/failure and sends alerts when downloaders are consistently failing
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List
|
||||
from modules.universal_logger import get_logger
|
||||
|
||||
|
||||
class DownloaderMonitor:
|
||||
"""Monitor downloader health and send alerts on persistent failures"""
|
||||
|
||||
def __init__(self, unified_db=None, settings_manager=None):
|
||||
"""
|
||||
Initialize monitor
|
||||
|
||||
Args:
|
||||
unified_db: UnifiedDatabase instance
|
||||
settings_manager: SettingsManager instance for config
|
||||
"""
|
||||
self.db = unified_db
|
||||
self.settings_manager = settings_manager
|
||||
self.logger = get_logger('DownloaderMonitor')
|
||||
|
||||
# Default config
|
||||
self.config = {
|
||||
'enabled': True,
|
||||
'failure_window_hours': 3,
|
||||
'min_consecutive_failures': 2,
|
||||
'pushover': {
|
||||
'enabled': True,
|
||||
'priority': 1 # High priority
|
||||
},
|
||||
'downloaders': {
|
||||
'fastdl': True,
|
||||
'imginn': True,
|
||||
'toolzu': True,
|
||||
'instagram': True,
|
||||
'snapchat': True,
|
||||
'tiktok': True,
|
||||
'forums': True
|
||||
}
|
||||
}
|
||||
|
||||
# Load config from settings manager
|
||||
if self.settings_manager:
|
||||
try:
|
||||
monitoring_config = self.settings_manager.get('monitoring', {})
|
||||
if monitoring_config:
|
||||
self.config.update(monitoring_config)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not load monitoring config: {e}")
|
||||
|
||||
def log_download_attempt(self, downloader: str, username: str, success: bool,
|
||||
file_count: int = 0, error_message: str = None):
|
||||
"""
|
||||
Log a download attempt
|
||||
|
||||
Args:
|
||||
downloader: Downloader name (fastdl, imginn, toolzu, etc.)
|
||||
username: Username being downloaded
|
||||
success: Whether download succeeded
|
||||
file_count: Number of files downloaded
|
||||
error_message: Error message if failed
|
||||
"""
|
||||
if not self.config.get('enabled', True):
|
||||
return
|
||||
|
||||
# Check if this downloader is being monitored
|
||||
if not self.config.get('downloaders', {}).get(downloader, True):
|
||||
return
|
||||
|
||||
try:
|
||||
with self.db.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO download_monitor
|
||||
(downloader, username, timestamp, success, file_count, error_message, alert_sent)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 0)
|
||||
""", (
|
||||
downloader,
|
||||
username,
|
||||
datetime.now().isoformat(),
|
||||
1 if success else 0,
|
||||
file_count,
|
||||
error_message
|
||||
))
|
||||
conn.commit()
|
||||
|
||||
self.logger.debug(f"Logged {downloader}/{username}: {'success' if success else 'failure'} ({file_count} files)")
|
||||
|
||||
# Check if we should send an alert
|
||||
if not success:
|
||||
self._check_and_alert(downloader, username)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to log download attempt: {e}")
|
||||
|
||||
def _check_and_alert(self, downloader: str, username: str):
|
||||
"""
|
||||
Check if downloader has been failing consistently and send alert
|
||||
|
||||
Args:
|
||||
downloader: Downloader name
|
||||
username: Username
|
||||
"""
|
||||
try:
|
||||
window_hours = self.config.get('failure_window_hours', 3)
|
||||
min_failures = self.config.get('min_consecutive_failures', 2)
|
||||
|
||||
cutoff_time = datetime.now() - timedelta(hours=window_hours)
|
||||
|
||||
with self.db.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get recent attempts within the window
|
||||
cursor.execute("""
|
||||
SELECT timestamp, success, file_count, error_message, alert_sent
|
||||
FROM download_monitor
|
||||
WHERE downloader = ? AND username = ?
|
||||
AND timestamp > ?
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 10
|
||||
""", (downloader, username, cutoff_time.isoformat()))
|
||||
|
||||
attempts = cursor.fetchall()
|
||||
|
||||
if not attempts:
|
||||
return
|
||||
|
||||
# Count consecutive failures from most recent
|
||||
consecutive_failures = 0
|
||||
latest_error = None
|
||||
last_success_time = None
|
||||
|
||||
for attempt in attempts:
|
||||
if attempt['success'] == 0:
|
||||
consecutive_failures += 1
|
||||
if latest_error is None and attempt['error_message']:
|
||||
latest_error = attempt['error_message']
|
||||
else:
|
||||
last_success_time = attempt['timestamp']
|
||||
break
|
||||
|
||||
# Check if we should alert
|
||||
if consecutive_failures >= min_failures:
|
||||
# Check if we already sent an alert recently
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM download_monitor
|
||||
WHERE downloader = ? AND username = ?
|
||||
AND alert_sent = 1
|
||||
AND timestamp > ?
|
||||
""", (downloader, username, cutoff_time.isoformat()))
|
||||
|
||||
result = cursor.fetchone()
|
||||
alert_count = result[0] if result else 0
|
||||
|
||||
if alert_count == 0:
|
||||
# Send alert
|
||||
self._send_alert(
|
||||
downloader,
|
||||
username,
|
||||
consecutive_failures,
|
||||
last_success_time,
|
||||
latest_error
|
||||
)
|
||||
|
||||
# Mark most recent failure as alerted
|
||||
cursor.execute("""
|
||||
UPDATE download_monitor
|
||||
SET alert_sent = 1
|
||||
WHERE id = (
|
||||
SELECT id FROM download_monitor
|
||||
WHERE downloader = ? AND username = ?
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1
|
||||
)
|
||||
""", (downloader, username))
|
||||
conn.commit()
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to check for alerts: {e}")
|
||||
|
||||
def _send_alert(self, downloader: str, username: str, failure_count: int,
|
||||
last_success_time: str, error_message: str):
|
||||
"""
|
||||
Send Pushover alert for persistent failures
|
||||
|
||||
Args:
|
||||
downloader: Downloader name
|
||||
username: Username
|
||||
failure_count: Number of consecutive failures
|
||||
last_success_time: Timestamp of last success (ISO format)
|
||||
error_message: Latest error message
|
||||
"""
|
||||
if not self.config.get('pushover', {}).get('enabled', True):
|
||||
return
|
||||
|
||||
try:
|
||||
from modules.pushover_notifier import PushoverNotifier
|
||||
|
||||
# Get pushover config from settings
|
||||
pushover_config = {}
|
||||
if self.settings_manager:
|
||||
pushover_config = self.settings_manager.get('pushover', {})
|
||||
|
||||
if not pushover_config.get('enabled'):
|
||||
return
|
||||
|
||||
notifier = PushoverNotifier(
|
||||
api_token=pushover_config.get('api_token'),
|
||||
user_key=pushover_config.get('user_key')
|
||||
)
|
||||
|
||||
# Calculate time since last success
|
||||
time_since_success = "Never"
|
||||
if last_success_time:
|
||||
try:
|
||||
last_success = datetime.fromisoformat(last_success_time)
|
||||
delta = datetime.now() - last_success
|
||||
hours = int(delta.total_seconds() / 3600)
|
||||
if hours < 24:
|
||||
time_since_success = f"{hours} hours ago"
|
||||
else:
|
||||
days = hours // 24
|
||||
time_since_success = f"{days} days ago"
|
||||
except (ValueError, TypeError) as e:
|
||||
self.logger.warning(f"Failed to parse last_success_time '{last_success_time}': {e}")
|
||||
time_since_success = "Unknown (parse error)"
|
||||
|
||||
# Format downloader name nicely
|
||||
downloader_display = downloader.replace('_', ' ').title()
|
||||
|
||||
# Build message
|
||||
title = f"🚨 {downloader_display} Failing"
|
||||
message = f"""Downloader has been failing for {self.config.get('failure_window_hours', 3)}+ hours
|
||||
|
||||
Username: {username}
|
||||
Consecutive Failures: {failure_count}
|
||||
Last Success: {time_since_success}
|
||||
Latest Error: {error_message or 'Unknown'}
|
||||
|
||||
Check logs for details."""
|
||||
|
||||
# Send notification with high priority
|
||||
notifier.send_notification(
|
||||
title=title,
|
||||
message=message,
|
||||
priority=self.config.get('pushover', {}).get('priority', 1)
|
||||
)
|
||||
|
||||
self.logger.warning(f"Sent alert for {downloader}/{username} ({failure_count} failures)")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to send alert: {e}")
|
||||
|
||||
def get_downloader_status(self, downloader: str = None, hours: int = 24) -> List[Dict]:
|
||||
"""
|
||||
Get recent status for downloader(s)
|
||||
|
||||
Args:
|
||||
downloader: Specific downloader (None = all)
|
||||
hours: How many hours to look back
|
||||
|
||||
Returns:
|
||||
List of status dicts with stats per downloader
|
||||
"""
|
||||
try:
|
||||
cutoff = datetime.now() - timedelta(hours=hours)
|
||||
|
||||
with self.db.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if downloader:
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
downloader,
|
||||
COUNT(*) as total_attempts,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
||||
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
||||
SUM(file_count) as total_files,
|
||||
MAX(CASE WHEN success = 1 THEN timestamp END) as last_success,
|
||||
MAX(timestamp) as last_attempt
|
||||
FROM download_monitor
|
||||
WHERE downloader = ? AND timestamp > ?
|
||||
GROUP BY downloader
|
||||
""", (downloader, cutoff.isoformat()))
|
||||
else:
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
downloader,
|
||||
COUNT(*) as total_attempts,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
||||
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
||||
SUM(file_count) as total_files,
|
||||
MAX(CASE WHEN success = 1 THEN timestamp END) as last_success,
|
||||
MAX(timestamp) as last_attempt
|
||||
FROM download_monitor
|
||||
WHERE timestamp > ?
|
||||
GROUP BY downloader
|
||||
ORDER BY downloader
|
||||
""", (cutoff.isoformat(),))
|
||||
|
||||
results = []
|
||||
for row in cursor.fetchall():
|
||||
results.append({
|
||||
'downloader': row['downloader'],
|
||||
'total_attempts': row['total_attempts'],
|
||||
'successful': row['successful'] or 0,
|
||||
'failed': row['failed'] or 0,
|
||||
'total_files': row['total_files'] or 0,
|
||||
'success_rate': round((row['successful'] or 0) / row['total_attempts'] * 100, 1) if row['total_attempts'] > 0 else 0,
|
||||
'last_success': row['last_success'],
|
||||
'last_attempt': row['last_attempt']
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to get downloader status: {e}")
|
||||
return []
|
||||
|
||||
def clear_old_logs(self, days: int = 30):
|
||||
"""
|
||||
Clear monitoring logs older than specified days
|
||||
|
||||
Args:
|
||||
days: How many days to keep
|
||||
"""
|
||||
try:
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
|
||||
with self.db.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
DELETE FROM download_monitor
|
||||
WHERE timestamp < ?
|
||||
""", (cutoff.isoformat(),))
|
||||
deleted = cursor.rowcount
|
||||
conn.commit()
|
||||
|
||||
self.logger.info(f"Cleared {deleted} old monitoring logs (older than {days} days)")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to clear old logs: {e}")
|
||||
|
||||
|
||||
# Singleton instance with thread-safe initialization
|
||||
_monitor_instance = None
|
||||
_monitor_instance_lock = __import__('threading').Lock()
|
||||
|
||||
|
||||
def get_monitor(unified_db=None, settings_manager=None):
|
||||
"""Get or create monitor singleton (thread-safe)"""
|
||||
global _monitor_instance
|
||||
if _monitor_instance is None:
|
||||
with _monitor_instance_lock:
|
||||
# Double-check inside lock to prevent race condition
|
||||
if _monitor_instance is None:
|
||||
# Auto-initialize database if not provided
|
||||
if unified_db is None:
|
||||
from modules.unified_database import UnifiedDatabase
|
||||
unified_db = UnifiedDatabase()
|
||||
|
||||
# Auto-initialize settings manager if not provided
|
||||
if settings_manager is None:
|
||||
from modules.settings_manager import SettingsManager
|
||||
settings_manager = SettingsManager('/opt/media-downloader/database/media_downloader.db')
|
||||
|
||||
_monitor_instance = DownloaderMonitor(unified_db, settings_manager)
|
||||
return _monitor_instance
|
||||
Reference in New Issue
Block a user