376 lines
14 KiB
Python
376 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Downloader Monitoring Module
|
|
Tracks download success/failure and sends alerts when downloaders are consistently failing
|
|
"""
|
|
|
|
import sqlite3
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Optional, Dict, List
|
|
from modules.universal_logger import get_logger
|
|
|
|
|
|
class DownloaderMonitor:
|
|
"""Monitor downloader health and send alerts on persistent failures"""
|
|
|
|
def __init__(self, unified_db=None, settings_manager=None):
|
|
"""
|
|
Initialize monitor
|
|
|
|
Args:
|
|
unified_db: UnifiedDatabase instance
|
|
settings_manager: SettingsManager instance for config
|
|
"""
|
|
self.db = unified_db
|
|
self.settings_manager = settings_manager
|
|
self.logger = get_logger('DownloaderMonitor')
|
|
|
|
# Default config
|
|
self.config = {
|
|
'enabled': True,
|
|
'failure_window_hours': 3,
|
|
'min_consecutive_failures': 2,
|
|
'pushover': {
|
|
'enabled': True,
|
|
'priority': 1 # High priority
|
|
},
|
|
'downloaders': {
|
|
'fastdl': True,
|
|
'imginn': True,
|
|
'toolzu': True,
|
|
'instagram': True,
|
|
'snapchat': True,
|
|
'tiktok': True,
|
|
'forums': True
|
|
}
|
|
}
|
|
|
|
# Load config from settings manager
|
|
if self.settings_manager:
|
|
try:
|
|
monitoring_config = self.settings_manager.get('monitoring', {})
|
|
if monitoring_config:
|
|
self.config.update(monitoring_config)
|
|
except Exception as e:
|
|
self.logger.warning(f"Could not load monitoring config: {e}")
|
|
|
|
def log_download_attempt(self, downloader: str, username: str, success: bool,
|
|
file_count: int = 0, error_message: str = None):
|
|
"""
|
|
Log a download attempt
|
|
|
|
Args:
|
|
downloader: Downloader name (fastdl, imginn, toolzu, etc.)
|
|
username: Username being downloaded
|
|
success: Whether download succeeded
|
|
file_count: Number of files downloaded
|
|
error_message: Error message if failed
|
|
"""
|
|
if not self.config.get('enabled', True):
|
|
return
|
|
|
|
# Check if this downloader is being monitored
|
|
if not self.config.get('downloaders', {}).get(downloader, True):
|
|
return
|
|
|
|
try:
|
|
with self.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT INTO download_monitor
|
|
(downloader, username, timestamp, success, file_count, error_message, alert_sent)
|
|
VALUES (?, ?, ?, ?, ?, ?, 0)
|
|
""", (
|
|
downloader,
|
|
username,
|
|
datetime.now().isoformat(),
|
|
1 if success else 0,
|
|
file_count,
|
|
error_message
|
|
))
|
|
conn.commit()
|
|
|
|
self.logger.debug(f"Logged {downloader}/{username}: {'success' if success else 'failure'} ({file_count} files)")
|
|
|
|
# Check if we should send an alert
|
|
if not success:
|
|
self._check_and_alert(downloader, username)
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to log download attempt: {e}")
|
|
|
|
def _check_and_alert(self, downloader: str, username: str):
|
|
"""
|
|
Check if downloader has been failing consistently and send alert
|
|
|
|
Args:
|
|
downloader: Downloader name
|
|
username: Username
|
|
"""
|
|
try:
|
|
window_hours = self.config.get('failure_window_hours', 3)
|
|
min_failures = self.config.get('min_consecutive_failures', 2)
|
|
|
|
cutoff_time = datetime.now() - timedelta(hours=window_hours)
|
|
|
|
with self.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Get recent attempts within the window
|
|
cursor.execute("""
|
|
SELECT timestamp, success, file_count, error_message, alert_sent
|
|
FROM download_monitor
|
|
WHERE downloader = ? AND username = ?
|
|
AND timestamp > ?
|
|
ORDER BY timestamp DESC
|
|
LIMIT 10
|
|
""", (downloader, username, cutoff_time.isoformat()))
|
|
|
|
attempts = cursor.fetchall()
|
|
|
|
if not attempts:
|
|
return
|
|
|
|
# Count consecutive failures from most recent
|
|
consecutive_failures = 0
|
|
latest_error = None
|
|
last_success_time = None
|
|
|
|
for attempt in attempts:
|
|
if attempt['success'] == 0:
|
|
consecutive_failures += 1
|
|
if latest_error is None and attempt['error_message']:
|
|
latest_error = attempt['error_message']
|
|
else:
|
|
last_success_time = attempt['timestamp']
|
|
break
|
|
|
|
# Check if we should alert
|
|
if consecutive_failures >= min_failures:
|
|
# Check if we already sent an alert recently
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM download_monitor
|
|
WHERE downloader = ? AND username = ?
|
|
AND alert_sent = 1
|
|
AND timestamp > ?
|
|
""", (downloader, username, cutoff_time.isoformat()))
|
|
|
|
result = cursor.fetchone()
|
|
alert_count = result[0] if result else 0
|
|
|
|
if alert_count == 0:
|
|
# Send alert
|
|
self._send_alert(
|
|
downloader,
|
|
username,
|
|
consecutive_failures,
|
|
last_success_time,
|
|
latest_error
|
|
)
|
|
|
|
# Mark most recent failure as alerted
|
|
cursor.execute("""
|
|
UPDATE download_monitor
|
|
SET alert_sent = 1
|
|
WHERE id = (
|
|
SELECT id FROM download_monitor
|
|
WHERE downloader = ? AND username = ?
|
|
ORDER BY timestamp DESC
|
|
LIMIT 1
|
|
)
|
|
""", (downloader, username))
|
|
conn.commit()
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to check for alerts: {e}")
|
|
|
|
def _send_alert(self, downloader: str, username: str, failure_count: int,
|
|
last_success_time: str, error_message: str):
|
|
"""
|
|
Send Pushover alert for persistent failures
|
|
|
|
Args:
|
|
downloader: Downloader name
|
|
username: Username
|
|
failure_count: Number of consecutive failures
|
|
last_success_time: Timestamp of last success (ISO format)
|
|
error_message: Latest error message
|
|
"""
|
|
if not self.config.get('pushover', {}).get('enabled', True):
|
|
return
|
|
|
|
try:
|
|
from modules.pushover_notifier import PushoverNotifier
|
|
|
|
# Get pushover config from settings
|
|
pushover_config = {}
|
|
if self.settings_manager:
|
|
pushover_config = self.settings_manager.get('pushover', {})
|
|
|
|
if not pushover_config.get('enabled'):
|
|
return
|
|
|
|
notifier = PushoverNotifier(
|
|
api_token=pushover_config.get('api_token'),
|
|
user_key=pushover_config.get('user_key')
|
|
)
|
|
|
|
# Calculate time since last success
|
|
time_since_success = "Never"
|
|
if last_success_time:
|
|
try:
|
|
last_success = datetime.fromisoformat(last_success_time)
|
|
delta = datetime.now() - last_success
|
|
hours = int(delta.total_seconds() / 3600)
|
|
if hours < 24:
|
|
time_since_success = f"{hours} hours ago"
|
|
else:
|
|
days = hours // 24
|
|
time_since_success = f"{days} days ago"
|
|
except (ValueError, TypeError) as e:
|
|
self.logger.warning(f"Failed to parse last_success_time '{last_success_time}': {e}")
|
|
time_since_success = "Unknown (parse error)"
|
|
|
|
# Format downloader name nicely
|
|
downloader_display = downloader.replace('_', ' ').title()
|
|
|
|
# Build message
|
|
title = f"🚨 {downloader_display} Failing"
|
|
message = f"""Downloader has been failing for {self.config.get('failure_window_hours', 3)}+ hours
|
|
|
|
Username: {username}
|
|
Consecutive Failures: {failure_count}
|
|
Last Success: {time_since_success}
|
|
Latest Error: {error_message or 'Unknown'}
|
|
|
|
Check logs for details."""
|
|
|
|
# Send notification with high priority
|
|
notifier.send_notification(
|
|
title=title,
|
|
message=message,
|
|
priority=self.config.get('pushover', {}).get('priority', 1)
|
|
)
|
|
|
|
self.logger.warning(f"Sent alert for {downloader}/{username} ({failure_count} failures)")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to send alert: {e}")
|
|
|
|
def get_downloader_status(self, downloader: str = None, hours: int = 24) -> List[Dict]:
|
|
"""
|
|
Get recent status for downloader(s)
|
|
|
|
Args:
|
|
downloader: Specific downloader (None = all)
|
|
hours: How many hours to look back
|
|
|
|
Returns:
|
|
List of status dicts with stats per downloader
|
|
"""
|
|
try:
|
|
cutoff = datetime.now() - timedelta(hours=hours)
|
|
|
|
with self.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
if downloader:
|
|
cursor.execute("""
|
|
SELECT
|
|
downloader,
|
|
COUNT(*) as total_attempts,
|
|
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
|
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
|
SUM(file_count) as total_files,
|
|
MAX(CASE WHEN success = 1 THEN timestamp END) as last_success,
|
|
MAX(timestamp) as last_attempt
|
|
FROM download_monitor
|
|
WHERE downloader = ? AND timestamp > ?
|
|
GROUP BY downloader
|
|
""", (downloader, cutoff.isoformat()))
|
|
else:
|
|
cursor.execute("""
|
|
SELECT
|
|
downloader,
|
|
COUNT(*) as total_attempts,
|
|
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
|
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
|
SUM(file_count) as total_files,
|
|
MAX(CASE WHEN success = 1 THEN timestamp END) as last_success,
|
|
MAX(timestamp) as last_attempt
|
|
FROM download_monitor
|
|
WHERE timestamp > ?
|
|
GROUP BY downloader
|
|
ORDER BY downloader
|
|
""", (cutoff.isoformat(),))
|
|
|
|
results = []
|
|
for row in cursor.fetchall():
|
|
results.append({
|
|
'downloader': row['downloader'],
|
|
'total_attempts': row['total_attempts'],
|
|
'successful': row['successful'] or 0,
|
|
'failed': row['failed'] or 0,
|
|
'total_files': row['total_files'] or 0,
|
|
'success_rate': round((row['successful'] or 0) / row['total_attempts'] * 100, 1) if row['total_attempts'] > 0 else 0,
|
|
'last_success': row['last_success'],
|
|
'last_attempt': row['last_attempt']
|
|
})
|
|
|
|
return results
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to get downloader status: {e}")
|
|
return []
|
|
|
|
def clear_old_logs(self, days: int = 30):
|
|
"""
|
|
Clear monitoring logs older than specified days
|
|
|
|
Args:
|
|
days: How many days to keep
|
|
"""
|
|
try:
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|
|
|
with self.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
DELETE FROM download_monitor
|
|
WHERE timestamp < ?
|
|
""", (cutoff.isoformat(),))
|
|
deleted = cursor.rowcount
|
|
conn.commit()
|
|
|
|
self.logger.info(f"Cleared {deleted} old monitoring logs (older than {days} days)")
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to clear old logs: {e}")
|
|
|
|
|
|
# Singleton instance with thread-safe initialization
|
|
_monitor_instance = None
|
|
_monitor_instance_lock = __import__('threading').Lock()
|
|
|
|
|
|
def get_monitor(unified_db=None, settings_manager=None):
|
|
"""Get or create monitor singleton (thread-safe)"""
|
|
global _monitor_instance
|
|
if _monitor_instance is None:
|
|
with _monitor_instance_lock:
|
|
# Double-check inside lock to prevent race condition
|
|
if _monitor_instance is None:
|
|
# Auto-initialize database if not provided
|
|
if unified_db is None:
|
|
from modules.unified_database import UnifiedDatabase
|
|
unified_db = UnifiedDatabase()
|
|
|
|
# Auto-initialize settings manager if not provided
|
|
if settings_manager is None:
|
|
from modules.settings_manager import SettingsManager
|
|
settings_manager = SettingsManager('/opt/media-downloader/database/media_downloader.db')
|
|
|
|
_monitor_instance = DownloaderMonitor(unified_db, settings_manager)
|
|
return _monitor_instance
|