635 lines
24 KiB
Python
635 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Dependency Updater - Automatically updates critical dependencies
|
|
Only runs in scheduler mode, once per day
|
|
|
|
Version Compatibility:
|
|
- bcrypt <5.0 required for passlib 1.7.4 compatibility
|
|
- passlib 1.7.4 requires bcrypt 4.x (not 5.x)
|
|
- uvicorn <0.35.0 required (0.40.0+ has breaking loop_factory changes)
|
|
- Pinned packages are skipped during auto-updates to prevent incompatibilities
|
|
"""
|
|
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict
|
|
from modules.universal_logger import get_logger
|
|
|
|
|
|
class DependencyUpdater:
|
|
"""Manages automatic updates for critical dependencies"""
|
|
|
|
def __init__(self,
|
|
state_file: str = "/opt/media-downloader/database/dependency_updates.json",
|
|
config: dict = None,
|
|
pushover_notifier = None,
|
|
scheduler_mode: bool = False):
|
|
"""
|
|
Initialize dependency updater
|
|
|
|
Args:
|
|
state_file: Path to JSON file storing update state
|
|
config: Configuration dict from settings.json
|
|
pushover_notifier: Instance of PushoverNotifier for alerts
|
|
scheduler_mode: Only run updates when True (scheduler mode)
|
|
"""
|
|
self.state_file = Path(state_file)
|
|
self.state_file.parent.mkdir(parents=True, exist_ok=True)
|
|
self.pushover = pushover_notifier
|
|
self.scheduler_mode = scheduler_mode
|
|
|
|
# Derive venv paths from module location (more portable than hardcoded path)
|
|
import sys
|
|
self._base_dir = Path(__file__).parent.parent
|
|
self._venv_pip = self._base_dir / 'venv' / 'bin' / 'pip'
|
|
self._venv_python = self._base_dir / 'venv' / 'bin' / 'python'
|
|
# Fallback to sys.executable's directory if venv not found
|
|
if not self._venv_pip.exists():
|
|
self._venv_pip = Path(sys.executable).parent / 'pip'
|
|
if not self._venv_python.exists():
|
|
self._venv_python = Path(sys.executable)
|
|
|
|
# Default configuration
|
|
self.config = {
|
|
'enabled': True,
|
|
'check_interval_hours': 24,
|
|
'auto_install': True,
|
|
'components': {
|
|
'flaresolverr': {
|
|
'enabled': True,
|
|
'notify_on_update': True
|
|
},
|
|
'playwright': {
|
|
'enabled': True,
|
|
'notify_on_update': False
|
|
},
|
|
'yt_dlp': {
|
|
'enabled': True,
|
|
'notify_on_update': False
|
|
},
|
|
'python_packages': {
|
|
'enabled': True,
|
|
'notify_on_update': True,
|
|
'packages': [
|
|
# Core API framework
|
|
'fastapi',
|
|
'uvicorn',
|
|
'pydantic',
|
|
'python-jose',
|
|
'passlib',
|
|
'slowapi',
|
|
'starlette',
|
|
'python-multipart',
|
|
'websockets',
|
|
# Security & Auth
|
|
'bcrypt',
|
|
'cryptography',
|
|
'certifi',
|
|
'2captcha-python',
|
|
'duo-universal',
|
|
# Image processing
|
|
'pillow',
|
|
'numpy',
|
|
# Face recognition
|
|
'insightface',
|
|
'onnxruntime',
|
|
'deepface',
|
|
'tensorflow',
|
|
'face-recognition',
|
|
'dlib',
|
|
# Web scraping & downloads
|
|
'requests',
|
|
'beautifulsoup4',
|
|
'selenium',
|
|
'playwright',
|
|
'playwright-stealth',
|
|
'instaloader',
|
|
'yt-dlp',
|
|
'curl-cffi',
|
|
'gallery-dl',
|
|
# Database
|
|
'psycopg2-binary',
|
|
# Utilities
|
|
'python-dotenv',
|
|
'python-dateutil',
|
|
'pyotp',
|
|
'click',
|
|
'attrs',
|
|
'charset-normalizer',
|
|
'idna',
|
|
'websocket-client',
|
|
'trio',
|
|
'typing_extensions'
|
|
]
|
|
}
|
|
},
|
|
'pushover': {
|
|
'enabled': True,
|
|
'priority': -1,
|
|
'sound': 'magic'
|
|
}
|
|
}
|
|
|
|
# Merge user config
|
|
if config:
|
|
self._deep_update(self.config, config)
|
|
|
|
# Load or initialize state
|
|
self.state = self._load_state()
|
|
|
|
# Setup logging
|
|
self.logger = get_logger('DependencyUpdater')
|
|
|
|
# Known version incompatibilities and constraints
|
|
# Format: package_name: [constraints, incompatible_with, reason]
|
|
self.version_constraints = {
|
|
'bcrypt': {
|
|
'constraint': '<5.0',
|
|
'reason': 'bcrypt 5.x is incompatible with passlib 1.7.4',
|
|
'incompatible_with': ['passlib>=1.7.4,<2.0']
|
|
},
|
|
'passlib': {
|
|
'constraint': '>=1.7.4,<2.0',
|
|
'reason': 'passlib 1.7.4 requires bcrypt <5.0',
|
|
'requires': ['bcrypt>=4.0.0,<5.0']
|
|
},
|
|
'uvicorn': {
|
|
'constraint': '<0.35.0',
|
|
'reason': 'uvicorn 0.40.0+ has breaking changes with loop_factory parameter that crashes on startup',
|
|
'known_working': '0.34.0'
|
|
}
|
|
}
|
|
|
|
# Packages that should not be auto-updated
|
|
self.pinned_packages = {
|
|
'bcrypt': 'Version constrained for passlib compatibility',
|
|
'passlib': 'Version constrained for bcrypt compatibility',
|
|
'uvicorn': 'Version 0.40.0+ has breaking changes with loop_factory parameter'
|
|
}
|
|
|
|
def _deep_update(self, base: dict, update: dict):
|
|
"""Deep update dict (recursive merge)"""
|
|
for key, value in update.items():
|
|
if isinstance(value, dict) and key in base and isinstance(base[key], dict):
|
|
self._deep_update(base[key], value)
|
|
else:
|
|
base[key] = value
|
|
|
|
def _load_state(self) -> Dict:
|
|
"""Load update state from file"""
|
|
if self.state_file.exists():
|
|
try:
|
|
with open(self.state_file, 'r') as f:
|
|
return json.load(f)
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to load update state: {e}")
|
|
|
|
# Initialize empty state
|
|
return {
|
|
'last_check': None,
|
|
'components': {}
|
|
}
|
|
|
|
def _save_state(self):
|
|
"""Save update state to file"""
|
|
try:
|
|
with open(self.state_file, 'w') as f:
|
|
json.dump(self.state, f, indent=2, default=str)
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to save update state: {e}")
|
|
|
|
def _should_check_updates(self, force: bool = False) -> bool:
|
|
"""Check if enough time has passed since last update check
|
|
|
|
Args:
|
|
force: If True, bypass all checks and return True
|
|
|
|
Returns:
|
|
True if updates should be checked, False otherwise
|
|
"""
|
|
if force:
|
|
return True
|
|
|
|
if not self.config.get('enabled', True):
|
|
return False
|
|
|
|
# Allow manual checks even outside scheduler mode
|
|
if not self.scheduler_mode:
|
|
# In non-scheduler mode, only proceed if explicitly called
|
|
# This allows manual force_update_check() to work
|
|
return False
|
|
|
|
last_check = self.state.get('last_check')
|
|
if not last_check:
|
|
return True
|
|
|
|
try:
|
|
last_check_time = datetime.fromisoformat(last_check)
|
|
interval_hours = self.config.get('check_interval_hours', 24)
|
|
return datetime.now() - last_check_time > timedelta(hours=interval_hours)
|
|
except Exception:
|
|
return True
|
|
|
|
def check_and_update_all(self, force: bool = False) -> Dict[str, bool]:
|
|
"""
|
|
Check and update all enabled components
|
|
|
|
Args:
|
|
force: If True, bypass interval checks and update immediately
|
|
|
|
Returns:
|
|
Dict mapping component name to update success status
|
|
"""
|
|
if not self._should_check_updates(force=force):
|
|
return {}
|
|
|
|
# Check if auto_install is enabled (default: True)
|
|
auto_install = self.config.get('auto_install', True)
|
|
|
|
if auto_install:
|
|
self.logger.info("Checking for dependency updates...")
|
|
else:
|
|
self.logger.info("Checking for dependency updates (auto_install disabled - check only)...")
|
|
return {} # Skip updates if auto_install is disabled
|
|
|
|
results = {}
|
|
|
|
# Update last check timestamp
|
|
self.state['last_check'] = datetime.now().isoformat()
|
|
self._save_state()
|
|
|
|
# Check each component
|
|
components = self.config.get('components', {})
|
|
|
|
if components.get('flaresolverr', {}).get('enabled', True):
|
|
results['flaresolverr'] = self._update_flaresolverr()
|
|
|
|
if components.get('playwright', {}).get('enabled', True):
|
|
results['playwright'] = self._update_playwright()
|
|
|
|
if components.get('yt_dlp', {}).get('enabled', True):
|
|
results['yt_dlp'] = self._update_yt_dlp()
|
|
|
|
if components.get('python_packages', {}).get('enabled', True):
|
|
results['python_packages'] = self._update_python_packages()
|
|
|
|
# Send summary notification if any updates installed
|
|
if any(results.values()) and self.pushover:
|
|
self._send_update_notification(results)
|
|
|
|
return results
|
|
|
|
def _update_flaresolverr(self) -> bool:
|
|
"""
|
|
Update FlareSolverr Docker container
|
|
|
|
Returns:
|
|
True if update was installed, False otherwise
|
|
"""
|
|
try:
|
|
self.logger.info("Checking FlareSolverr for updates...")
|
|
|
|
# Pull latest image
|
|
result = subprocess.run(
|
|
['docker', 'pull', 'ghcr.io/flaresolverr/flaresolverr:latest'],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=300
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
self.logger.error(f"Failed to pull FlareSolverr image: {result.stderr}")
|
|
return False
|
|
|
|
# Check if image was updated (look for "Downloaded newer image" or "Image is up to date")
|
|
output = result.stdout + result.stderr
|
|
updated = "Downloaded newer image" in output or "pulling from" in output.lower()
|
|
|
|
if not updated:
|
|
self.logger.info("FlareSolverr is already up to date")
|
|
self._update_component_state('flaresolverr', False)
|
|
return False
|
|
|
|
# Image was updated - restart container if running
|
|
self.logger.info("FlareSolverr image updated, restarting container...")
|
|
|
|
# Check if container exists
|
|
check_result = subprocess.run(
|
|
['docker', 'ps', '-a', '--filter', 'name=flaresolverr', '--format', '{{.Names}}'],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
|
|
if 'flaresolverr' in check_result.stdout:
|
|
# Stop and remove old container
|
|
subprocess.run(['docker', 'stop', 'flaresolverr'], capture_output=True)
|
|
subprocess.run(['docker', 'rm', 'flaresolverr'], capture_output=True)
|
|
|
|
# Start new container with latest image
|
|
subprocess.run([
|
|
'docker', 'run', '-d',
|
|
'--name', 'flaresolverr',
|
|
'-p', '8191:8191',
|
|
'-e', 'LOG_LEVEL=info',
|
|
'--restart', 'unless-stopped',
|
|
'ghcr.io/flaresolverr/flaresolverr:latest'
|
|
], capture_output=True)
|
|
|
|
self.logger.info("✓ FlareSolverr updated and restarted successfully")
|
|
else:
|
|
self.logger.info("✓ FlareSolverr image updated (container not running)")
|
|
|
|
self._update_component_state('flaresolverr', True)
|
|
return True
|
|
|
|
except subprocess.TimeoutExpired:
|
|
self.logger.error("FlareSolverr update timed out")
|
|
return False
|
|
except Exception as e:
|
|
self.logger.error(f"FlareSolverr update error: {e}")
|
|
return False
|
|
|
|
def _update_playwright(self) -> bool:
|
|
"""
|
|
Update Playwright browsers (Chromium and Firefox)
|
|
|
|
Returns:
|
|
True if update was installed, False otherwise
|
|
"""
|
|
try:
|
|
self.logger.info("Checking Playwright browsers for updates...")
|
|
|
|
# Use venv python for playwright commands
|
|
venv_python = str(self._venv_python)
|
|
|
|
# Update Chromium
|
|
result_chromium = subprocess.run(
|
|
[venv_python, '-m', 'playwright', 'install', 'chromium'],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=600,
|
|
cwd=str(self._base_dir)
|
|
)
|
|
|
|
# Update Firefox
|
|
result_firefox = subprocess.run(
|
|
[venv_python, '-m', 'playwright', 'install', 'firefox'],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=600,
|
|
cwd=str(self._base_dir)
|
|
)
|
|
|
|
success = result_chromium.returncode == 0 and result_firefox.returncode == 0
|
|
|
|
if success:
|
|
# Check if anything was actually updated
|
|
output = result_chromium.stdout + result_firefox.stdout
|
|
updated = "Downloading" in output or "Installing" in output
|
|
|
|
if updated:
|
|
self.logger.info("✓ Playwright browsers updated successfully")
|
|
self._update_component_state('playwright', True)
|
|
return True
|
|
else:
|
|
self.logger.info("Playwright browsers already up to date")
|
|
self._update_component_state('playwright', False)
|
|
return False
|
|
else:
|
|
self.logger.error("Failed to update Playwright browsers")
|
|
return False
|
|
|
|
except subprocess.TimeoutExpired:
|
|
self.logger.error("Playwright update timed out")
|
|
return False
|
|
except Exception as e:
|
|
self.logger.error(f"Playwright update error: {e}")
|
|
return False
|
|
|
|
def _update_yt_dlp(self) -> bool:
|
|
"""
|
|
Update yt-dlp (critical for TikTok downloads)
|
|
|
|
Returns:
|
|
True if update was installed, False otherwise
|
|
"""
|
|
try:
|
|
self.logger.info("Checking yt-dlp for updates...")
|
|
|
|
# Use venv pip (derived from module location for portability)
|
|
venv_pip = str(self._venv_pip)
|
|
|
|
# Try updating via pip
|
|
result = subprocess.run(
|
|
[venv_pip, 'install', '--upgrade', 'yt-dlp'],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
self.logger.error(f"Failed to update yt-dlp: {result.stderr}")
|
|
return False
|
|
|
|
# Check if update was installed
|
|
output = result.stdout + result.stderr
|
|
updated = "Successfully installed" in output and "yt-dlp" in output
|
|
|
|
if updated:
|
|
self.logger.info("✓ yt-dlp updated successfully")
|
|
self._update_component_state('yt_dlp', True)
|
|
return True
|
|
else:
|
|
self.logger.info("yt-dlp already up to date")
|
|
self._update_component_state('yt_dlp', False)
|
|
return False
|
|
|
|
except subprocess.TimeoutExpired:
|
|
self.logger.error("yt-dlp update timed out")
|
|
return False
|
|
except Exception as e:
|
|
self.logger.error(f"yt-dlp update error: {e}")
|
|
return False
|
|
|
|
def _update_python_packages(self) -> bool:
|
|
"""
|
|
Update Python packages (FastAPI, Uvicorn, Pydantic, etc.)
|
|
|
|
Returns:
|
|
True if any updates were installed, False otherwise
|
|
"""
|
|
try:
|
|
self.logger.info("Checking Python packages for updates...")
|
|
|
|
# Get list of packages to update
|
|
packages = self.config.get('components', {}).get('python_packages', {}).get('packages', [])
|
|
if not packages:
|
|
self.logger.info("No Python packages configured for updates")
|
|
return False
|
|
|
|
# Use venv pip (derived from module location for portability)
|
|
venv_pip = str(self._venv_pip)
|
|
|
|
updated_packages = []
|
|
|
|
for package in packages:
|
|
try:
|
|
# Check if package is pinned (should not be auto-updated)
|
|
if package in self.pinned_packages:
|
|
self.logger.info(f"⚠ Skipping {package}: {self.pinned_packages[package]}")
|
|
continue
|
|
|
|
# Check for version constraints
|
|
if package in self.version_constraints:
|
|
constraint_info = self.version_constraints[package]
|
|
constraint = constraint_info.get('constraint', '')
|
|
reason = constraint_info.get('reason', 'Version constraint')
|
|
|
|
if constraint:
|
|
# Install with constraint instead of --upgrade
|
|
package_spec = f"{package}{constraint}"
|
|
self.logger.info(f"📌 {package}: Applying constraint {constraint} ({reason})")
|
|
|
|
result = subprocess.run(
|
|
[venv_pip, 'install', package_spec],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120
|
|
)
|
|
else:
|
|
# No constraint, normal upgrade
|
|
result = subprocess.run(
|
|
[venv_pip, 'install', '--upgrade', package],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120
|
|
)
|
|
else:
|
|
# Update package normally
|
|
result = subprocess.run(
|
|
[venv_pip, 'install', '--upgrade', package],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
output = result.stdout + result.stderr
|
|
# Check if package was actually updated
|
|
if "Successfully installed" in output and package in output:
|
|
updated_packages.append(package)
|
|
self.logger.info(f"✓ {package} updated")
|
|
elif "Requirement already satisfied" in output:
|
|
self.logger.debug(f" {package} already up to date")
|
|
else:
|
|
self.logger.debug(f" {package} checked")
|
|
else:
|
|
self.logger.warning(f"Failed to update {package}: {result.stderr}")
|
|
|
|
except subprocess.TimeoutExpired:
|
|
self.logger.warning(f"{package} update timed out")
|
|
except Exception as e:
|
|
self.logger.warning(f"Error updating {package}: {e}")
|
|
|
|
if updated_packages:
|
|
self.logger.info(f"✓ Updated {len(updated_packages)} Python package(s): {', '.join(updated_packages)}")
|
|
self._update_component_state('python_packages', True)
|
|
|
|
# Store list of updated packages in state
|
|
if 'components' not in self.state:
|
|
self.state['components'] = {}
|
|
if 'python_packages' not in self.state['components']:
|
|
self.state['components']['python_packages'] = {}
|
|
self.state['components']['python_packages']['updated_packages'] = updated_packages
|
|
self._save_state()
|
|
|
|
return True
|
|
else:
|
|
self.logger.info("All Python packages already up to date")
|
|
self._update_component_state('python_packages', False)
|
|
return False
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Python packages update error: {e}")
|
|
return False
|
|
|
|
def _update_component_state(self, component: str, updated: bool):
|
|
"""Update component state in JSON"""
|
|
if 'components' not in self.state:
|
|
self.state['components'] = {}
|
|
|
|
if component not in self.state['components']:
|
|
self.state['components'][component] = {}
|
|
|
|
self.state['components'][component]['last_update'] = datetime.now().isoformat() if updated else self.state['components'][component].get('last_update')
|
|
self.state['components'][component]['last_check'] = datetime.now().isoformat()
|
|
self.state['components'][component]['status'] = 'updated' if updated else 'current'
|
|
|
|
self._save_state()
|
|
|
|
def _send_update_notification(self, results: Dict[str, bool]):
|
|
"""Send Pushover notification about installed updates"""
|
|
if not self.config.get('pushover', {}).get('enabled', True):
|
|
return
|
|
|
|
# Build list of updated components
|
|
updated_components = [name for name, updated in results.items() if updated]
|
|
|
|
if not updated_components:
|
|
return
|
|
|
|
# Check which components should send notifications
|
|
notify_components = []
|
|
for component in updated_components:
|
|
component_config = self.config.get('components', {}).get(component, {})
|
|
if component_config.get('notify_on_update', True):
|
|
notify_components.append(component)
|
|
|
|
if not notify_components:
|
|
return
|
|
|
|
# Format component names
|
|
component_map = {
|
|
'flaresolverr': 'FlareSolverr',
|
|
'playwright': 'Playwright Browsers',
|
|
'yt_dlp': 'yt-dlp',
|
|
'python_packages': 'Python Packages'
|
|
}
|
|
|
|
formatted_names = [component_map.get(c, c) for c in notify_components]
|
|
|
|
title = "🔄 Dependencies Updated"
|
|
if len(formatted_names) == 1:
|
|
message = f"{formatted_names[0]} has been updated to the latest version."
|
|
else:
|
|
message = f"The following components have been updated:\n\n"
|
|
for name in formatted_names:
|
|
message += f"• {name}\n"
|
|
|
|
message += f"\nUpdated at: {datetime.now().strftime('%b %d, %I:%M %p')}"
|
|
|
|
try:
|
|
priority = self.config.get('pushover', {}).get('priority', -1)
|
|
sound = self.config.get('pushover', {}).get('sound', 'magic')
|
|
|
|
self.pushover.send_notification(
|
|
title=title,
|
|
message=message,
|
|
priority=priority,
|
|
sound=sound
|
|
)
|
|
|
|
self.logger.info(f"Sent update notification for: {', '.join(formatted_names)}")
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to send update notification: {e}")
|
|
|
|
def get_update_status(self) -> Dict:
|
|
"""Get current update status for all components"""
|
|
return self.state.copy()
|
|
|
|
def force_update_check(self) -> Dict[str, bool]:
|
|
"""Force immediate update check regardless of interval or scheduler mode"""
|
|
return self.check_and_update_all(force=True)
|