Todd
634
modules/dependency_updater.py
Normal file
634
modules/dependency_updater.py
Normal file
@@ -0,0 +1,634 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Dependency Updater - Automatically updates critical dependencies
|
||||
Only runs in scheduler mode, once per day
|
||||
|
||||
Version Compatibility:
|
||||
- bcrypt <5.0 required for passlib 1.7.4 compatibility
|
||||
- passlib 1.7.4 requires bcrypt 4.x (not 5.x)
|
||||
- uvicorn <0.35.0 required (0.40.0+ has breaking loop_factory changes)
|
||||
- Pinned packages are skipped during auto-updates to prevent incompatibilities
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict
|
||||
from modules.universal_logger import get_logger
|
||||
|
||||
|
||||
class DependencyUpdater:
|
||||
"""Manages automatic updates for critical dependencies"""
|
||||
|
||||
def __init__(self,
|
||||
state_file: str = "/opt/media-downloader/database/dependency_updates.json",
|
||||
config: dict = None,
|
||||
pushover_notifier = None,
|
||||
scheduler_mode: bool = False):
|
||||
"""
|
||||
Initialize dependency updater
|
||||
|
||||
Args:
|
||||
state_file: Path to JSON file storing update state
|
||||
config: Configuration dict from settings.json
|
||||
pushover_notifier: Instance of PushoverNotifier for alerts
|
||||
scheduler_mode: Only run updates when True (scheduler mode)
|
||||
"""
|
||||
self.state_file = Path(state_file)
|
||||
self.state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.pushover = pushover_notifier
|
||||
self.scheduler_mode = scheduler_mode
|
||||
|
||||
# Derive venv paths from module location (more portable than hardcoded path)
|
||||
import sys
|
||||
self._base_dir = Path(__file__).parent.parent
|
||||
self._venv_pip = self._base_dir / 'venv' / 'bin' / 'pip'
|
||||
self._venv_python = self._base_dir / 'venv' / 'bin' / 'python'
|
||||
# Fallback to sys.executable's directory if venv not found
|
||||
if not self._venv_pip.exists():
|
||||
self._venv_pip = Path(sys.executable).parent / 'pip'
|
||||
if not self._venv_python.exists():
|
||||
self._venv_python = Path(sys.executable)
|
||||
|
||||
# Default configuration
|
||||
self.config = {
|
||||
'enabled': True,
|
||||
'check_interval_hours': 24,
|
||||
'auto_install': True,
|
||||
'components': {
|
||||
'flaresolverr': {
|
||||
'enabled': True,
|
||||
'notify_on_update': True
|
||||
},
|
||||
'playwright': {
|
||||
'enabled': True,
|
||||
'notify_on_update': False
|
||||
},
|
||||
'yt_dlp': {
|
||||
'enabled': True,
|
||||
'notify_on_update': False
|
||||
},
|
||||
'python_packages': {
|
||||
'enabled': True,
|
||||
'notify_on_update': True,
|
||||
'packages': [
|
||||
# Core API framework
|
||||
'fastapi',
|
||||
'uvicorn',
|
||||
'pydantic',
|
||||
'python-jose',
|
||||
'passlib',
|
||||
'slowapi',
|
||||
'starlette',
|
||||
'python-multipart',
|
||||
'websockets',
|
||||
# Security & Auth
|
||||
'bcrypt',
|
||||
'cryptography',
|
||||
'certifi',
|
||||
'2captcha-python',
|
||||
'duo-universal',
|
||||
# Image processing
|
||||
'pillow',
|
||||
'numpy',
|
||||
# Face recognition
|
||||
'insightface',
|
||||
'onnxruntime',
|
||||
'deepface',
|
||||
'tensorflow',
|
||||
'face-recognition',
|
||||
'dlib',
|
||||
# Web scraping & downloads
|
||||
'requests',
|
||||
'beautifulsoup4',
|
||||
'selenium',
|
||||
'playwright',
|
||||
'playwright-stealth',
|
||||
'instaloader',
|
||||
'yt-dlp',
|
||||
'curl-cffi',
|
||||
'gallery-dl',
|
||||
# Database
|
||||
'psycopg2-binary',
|
||||
# Utilities
|
||||
'python-dotenv',
|
||||
'python-dateutil',
|
||||
'pyotp',
|
||||
'click',
|
||||
'attrs',
|
||||
'charset-normalizer',
|
||||
'idna',
|
||||
'websocket-client',
|
||||
'trio',
|
||||
'typing_extensions'
|
||||
]
|
||||
}
|
||||
},
|
||||
'pushover': {
|
||||
'enabled': True,
|
||||
'priority': -1,
|
||||
'sound': 'magic'
|
||||
}
|
||||
}
|
||||
|
||||
# Merge user config
|
||||
if config:
|
||||
self._deep_update(self.config, config)
|
||||
|
||||
# Load or initialize state
|
||||
self.state = self._load_state()
|
||||
|
||||
# Setup logging
|
||||
self.logger = get_logger('DependencyUpdater')
|
||||
|
||||
# Known version incompatibilities and constraints
|
||||
# Format: package_name: [constraints, incompatible_with, reason]
|
||||
self.version_constraints = {
|
||||
'bcrypt': {
|
||||
'constraint': '<5.0',
|
||||
'reason': 'bcrypt 5.x is incompatible with passlib 1.7.4',
|
||||
'incompatible_with': ['passlib>=1.7.4,<2.0']
|
||||
},
|
||||
'passlib': {
|
||||
'constraint': '>=1.7.4,<2.0',
|
||||
'reason': 'passlib 1.7.4 requires bcrypt <5.0',
|
||||
'requires': ['bcrypt>=4.0.0,<5.0']
|
||||
},
|
||||
'uvicorn': {
|
||||
'constraint': '<0.35.0',
|
||||
'reason': 'uvicorn 0.40.0+ has breaking changes with loop_factory parameter that crashes on startup',
|
||||
'known_working': '0.34.0'
|
||||
}
|
||||
}
|
||||
|
||||
# Packages that should not be auto-updated
|
||||
self.pinned_packages = {
|
||||
'bcrypt': 'Version constrained for passlib compatibility',
|
||||
'passlib': 'Version constrained for bcrypt compatibility',
|
||||
'uvicorn': 'Version 0.40.0+ has breaking changes with loop_factory parameter'
|
||||
}
|
||||
|
||||
def _deep_update(self, base: dict, update: dict):
|
||||
"""Deep update dict (recursive merge)"""
|
||||
for key, value in update.items():
|
||||
if isinstance(value, dict) and key in base and isinstance(base[key], dict):
|
||||
self._deep_update(base[key], value)
|
||||
else:
|
||||
base[key] = value
|
||||
|
||||
def _load_state(self) -> Dict:
|
||||
"""Load update state from file"""
|
||||
if self.state_file.exists():
|
||||
try:
|
||||
with open(self.state_file, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to load update state: {e}")
|
||||
|
||||
# Initialize empty state
|
||||
return {
|
||||
'last_check': None,
|
||||
'components': {}
|
||||
}
|
||||
|
||||
def _save_state(self):
|
||||
"""Save update state to file"""
|
||||
try:
|
||||
with open(self.state_file, 'w') as f:
|
||||
json.dump(self.state, f, indent=2, default=str)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to save update state: {e}")
|
||||
|
||||
def _should_check_updates(self, force: bool = False) -> bool:
|
||||
"""Check if enough time has passed since last update check
|
||||
|
||||
Args:
|
||||
force: If True, bypass all checks and return True
|
||||
|
||||
Returns:
|
||||
True if updates should be checked, False otherwise
|
||||
"""
|
||||
if force:
|
||||
return True
|
||||
|
||||
if not self.config.get('enabled', True):
|
||||
return False
|
||||
|
||||
# Allow manual checks even outside scheduler mode
|
||||
if not self.scheduler_mode:
|
||||
# In non-scheduler mode, only proceed if explicitly called
|
||||
# This allows manual force_update_check() to work
|
||||
return False
|
||||
|
||||
last_check = self.state.get('last_check')
|
||||
if not last_check:
|
||||
return True
|
||||
|
||||
try:
|
||||
last_check_time = datetime.fromisoformat(last_check)
|
||||
interval_hours = self.config.get('check_interval_hours', 24)
|
||||
return datetime.now() - last_check_time > timedelta(hours=interval_hours)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
def check_and_update_all(self, force: bool = False) -> Dict[str, bool]:
|
||||
"""
|
||||
Check and update all enabled components
|
||||
|
||||
Args:
|
||||
force: If True, bypass interval checks and update immediately
|
||||
|
||||
Returns:
|
||||
Dict mapping component name to update success status
|
||||
"""
|
||||
if not self._should_check_updates(force=force):
|
||||
return {}
|
||||
|
||||
# Check if auto_install is enabled (default: True)
|
||||
auto_install = self.config.get('auto_install', True)
|
||||
|
||||
if auto_install:
|
||||
self.logger.info("Checking for dependency updates...")
|
||||
else:
|
||||
self.logger.info("Checking for dependency updates (auto_install disabled - check only)...")
|
||||
return {} # Skip updates if auto_install is disabled
|
||||
|
||||
results = {}
|
||||
|
||||
# Update last check timestamp
|
||||
self.state['last_check'] = datetime.now().isoformat()
|
||||
self._save_state()
|
||||
|
||||
# Check each component
|
||||
components = self.config.get('components', {})
|
||||
|
||||
if components.get('flaresolverr', {}).get('enabled', True):
|
||||
results['flaresolverr'] = self._update_flaresolverr()
|
||||
|
||||
if components.get('playwright', {}).get('enabled', True):
|
||||
results['playwright'] = self._update_playwright()
|
||||
|
||||
if components.get('yt_dlp', {}).get('enabled', True):
|
||||
results['yt_dlp'] = self._update_yt_dlp()
|
||||
|
||||
if components.get('python_packages', {}).get('enabled', True):
|
||||
results['python_packages'] = self._update_python_packages()
|
||||
|
||||
# Send summary notification if any updates installed
|
||||
if any(results.values()) and self.pushover:
|
||||
self._send_update_notification(results)
|
||||
|
||||
return results
|
||||
|
||||
def _update_flaresolverr(self) -> bool:
|
||||
"""
|
||||
Update FlareSolverr Docker container
|
||||
|
||||
Returns:
|
||||
True if update was installed, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Checking FlareSolverr for updates...")
|
||||
|
||||
# Pull latest image
|
||||
result = subprocess.run(
|
||||
['docker', 'pull', 'ghcr.io/flaresolverr/flaresolverr:latest'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
self.logger.error(f"Failed to pull FlareSolverr image: {result.stderr}")
|
||||
return False
|
||||
|
||||
# Check if image was updated (look for "Downloaded newer image" or "Image is up to date")
|
||||
output = result.stdout + result.stderr
|
||||
updated = "Downloaded newer image" in output or "pulling from" in output.lower()
|
||||
|
||||
if not updated:
|
||||
self.logger.info("FlareSolverr is already up to date")
|
||||
self._update_component_state('flaresolverr', False)
|
||||
return False
|
||||
|
||||
# Image was updated - restart container if running
|
||||
self.logger.info("FlareSolverr image updated, restarting container...")
|
||||
|
||||
# Check if container exists
|
||||
check_result = subprocess.run(
|
||||
['docker', 'ps', '-a', '--filter', 'name=flaresolverr', '--format', '{{.Names}}'],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if 'flaresolverr' in check_result.stdout:
|
||||
# Stop and remove old container
|
||||
subprocess.run(['docker', 'stop', 'flaresolverr'], capture_output=True)
|
||||
subprocess.run(['docker', 'rm', 'flaresolverr'], capture_output=True)
|
||||
|
||||
# Start new container with latest image
|
||||
subprocess.run([
|
||||
'docker', 'run', '-d',
|
||||
'--name', 'flaresolverr',
|
||||
'-p', '8191:8191',
|
||||
'-e', 'LOG_LEVEL=info',
|
||||
'--restart', 'unless-stopped',
|
||||
'ghcr.io/flaresolverr/flaresolverr:latest'
|
||||
], capture_output=True)
|
||||
|
||||
self.logger.info("✓ FlareSolverr updated and restarted successfully")
|
||||
else:
|
||||
self.logger.info("✓ FlareSolverr image updated (container not running)")
|
||||
|
||||
self._update_component_state('flaresolverr', True)
|
||||
return True
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.error("FlareSolverr update timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
self.logger.error(f"FlareSolverr update error: {e}")
|
||||
return False
|
||||
|
||||
def _update_playwright(self) -> bool:
|
||||
"""
|
||||
Update Playwright browsers (Chromium and Firefox)
|
||||
|
||||
Returns:
|
||||
True if update was installed, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Checking Playwright browsers for updates...")
|
||||
|
||||
# Use venv python for playwright commands
|
||||
venv_python = str(self._venv_python)
|
||||
|
||||
# Update Chromium
|
||||
result_chromium = subprocess.run(
|
||||
[venv_python, '-m', 'playwright', 'install', 'chromium'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
cwd=str(self._base_dir)
|
||||
)
|
||||
|
||||
# Update Firefox
|
||||
result_firefox = subprocess.run(
|
||||
[venv_python, '-m', 'playwright', 'install', 'firefox'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
cwd=str(self._base_dir)
|
||||
)
|
||||
|
||||
success = result_chromium.returncode == 0 and result_firefox.returncode == 0
|
||||
|
||||
if success:
|
||||
# Check if anything was actually updated
|
||||
output = result_chromium.stdout + result_firefox.stdout
|
||||
updated = "Downloading" in output or "Installing" in output
|
||||
|
||||
if updated:
|
||||
self.logger.info("✓ Playwright browsers updated successfully")
|
||||
self._update_component_state('playwright', True)
|
||||
return True
|
||||
else:
|
||||
self.logger.info("Playwright browsers already up to date")
|
||||
self._update_component_state('playwright', False)
|
||||
return False
|
||||
else:
|
||||
self.logger.error("Failed to update Playwright browsers")
|
||||
return False
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.error("Playwright update timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
self.logger.error(f"Playwright update error: {e}")
|
||||
return False
|
||||
|
||||
def _update_yt_dlp(self) -> bool:
|
||||
"""
|
||||
Update yt-dlp (critical for TikTok downloads)
|
||||
|
||||
Returns:
|
||||
True if update was installed, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Checking yt-dlp for updates...")
|
||||
|
||||
# Use venv pip (derived from module location for portability)
|
||||
venv_pip = str(self._venv_pip)
|
||||
|
||||
# Try updating via pip
|
||||
result = subprocess.run(
|
||||
[venv_pip, 'install', '--upgrade', 'yt-dlp'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
self.logger.error(f"Failed to update yt-dlp: {result.stderr}")
|
||||
return False
|
||||
|
||||
# Check if update was installed
|
||||
output = result.stdout + result.stderr
|
||||
updated = "Successfully installed" in output and "yt-dlp" in output
|
||||
|
||||
if updated:
|
||||
self.logger.info("✓ yt-dlp updated successfully")
|
||||
self._update_component_state('yt_dlp', True)
|
||||
return True
|
||||
else:
|
||||
self.logger.info("yt-dlp already up to date")
|
||||
self._update_component_state('yt_dlp', False)
|
||||
return False
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.error("yt-dlp update timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
self.logger.error(f"yt-dlp update error: {e}")
|
||||
return False
|
||||
|
||||
def _update_python_packages(self) -> bool:
|
||||
"""
|
||||
Update Python packages (FastAPI, Uvicorn, Pydantic, etc.)
|
||||
|
||||
Returns:
|
||||
True if any updates were installed, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.logger.info("Checking Python packages for updates...")
|
||||
|
||||
# Get list of packages to update
|
||||
packages = self.config.get('components', {}).get('python_packages', {}).get('packages', [])
|
||||
if not packages:
|
||||
self.logger.info("No Python packages configured for updates")
|
||||
return False
|
||||
|
||||
# Use venv pip (derived from module location for portability)
|
||||
venv_pip = str(self._venv_pip)
|
||||
|
||||
updated_packages = []
|
||||
|
||||
for package in packages:
|
||||
try:
|
||||
# Check if package is pinned (should not be auto-updated)
|
||||
if package in self.pinned_packages:
|
||||
self.logger.info(f"⚠ Skipping {package}: {self.pinned_packages[package]}")
|
||||
continue
|
||||
|
||||
# Check for version constraints
|
||||
if package in self.version_constraints:
|
||||
constraint_info = self.version_constraints[package]
|
||||
constraint = constraint_info.get('constraint', '')
|
||||
reason = constraint_info.get('reason', 'Version constraint')
|
||||
|
||||
if constraint:
|
||||
# Install with constraint instead of --upgrade
|
||||
package_spec = f"{package}{constraint}"
|
||||
self.logger.info(f"📌 {package}: Applying constraint {constraint} ({reason})")
|
||||
|
||||
result = subprocess.run(
|
||||
[venv_pip, 'install', package_spec],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
else:
|
||||
# No constraint, normal upgrade
|
||||
result = subprocess.run(
|
||||
[venv_pip, 'install', '--upgrade', package],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
else:
|
||||
# Update package normally
|
||||
result = subprocess.run(
|
||||
[venv_pip, 'install', '--upgrade', package],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
output = result.stdout + result.stderr
|
||||
# Check if package was actually updated
|
||||
if "Successfully installed" in output and package in output:
|
||||
updated_packages.append(package)
|
||||
self.logger.info(f"✓ {package} updated")
|
||||
elif "Requirement already satisfied" in output:
|
||||
self.logger.debug(f" {package} already up to date")
|
||||
else:
|
||||
self.logger.debug(f" {package} checked")
|
||||
else:
|
||||
self.logger.warning(f"Failed to update {package}: {result.stderr}")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.warning(f"{package} update timed out")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error updating {package}: {e}")
|
||||
|
||||
if updated_packages:
|
||||
self.logger.info(f"✓ Updated {len(updated_packages)} Python package(s): {', '.join(updated_packages)}")
|
||||
self._update_component_state('python_packages', True)
|
||||
|
||||
# Store list of updated packages in state
|
||||
if 'components' not in self.state:
|
||||
self.state['components'] = {}
|
||||
if 'python_packages' not in self.state['components']:
|
||||
self.state['components']['python_packages'] = {}
|
||||
self.state['components']['python_packages']['updated_packages'] = updated_packages
|
||||
self._save_state()
|
||||
|
||||
return True
|
||||
else:
|
||||
self.logger.info("All Python packages already up to date")
|
||||
self._update_component_state('python_packages', False)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Python packages update error: {e}")
|
||||
return False
|
||||
|
||||
def _update_component_state(self, component: str, updated: bool):
|
||||
"""Update component state in JSON"""
|
||||
if 'components' not in self.state:
|
||||
self.state['components'] = {}
|
||||
|
||||
if component not in self.state['components']:
|
||||
self.state['components'][component] = {}
|
||||
|
||||
self.state['components'][component]['last_update'] = datetime.now().isoformat() if updated else self.state['components'][component].get('last_update')
|
||||
self.state['components'][component]['last_check'] = datetime.now().isoformat()
|
||||
self.state['components'][component]['status'] = 'updated' if updated else 'current'
|
||||
|
||||
self._save_state()
|
||||
|
||||
def _send_update_notification(self, results: Dict[str, bool]):
|
||||
"""Send Pushover notification about installed updates"""
|
||||
if not self.config.get('pushover', {}).get('enabled', True):
|
||||
return
|
||||
|
||||
# Build list of updated components
|
||||
updated_components = [name for name, updated in results.items() if updated]
|
||||
|
||||
if not updated_components:
|
||||
return
|
||||
|
||||
# Check which components should send notifications
|
||||
notify_components = []
|
||||
for component in updated_components:
|
||||
component_config = self.config.get('components', {}).get(component, {})
|
||||
if component_config.get('notify_on_update', True):
|
||||
notify_components.append(component)
|
||||
|
||||
if not notify_components:
|
||||
return
|
||||
|
||||
# Format component names
|
||||
component_map = {
|
||||
'flaresolverr': 'FlareSolverr',
|
||||
'playwright': 'Playwright Browsers',
|
||||
'yt_dlp': 'yt-dlp',
|
||||
'python_packages': 'Python Packages'
|
||||
}
|
||||
|
||||
formatted_names = [component_map.get(c, c) for c in notify_components]
|
||||
|
||||
title = "🔄 Dependencies Updated"
|
||||
if len(formatted_names) == 1:
|
||||
message = f"{formatted_names[0]} has been updated to the latest version."
|
||||
else:
|
||||
message = f"The following components have been updated:\n\n"
|
||||
for name in formatted_names:
|
||||
message += f"• {name}\n"
|
||||
|
||||
message += f"\nUpdated at: {datetime.now().strftime('%b %d, %I:%M %p')}"
|
||||
|
||||
try:
|
||||
priority = self.config.get('pushover', {}).get('priority', -1)
|
||||
sound = self.config.get('pushover', {}).get('sound', 'magic')
|
||||
|
||||
self.pushover.send_notification(
|
||||
title=title,
|
||||
message=message,
|
||||
priority=priority,
|
||||
sound=sound
|
||||
)
|
||||
|
||||
self.logger.info(f"Sent update notification for: {', '.join(formatted_names)}")
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to send update notification: {e}")
|
||||
|
||||
def get_update_status(self) -> Dict:
|
||||
"""Get current update status for all components"""
|
||||
return self.state.copy()
|
||||
|
||||
def force_update_check(self) -> Dict[str, bool]:
|
||||
"""Force immediate update check regardless of interval or scheduler mode"""
|
||||
return self.check_and_update_all(force=True)
|
||||
Reference in New Issue
Block a user