#!/usr/bin/env python3 """ Dependency Updater - Automatically updates critical dependencies Only runs in scheduler mode, once per day Version Compatibility: - bcrypt <5.0 required for passlib 1.7.4 compatibility - passlib 1.7.4 requires bcrypt 4.x (not 5.x) - uvicorn <0.35.0 required (0.40.0+ has breaking loop_factory changes) - Pinned packages are skipped during auto-updates to prevent incompatibilities """ import json import subprocess from pathlib import Path from datetime import datetime, timedelta from typing import Dict from modules.universal_logger import get_logger class DependencyUpdater: """Manages automatic updates for critical dependencies""" def __init__(self, state_file: str = "/opt/media-downloader/database/dependency_updates.json", config: dict = None, pushover_notifier = None, scheduler_mode: bool = False): """ Initialize dependency updater Args: state_file: Path to JSON file storing update state config: Configuration dict from settings.json pushover_notifier: Instance of PushoverNotifier for alerts scheduler_mode: Only run updates when True (scheduler mode) """ self.state_file = Path(state_file) self.state_file.parent.mkdir(parents=True, exist_ok=True) self.pushover = pushover_notifier self.scheduler_mode = scheduler_mode # Derive venv paths from module location (more portable than hardcoded path) import sys self._base_dir = Path(__file__).parent.parent self._venv_pip = self._base_dir / 'venv' / 'bin' / 'pip' self._venv_python = self._base_dir / 'venv' / 'bin' / 'python' # Fallback to sys.executable's directory if venv not found if not self._venv_pip.exists(): self._venv_pip = Path(sys.executable).parent / 'pip' if not self._venv_python.exists(): self._venv_python = Path(sys.executable) # Default configuration self.config = { 'enabled': True, 'check_interval_hours': 24, 'auto_install': True, 'components': { 'flaresolverr': { 'enabled': True, 'notify_on_update': True }, 'playwright': { 'enabled': True, 'notify_on_update': False }, 'yt_dlp': { 'enabled': True, 'notify_on_update': False }, 'python_packages': { 'enabled': True, 'notify_on_update': True, 'packages': [ # Core API framework 'fastapi', 'uvicorn', 'pydantic', 'python-jose', 'passlib', 'slowapi', 'starlette', 'python-multipart', 'websockets', # Security & Auth 'bcrypt', 'cryptography', 'certifi', '2captcha-python', 'duo-universal', # Image processing 'pillow', 'numpy', # Face recognition 'insightface', 'onnxruntime', 'deepface', 'tensorflow', 'face-recognition', 'dlib', # Web scraping & downloads 'requests', 'beautifulsoup4', 'selenium', 'playwright', 'playwright-stealth', 'instaloader', 'yt-dlp', 'curl-cffi', 'gallery-dl', # Database 'psycopg2-binary', # Utilities 'python-dotenv', 'python-dateutil', 'pyotp', 'click', 'attrs', 'charset-normalizer', 'idna', 'websocket-client', 'trio', 'typing_extensions' ] } }, 'pushover': { 'enabled': True, 'priority': -1, 'sound': 'magic' } } # Merge user config if config: self._deep_update(self.config, config) # Load or initialize state self.state = self._load_state() # Setup logging self.logger = get_logger('DependencyUpdater') # Known version incompatibilities and constraints # Format: package_name: [constraints, incompatible_with, reason] self.version_constraints = { 'bcrypt': { 'constraint': '<5.0', 'reason': 'bcrypt 5.x is incompatible with passlib 1.7.4', 'incompatible_with': ['passlib>=1.7.4,<2.0'] }, 'passlib': { 'constraint': '>=1.7.4,<2.0', 'reason': 'passlib 1.7.4 requires bcrypt <5.0', 'requires': ['bcrypt>=4.0.0,<5.0'] }, 'uvicorn': { 'constraint': '<0.35.0', 'reason': 'uvicorn 0.40.0+ has breaking changes with loop_factory parameter that crashes on startup', 'known_working': '0.34.0' } } # Packages that should not be auto-updated self.pinned_packages = { 'bcrypt': 'Version constrained for passlib compatibility', 'passlib': 'Version constrained for bcrypt compatibility', 'uvicorn': 'Version 0.40.0+ has breaking changes with loop_factory parameter' } def _deep_update(self, base: dict, update: dict): """Deep update dict (recursive merge)""" for key, value in update.items(): if isinstance(value, dict) and key in base and isinstance(base[key], dict): self._deep_update(base[key], value) else: base[key] = value def _load_state(self) -> Dict: """Load update state from file""" if self.state_file.exists(): try: with open(self.state_file, 'r') as f: return json.load(f) except Exception as e: self.logger.error(f"Failed to load update state: {e}") # Initialize empty state return { 'last_check': None, 'components': {} } def _save_state(self): """Save update state to file""" try: with open(self.state_file, 'w') as f: json.dump(self.state, f, indent=2, default=str) except Exception as e: self.logger.error(f"Failed to save update state: {e}") def _should_check_updates(self, force: bool = False) -> bool: """Check if enough time has passed since last update check Args: force: If True, bypass all checks and return True Returns: True if updates should be checked, False otherwise """ if force: return True if not self.config.get('enabled', True): return False # Allow manual checks even outside scheduler mode if not self.scheduler_mode: # In non-scheduler mode, only proceed if explicitly called # This allows manual force_update_check() to work return False last_check = self.state.get('last_check') if not last_check: return True try: last_check_time = datetime.fromisoformat(last_check) interval_hours = self.config.get('check_interval_hours', 24) return datetime.now() - last_check_time > timedelta(hours=interval_hours) except Exception: return True def check_and_update_all(self, force: bool = False) -> Dict[str, bool]: """ Check and update all enabled components Args: force: If True, bypass interval checks and update immediately Returns: Dict mapping component name to update success status """ if not self._should_check_updates(force=force): return {} # Check if auto_install is enabled (default: True) auto_install = self.config.get('auto_install', True) if auto_install: self.logger.info("Checking for dependency updates...") else: self.logger.info("Checking for dependency updates (auto_install disabled - check only)...") return {} # Skip updates if auto_install is disabled results = {} # Update last check timestamp self.state['last_check'] = datetime.now().isoformat() self._save_state() # Check each component components = self.config.get('components', {}) if components.get('flaresolverr', {}).get('enabled', True): results['flaresolverr'] = self._update_flaresolverr() if components.get('playwright', {}).get('enabled', True): results['playwright'] = self._update_playwright() if components.get('yt_dlp', {}).get('enabled', True): results['yt_dlp'] = self._update_yt_dlp() if components.get('python_packages', {}).get('enabled', True): results['python_packages'] = self._update_python_packages() # Send summary notification if any updates installed if any(results.values()) and self.pushover: self._send_update_notification(results) return results def _update_flaresolverr(self) -> bool: """ Update FlareSolverr Docker container Returns: True if update was installed, False otherwise """ try: self.logger.info("Checking FlareSolverr for updates...") # Pull latest image result = subprocess.run( ['docker', 'pull', 'ghcr.io/flaresolverr/flaresolverr:latest'], capture_output=True, text=True, timeout=300 ) if result.returncode != 0: self.logger.error(f"Failed to pull FlareSolverr image: {result.stderr}") return False # Check if image was updated (look for "Downloaded newer image" or "Image is up to date") output = result.stdout + result.stderr updated = "Downloaded newer image" in output or "pulling from" in output.lower() if not updated: self.logger.info("FlareSolverr is already up to date") self._update_component_state('flaresolverr', False) return False # Image was updated - restart container if running self.logger.info("FlareSolverr image updated, restarting container...") # Check if container exists check_result = subprocess.run( ['docker', 'ps', '-a', '--filter', 'name=flaresolverr', '--format', '{{.Names}}'], capture_output=True, text=True ) if 'flaresolverr' in check_result.stdout: # Stop and remove old container subprocess.run(['docker', 'stop', 'flaresolverr'], capture_output=True) subprocess.run(['docker', 'rm', 'flaresolverr'], capture_output=True) # Start new container with latest image subprocess.run([ 'docker', 'run', '-d', '--name', 'flaresolverr', '-p', '8191:8191', '-e', 'LOG_LEVEL=info', '--restart', 'unless-stopped', 'ghcr.io/flaresolverr/flaresolverr:latest' ], capture_output=True) self.logger.info("✓ FlareSolverr updated and restarted successfully") else: self.logger.info("✓ FlareSolverr image updated (container not running)") self._update_component_state('flaresolverr', True) return True except subprocess.TimeoutExpired: self.logger.error("FlareSolverr update timed out") return False except Exception as e: self.logger.error(f"FlareSolverr update error: {e}") return False def _update_playwright(self) -> bool: """ Update Playwright browsers (Chromium and Firefox) Returns: True if update was installed, False otherwise """ try: self.logger.info("Checking Playwright browsers for updates...") # Use venv python for playwright commands venv_python = str(self._venv_python) # Update Chromium result_chromium = subprocess.run( [venv_python, '-m', 'playwright', 'install', 'chromium'], capture_output=True, text=True, timeout=600, cwd=str(self._base_dir) ) # Update Firefox result_firefox = subprocess.run( [venv_python, '-m', 'playwright', 'install', 'firefox'], capture_output=True, text=True, timeout=600, cwd=str(self._base_dir) ) success = result_chromium.returncode == 0 and result_firefox.returncode == 0 if success: # Check if anything was actually updated output = result_chromium.stdout + result_firefox.stdout updated = "Downloading" in output or "Installing" in output if updated: self.logger.info("✓ Playwright browsers updated successfully") self._update_component_state('playwright', True) return True else: self.logger.info("Playwright browsers already up to date") self._update_component_state('playwright', False) return False else: self.logger.error("Failed to update Playwright browsers") return False except subprocess.TimeoutExpired: self.logger.error("Playwright update timed out") return False except Exception as e: self.logger.error(f"Playwright update error: {e}") return False def _update_yt_dlp(self) -> bool: """ Update yt-dlp (critical for TikTok downloads) Returns: True if update was installed, False otherwise """ try: self.logger.info("Checking yt-dlp for updates...") # Use venv pip (derived from module location for portability) venv_pip = str(self._venv_pip) # Try updating via pip result = subprocess.run( [venv_pip, 'install', '--upgrade', 'yt-dlp'], capture_output=True, text=True, timeout=120 ) if result.returncode != 0: self.logger.error(f"Failed to update yt-dlp: {result.stderr}") return False # Check if update was installed output = result.stdout + result.stderr updated = "Successfully installed" in output and "yt-dlp" in output if updated: self.logger.info("✓ yt-dlp updated successfully") self._update_component_state('yt_dlp', True) return True else: self.logger.info("yt-dlp already up to date") self._update_component_state('yt_dlp', False) return False except subprocess.TimeoutExpired: self.logger.error("yt-dlp update timed out") return False except Exception as e: self.logger.error(f"yt-dlp update error: {e}") return False def _update_python_packages(self) -> bool: """ Update Python packages (FastAPI, Uvicorn, Pydantic, etc.) Returns: True if any updates were installed, False otherwise """ try: self.logger.info("Checking Python packages for updates...") # Get list of packages to update packages = self.config.get('components', {}).get('python_packages', {}).get('packages', []) if not packages: self.logger.info("No Python packages configured for updates") return False # Use venv pip (derived from module location for portability) venv_pip = str(self._venv_pip) updated_packages = [] for package in packages: try: # Check if package is pinned (should not be auto-updated) if package in self.pinned_packages: self.logger.info(f"⚠ Skipping {package}: {self.pinned_packages[package]}") continue # Check for version constraints if package in self.version_constraints: constraint_info = self.version_constraints[package] constraint = constraint_info.get('constraint', '') reason = constraint_info.get('reason', 'Version constraint') if constraint: # Install with constraint instead of --upgrade package_spec = f"{package}{constraint}" self.logger.info(f"📌 {package}: Applying constraint {constraint} ({reason})") result = subprocess.run( [venv_pip, 'install', package_spec], capture_output=True, text=True, timeout=120 ) else: # No constraint, normal upgrade result = subprocess.run( [venv_pip, 'install', '--upgrade', package], capture_output=True, text=True, timeout=120 ) else: # Update package normally result = subprocess.run( [venv_pip, 'install', '--upgrade', package], capture_output=True, text=True, timeout=120 ) if result.returncode == 0: output = result.stdout + result.stderr # Check if package was actually updated if "Successfully installed" in output and package in output: updated_packages.append(package) self.logger.info(f"✓ {package} updated") elif "Requirement already satisfied" in output: self.logger.debug(f" {package} already up to date") else: self.logger.debug(f" {package} checked") else: self.logger.warning(f"Failed to update {package}: {result.stderr}") except subprocess.TimeoutExpired: self.logger.warning(f"{package} update timed out") except Exception as e: self.logger.warning(f"Error updating {package}: {e}") if updated_packages: self.logger.info(f"✓ Updated {len(updated_packages)} Python package(s): {', '.join(updated_packages)}") self._update_component_state('python_packages', True) # Store list of updated packages in state if 'components' not in self.state: self.state['components'] = {} if 'python_packages' not in self.state['components']: self.state['components']['python_packages'] = {} self.state['components']['python_packages']['updated_packages'] = updated_packages self._save_state() return True else: self.logger.info("All Python packages already up to date") self._update_component_state('python_packages', False) return False except Exception as e: self.logger.error(f"Python packages update error: {e}") return False def _update_component_state(self, component: str, updated: bool): """Update component state in JSON""" if 'components' not in self.state: self.state['components'] = {} if component not in self.state['components']: self.state['components'][component] = {} self.state['components'][component]['last_update'] = datetime.now().isoformat() if updated else self.state['components'][component].get('last_update') self.state['components'][component]['last_check'] = datetime.now().isoformat() self.state['components'][component]['status'] = 'updated' if updated else 'current' self._save_state() def _send_update_notification(self, results: Dict[str, bool]): """Send Pushover notification about installed updates""" if not self.config.get('pushover', {}).get('enabled', True): return # Build list of updated components updated_components = [name for name, updated in results.items() if updated] if not updated_components: return # Check which components should send notifications notify_components = [] for component in updated_components: component_config = self.config.get('components', {}).get(component, {}) if component_config.get('notify_on_update', True): notify_components.append(component) if not notify_components: return # Format component names component_map = { 'flaresolverr': 'FlareSolverr', 'playwright': 'Playwright Browsers', 'yt_dlp': 'yt-dlp', 'python_packages': 'Python Packages' } formatted_names = [component_map.get(c, c) for c in notify_components] title = "🔄 Dependencies Updated" if len(formatted_names) == 1: message = f"{formatted_names[0]} has been updated to the latest version." else: message = f"The following components have been updated:\n\n" for name in formatted_names: message += f"• {name}\n" message += f"\nUpdated at: {datetime.now().strftime('%b %d, %I:%M %p')}" try: priority = self.config.get('pushover', {}).get('priority', -1) sound = self.config.get('pushover', {}).get('sound', 'magic') self.pushover.send_notification( title=title, message=message, priority=priority, sound=sound ) self.logger.info(f"Sent update notification for: {', '.join(formatted_names)}") except Exception as e: self.logger.error(f"Failed to send update notification: {e}") def get_update_status(self) -> Dict: """Get current update status for all components""" return self.state.copy() def force_update_check(self) -> Dict[str, bool]: """Force immediate update check regardless of interval or scheduler mode""" return self.check_and_update_all(force=True)