1885 lines
64 KiB
Python
1885 lines
64 KiB
Python
"""
|
|
Video Download Queue Router
|
|
|
|
Unified queue for managing video downloads from all sources.
|
|
Supports:
|
|
- Adding/removing videos from queue
|
|
- Editing video metadata (title, date)
|
|
- Priority management
|
|
- Download progress tracking
|
|
- Queue processing with start/pause controls
|
|
- Integration with celebrity discovery and manual downloads
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional
|
|
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, Query, Request
|
|
from pydantic import BaseModel
|
|
from slowapi import Limiter
|
|
from slowapi.util import get_remote_address
|
|
|
|
from ..core.dependencies import get_current_user, get_app_state
|
|
from ..core.config import settings
|
|
from ..core.exceptions import handle_exceptions, RecordNotFoundError, ValidationError
|
|
from modules.universal_logger import get_logger
|
|
|
|
logger = get_logger('API')
|
|
|
|
router = APIRouter(prefix="/api/video-queue", tags=["Video Queue"])
|
|
limiter = Limiter(key_func=get_remote_address)
|
|
|
|
|
|
# ============================================================================
|
|
# THUMBNAIL CACHING HELPER
|
|
# ============================================================================
|
|
|
|
async def cache_queue_thumbnail(platform: str, video_id: str, thumbnail_url: str, db) -> None:
|
|
"""Pre-cache thumbnail for faster Download Queue page loading."""
|
|
if not thumbnail_url:
|
|
return
|
|
try:
|
|
import httpx
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
response = await client.get(thumbnail_url, headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
if response.status_code == 200 and response.content:
|
|
with db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET thumbnail_data = ?
|
|
WHERE platform = ? AND video_id = ?
|
|
''', (response.content, platform, video_id))
|
|
conn.commit()
|
|
except Exception:
|
|
pass # Caching is optional, don't fail the request
|
|
|
|
|
|
# ============================================================================
|
|
# QUEUE PROCESSOR STATE
|
|
# ============================================================================
|
|
|
|
class QueueProcessor:
|
|
"""Manages the download queue processing state."""
|
|
|
|
def __init__(self):
|
|
self.is_running = False
|
|
self.is_paused = False
|
|
self.current_item_id: Optional[int] = None
|
|
self.current_video_id: Optional[str] = None
|
|
self.current_title: Optional[str] = None
|
|
self.processed_count = 0
|
|
self.failed_count = 0
|
|
self.started_at: Optional[str] = None
|
|
self._task: Optional[asyncio.Task] = None
|
|
self._lock = threading.Lock()
|
|
|
|
def get_status(self) -> Dict:
|
|
with self._lock:
|
|
return {
|
|
"is_running": self.is_running,
|
|
"is_paused": self.is_paused,
|
|
"current_item_id": self.current_item_id,
|
|
"current_video_id": self.current_video_id,
|
|
"current_title": self.current_title,
|
|
"processed_count": self.processed_count,
|
|
"failed_count": self.failed_count,
|
|
"started_at": self.started_at
|
|
}
|
|
|
|
def start(self):
|
|
with self._lock:
|
|
self.is_running = True
|
|
self.is_paused = False
|
|
self.started_at = datetime.now().isoformat()
|
|
|
|
def pause(self):
|
|
with self._lock:
|
|
self.is_paused = True
|
|
|
|
def resume(self):
|
|
with self._lock:
|
|
self.is_paused = False
|
|
|
|
def stop(self):
|
|
with self._lock:
|
|
self.is_running = False
|
|
self.is_paused = False
|
|
self.current_item_id = None
|
|
self.current_video_id = None
|
|
self.current_title = None
|
|
|
|
def reset_counts(self):
|
|
with self._lock:
|
|
self.processed_count = 0
|
|
self.failed_count = 0
|
|
|
|
def set_current(self, item_id: int, video_id: str, title: str):
|
|
with self._lock:
|
|
self.current_item_id = item_id
|
|
self.current_video_id = video_id
|
|
self.current_title = title
|
|
|
|
def increment_processed(self):
|
|
with self._lock:
|
|
self.processed_count += 1
|
|
|
|
def increment_failed(self):
|
|
with self._lock:
|
|
self.failed_count += 1
|
|
|
|
# Global queue processor instance
|
|
queue_processor = QueueProcessor()
|
|
|
|
|
|
def trigger_immich_scan(app_state) -> bool:
|
|
"""Trigger Immich library scan if configured.
|
|
|
|
Returns True if scan was triggered, False otherwise.
|
|
"""
|
|
import requests
|
|
|
|
# Get immich config from settings
|
|
immich_config = app_state.settings.get('immich', {})
|
|
|
|
# Check if immich is enabled and scan_after_download is enabled
|
|
if not immich_config.get('enabled'):
|
|
logger.debug("Immich not enabled, skipping scan", module="VideoQueue")
|
|
return False
|
|
|
|
if not immich_config.get('scan_after_download'):
|
|
logger.debug("Immich scan_after_download not enabled, skipping scan", module="VideoQueue")
|
|
return False
|
|
|
|
api_url = immich_config.get('api_url')
|
|
api_key = immich_config.get('api_key')
|
|
library_id = immich_config.get('library_id')
|
|
|
|
if not all([api_url, api_key, library_id]):
|
|
logger.warning("Immich config incomplete (missing api_url, api_key, or library_id)", module="VideoQueue")
|
|
return False
|
|
|
|
try:
|
|
response = requests.post(
|
|
f"{api_url}/libraries/{library_id}/scan",
|
|
headers={'X-API-KEY': api_key},
|
|
timeout=30
|
|
)
|
|
if response.status_code in [200, 201, 204]:
|
|
logger.info(f"Successfully triggered Immich scan for library {library_id}", module="VideoQueue")
|
|
return True
|
|
else:
|
|
logger.warning(f"Immich scan trigger failed: {response.status_code}", module="VideoQueue")
|
|
return False
|
|
except requests.Timeout:
|
|
logger.warning("Immich scan request timed out after 30 seconds", module="VideoQueue")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"Error triggering Immich scan: {e}", module="VideoQueue")
|
|
return False
|
|
|
|
|
|
# ============================================================================
|
|
# PYDANTIC MODELS
|
|
# ============================================================================
|
|
|
|
class QueueItemAdd(BaseModel):
|
|
platform: str = 'youtube'
|
|
video_id: str
|
|
url: str
|
|
title: str
|
|
custom_title: Optional[str] = None
|
|
channel_name: Optional[str] = None
|
|
thumbnail: Optional[str] = None
|
|
duration: Optional[int] = None
|
|
upload_date: Optional[str] = None
|
|
custom_date: Optional[str] = None
|
|
view_count: Optional[int] = None
|
|
description: Optional[str] = None
|
|
source_type: Optional[str] = None # 'celebrity', 'manual', 'search'
|
|
source_id: Optional[int] = None
|
|
source_name: Optional[str] = None
|
|
priority: int = 5
|
|
metadata: Optional[Dict] = None
|
|
|
|
|
|
class QueueItemUpdate(BaseModel):
|
|
custom_title: Optional[str] = None
|
|
custom_date: Optional[str] = None
|
|
priority: Optional[int] = None
|
|
status: Optional[str] = None
|
|
|
|
|
|
class BulkQueueAdd(BaseModel):
|
|
items: List[QueueItemAdd]
|
|
|
|
|
|
class BulkQueueAction(BaseModel):
|
|
ids: List[int]
|
|
action: str # 'remove', 'pause', 'resume', 'retry', 'prioritize'
|
|
priority: Optional[int] = None
|
|
|
|
|
|
# ============================================================================
|
|
# HELPER FUNCTIONS
|
|
# ============================================================================
|
|
|
|
def format_queue_item(row) -> Dict:
|
|
"""Format a queue item row for API response."""
|
|
# Handle max_resolution/max_width which may not exist in older connections
|
|
try:
|
|
max_res = row['max_resolution']
|
|
except (IndexError, KeyError):
|
|
max_res = None
|
|
try:
|
|
max_width = row['max_width']
|
|
except (IndexError, KeyError):
|
|
max_width = None
|
|
|
|
return {
|
|
'id': row['id'],
|
|
'platform': row['platform'],
|
|
'video_id': row['video_id'],
|
|
'url': row['url'],
|
|
'title': row['title'],
|
|
'custom_title': row['custom_title'],
|
|
'display_title': row['custom_title'] or row['title'],
|
|
'channel_name': row['channel_name'],
|
|
'thumbnail': row['thumbnail'],
|
|
'duration': row['duration'],
|
|
'upload_date': row['upload_date'],
|
|
'custom_date': row['custom_date'],
|
|
'display_date': row['custom_date'] or row['upload_date'],
|
|
'view_count': row['view_count'],
|
|
'max_resolution': max_res,
|
|
'max_width': max_width,
|
|
'description': row['description'],
|
|
'source_type': row['source_type'],
|
|
'source_id': row['source_id'],
|
|
'source_name': row['source_name'],
|
|
'priority': row['priority'],
|
|
'status': row['status'],
|
|
'progress': row['progress'],
|
|
'file_path': row['file_path'],
|
|
'file_size': row['file_size'],
|
|
'error_message': row['error_message'],
|
|
'attempts': row['attempts'],
|
|
'added_at': row['added_at'],
|
|
'started_at': row['started_at'],
|
|
'completed_at': row['completed_at'],
|
|
'metadata': json.loads(row['metadata']) if row['metadata'] else None
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# QUEUE ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_queue(
|
|
request: Request,
|
|
status: Optional[str] = Query(None),
|
|
source_type: Optional[str] = Query(None),
|
|
search: Optional[str] = Query(None),
|
|
sort: Optional[str] = Query('download_order'),
|
|
limit: int = Query(50, ge=1, le=200),
|
|
offset: int = Query(0, ge=0),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get video download queue with filters."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Build query
|
|
query = 'SELECT * FROM video_download_queue WHERE 1=1'
|
|
count_query = 'SELECT COUNT(*) FROM video_download_queue WHERE 1=1'
|
|
params = []
|
|
|
|
if status:
|
|
if status == 'pending':
|
|
# Pending filter shows both pending and downloading items
|
|
query += ' AND status IN (?, ?)'
|
|
count_query += ' AND status IN (?, ?)'
|
|
params.extend(['pending', 'downloading'])
|
|
else:
|
|
query += ' AND status = ?'
|
|
count_query += ' AND status = ?'
|
|
params.append(status)
|
|
|
|
if source_type:
|
|
query += ' AND source_type = ?'
|
|
count_query += ' AND source_type = ?'
|
|
params.append(source_type)
|
|
|
|
if search:
|
|
# Search in title, url, platform, and channel_name
|
|
search_pattern = f'%{search}%'
|
|
query += ' AND (title LIKE ? OR url LIKE ? OR platform LIKE ? OR channel_name LIKE ?)'
|
|
count_query += ' AND (title LIKE ? OR url LIKE ? OR platform LIKE ? OR channel_name LIKE ?)'
|
|
params.extend([search_pattern, search_pattern, search_pattern, search_pattern])
|
|
|
|
# Get total count
|
|
cursor.execute(count_query, params)
|
|
total = cursor.fetchone()[0]
|
|
|
|
# Get items - sort order based on parameter
|
|
if sort == 'recently_added':
|
|
# Recently added first
|
|
query += ' ORDER BY added_at DESC LIMIT ? OFFSET ?'
|
|
else:
|
|
# Download order (matches processor: priority ASC, oldest first)
|
|
query += ' ORDER BY priority ASC, added_at ASC LIMIT ? OFFSET ?'
|
|
params.extend([limit, offset])
|
|
cursor.execute(query, params)
|
|
|
|
items = [format_queue_item(row) for row in cursor.fetchall()]
|
|
|
|
# Get stats
|
|
cursor.execute('''
|
|
SELECT status, COUNT(*) as count
|
|
FROM video_download_queue
|
|
GROUP BY status
|
|
''')
|
|
status_counts = {row['status']: row['count'] for row in cursor.fetchall()}
|
|
|
|
# Build stats object with all expected fields
|
|
stats = {
|
|
"total": total,
|
|
"pending": status_counts.get('pending', 0),
|
|
"downloading": status_counts.get('downloading', 0),
|
|
"completed": status_counts.get('completed', 0),
|
|
"failed": status_counts.get('failed', 0),
|
|
"paused": status_counts.get('paused', 0)
|
|
}
|
|
|
|
return {
|
|
"success": True,
|
|
"items": items,
|
|
"total": total,
|
|
"stats": stats
|
|
}
|
|
|
|
|
|
@router.get("/stats")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_queue_stats(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get queue statistics."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Status counts
|
|
cursor.execute('''
|
|
SELECT status, COUNT(*) as count
|
|
FROM video_download_queue
|
|
GROUP BY status
|
|
''')
|
|
by_status = {row['status']: row['count'] for row in cursor.fetchall()}
|
|
|
|
# Source type counts
|
|
cursor.execute('''
|
|
SELECT source_type, COUNT(*) as count
|
|
FROM video_download_queue
|
|
GROUP BY source_type
|
|
''')
|
|
by_source = {row['source_type'] or 'unknown': row['count'] for row in cursor.fetchall()}
|
|
|
|
# Total and recent
|
|
cursor.execute('SELECT COUNT(*) FROM video_download_queue')
|
|
total = cursor.fetchone()[0]
|
|
|
|
cursor.execute('''
|
|
SELECT COUNT(*) FROM video_download_queue
|
|
WHERE added_at > datetime('now', '-24 hours')
|
|
''')
|
|
last_24h = cursor.fetchone()[0]
|
|
|
|
return {
|
|
"success": True,
|
|
"stats": {
|
|
"total": total,
|
|
"by_status": by_status,
|
|
"by_source": by_source,
|
|
"pending": by_status.get('pending', 0),
|
|
"downloading": by_status.get('downloading', 0),
|
|
"completed": by_status.get('completed', 0),
|
|
"failed": by_status.get('failed', 0),
|
|
"last_24h": last_24h
|
|
}
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# QUEUE SETTINGS ENDPOINTS (must be before /{queue_id} to avoid route conflict)
|
|
# ============================================================================
|
|
|
|
class QueueSettingsUpdate(BaseModel):
|
|
download_delay_seconds: Optional[int] = None
|
|
base_directory: Optional[str] = None
|
|
stop_on_cookie_error: Optional[bool] = None
|
|
send_cookie_notification: Optional[bool] = None
|
|
auto_start_on_restart: Optional[bool] = None
|
|
|
|
|
|
@router.get("/settings")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_queue_settings(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get video queue settings."""
|
|
app_state = get_app_state()
|
|
|
|
# Default settings
|
|
settings = {
|
|
"download_delay_seconds": 15,
|
|
"base_directory": "/opt/immich/md",
|
|
"stop_on_cookie_error": True,
|
|
"send_cookie_notification": True,
|
|
"auto_start_on_restart": False
|
|
}
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
# Get queue settings
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'video_queue'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
try:
|
|
stored = json.loads(row[0])
|
|
settings.update(stored)
|
|
except Exception:
|
|
pass
|
|
|
|
# Also check download_settings for base_directory (legacy compatibility)
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'download_settings'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
try:
|
|
stored = json.loads(row[0])
|
|
if 'base_directory' in stored:
|
|
settings['base_directory'] = stored['base_directory']
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"success": True,
|
|
"settings": settings
|
|
}
|
|
|
|
|
|
@router.post("/settings")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_queue_settings(
|
|
request: Request,
|
|
update: QueueSettingsUpdate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update video queue settings."""
|
|
import os
|
|
app_state = get_app_state()
|
|
|
|
# Get current settings
|
|
settings = {
|
|
"download_delay_seconds": 15,
|
|
"base_directory": "/opt/immich/md",
|
|
"stop_on_cookie_error": True,
|
|
"send_cookie_notification": True,
|
|
"auto_start_on_restart": False
|
|
}
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'video_queue'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
try:
|
|
stored = json.loads(row[0])
|
|
settings.update(stored)
|
|
except Exception:
|
|
pass
|
|
|
|
# Update with new values
|
|
if update.download_delay_seconds is not None:
|
|
settings['download_delay_seconds'] = max(0, min(120, update.download_delay_seconds))
|
|
|
|
if update.base_directory is not None:
|
|
# Validate path exists or can be created
|
|
base_dir = update.base_directory.strip()
|
|
if base_dir:
|
|
try:
|
|
os.makedirs(base_dir, exist_ok=True)
|
|
settings['base_directory'] = base_dir
|
|
except Exception as e:
|
|
raise ValueError(f"Cannot create directory {base_dir}: {e}")
|
|
|
|
if update.stop_on_cookie_error is not None:
|
|
settings['stop_on_cookie_error'] = update.stop_on_cookie_error
|
|
|
|
if update.send_cookie_notification is not None:
|
|
settings['send_cookie_notification'] = update.send_cookie_notification
|
|
|
|
if update.auto_start_on_restart is not None:
|
|
settings['auto_start_on_restart'] = update.auto_start_on_restart
|
|
|
|
# Save settings to video_queue key
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO settings (key, value, value_type, category, description)
|
|
VALUES ('video_queue', ?, 'json', 'queue', 'Video download queue settings')
|
|
''', (json.dumps(settings),))
|
|
|
|
# Also save to download_settings for universal_video_downloader compatibility
|
|
download_settings = {"base_directory": settings['base_directory']}
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO settings (key, value, value_type, category, description)
|
|
VALUES ('download_settings', ?, 'json', 'downloads', 'Download settings')
|
|
''', (json.dumps(download_settings),))
|
|
|
|
conn.commit()
|
|
|
|
logger.info(f"Queue settings updated: delay={settings['download_delay_seconds']}s, base_dir={settings['base_directory']}", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"settings": settings
|
|
}
|
|
|
|
|
|
# Default anti-bot settings
|
|
DEFAULT_ANTIBOT_SETTINGS = {
|
|
'browser': 'edge',
|
|
'custom_user_agent': '',
|
|
'limit_rate': '2M',
|
|
'throttled_rate': '100K',
|
|
'sleep_requests_min': 1,
|
|
'sleep_requests_max': 3,
|
|
'retries': 10,
|
|
'fragment_retries': 10,
|
|
'concurrent_fragments': 1,
|
|
'socket_timeout': 30,
|
|
'enabled': True,
|
|
}
|
|
|
|
|
|
@router.get("/antibot-settings")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_antibot_settings(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get anti-bot protection settings."""
|
|
app_state = get_app_state()
|
|
|
|
settings = DEFAULT_ANTIBOT_SETTINGS.copy()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'antibot_settings'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
try:
|
|
stored = json.loads(row[0])
|
|
settings.update(stored)
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"success": True,
|
|
"settings": settings
|
|
}
|
|
|
|
|
|
class AntibotSettingsUpdate(BaseModel):
|
|
browser: Optional[str] = None
|
|
custom_user_agent: Optional[str] = None
|
|
limit_rate: Optional[str] = None
|
|
throttled_rate: Optional[str] = None
|
|
sleep_requests_min: Optional[int] = None
|
|
sleep_requests_max: Optional[int] = None
|
|
retries: Optional[int] = None
|
|
fragment_retries: Optional[int] = None
|
|
concurrent_fragments: Optional[int] = None
|
|
socket_timeout: Optional[int] = None
|
|
enabled: Optional[bool] = None
|
|
|
|
|
|
@router.post("/antibot-settings")
|
|
@limiter.limit("20/minute")
|
|
@handle_exceptions
|
|
async def update_antibot_settings(
|
|
request: Request,
|
|
update: AntibotSettingsUpdate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update anti-bot protection settings."""
|
|
app_state = get_app_state()
|
|
|
|
settings = DEFAULT_ANTIBOT_SETTINGS.copy()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'antibot_settings'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
try:
|
|
stored = json.loads(row[0])
|
|
settings.update(stored)
|
|
except Exception:
|
|
pass
|
|
|
|
# Update with new values
|
|
update_dict = update.model_dump(exclude_none=True)
|
|
settings.update(update_dict)
|
|
|
|
# Validate values
|
|
if settings.get('browser') not in ['edge', 'chrome', 'firefox', 'safari', 'custom']:
|
|
settings['browser'] = 'edge'
|
|
|
|
if settings.get('sleep_requests_min', 0) < 0:
|
|
settings['sleep_requests_min'] = 0
|
|
if settings.get('sleep_requests_max', 0) < settings.get('sleep_requests_min', 0):
|
|
settings['sleep_requests_max'] = settings['sleep_requests_min']
|
|
|
|
if settings.get('retries', 1) < 1:
|
|
settings['retries'] = 1
|
|
if settings.get('fragment_retries', 1) < 1:
|
|
settings['fragment_retries'] = 1
|
|
|
|
if settings.get('concurrent_fragments', 1) < 1:
|
|
settings['concurrent_fragments'] = 1
|
|
|
|
if settings.get('socket_timeout', 10) < 10:
|
|
settings['socket_timeout'] = 10
|
|
|
|
# Save settings
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO settings (key, value, value_type, category, description)
|
|
VALUES ('antibot_settings', ?, 'json', 'queue', 'Anti-bot protection settings for yt-dlp')
|
|
''', (json.dumps(settings),))
|
|
|
|
conn.commit()
|
|
|
|
logger.info(f"Anti-bot settings updated: browser={settings['browser']}, enabled={settings['enabled']}", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"settings": settings
|
|
}
|
|
|
|
|
|
@router.post("/add")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def add_to_queue(
|
|
request: Request,
|
|
item: QueueItemAdd,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add a video to the download queue."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
try:
|
|
cursor.execute('''
|
|
INSERT INTO video_download_queue (
|
|
platform, video_id, url, title, custom_title, channel_name,
|
|
thumbnail, duration, upload_date, custom_date, view_count,
|
|
description, source_type, source_id, source_name, priority, metadata
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (
|
|
item.platform, item.video_id, item.url, item.title, item.custom_title,
|
|
item.channel_name, item.thumbnail, item.duration, item.upload_date,
|
|
item.custom_date, item.view_count, item.description, item.source_type,
|
|
item.source_id, item.source_name, item.priority,
|
|
json.dumps(item.metadata) if item.metadata else None
|
|
))
|
|
conn.commit()
|
|
|
|
queue_id = cursor.lastrowid
|
|
|
|
# Pre-cache thumbnail for faster page loading
|
|
if item.thumbnail:
|
|
await cache_queue_thumbnail(item.platform, item.video_id, item.thumbnail, app_state.db)
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "Added to queue",
|
|
"id": queue_id
|
|
}
|
|
|
|
except Exception as e:
|
|
if "UNIQUE constraint failed" in str(e):
|
|
return {
|
|
"success": False,
|
|
"message": "Video already in queue"
|
|
}
|
|
raise
|
|
|
|
|
|
@router.post("/add-bulk")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def add_bulk_to_queue(
|
|
request: Request,
|
|
data: BulkQueueAdd,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add multiple videos to the download queue."""
|
|
app_state = get_app_state()
|
|
|
|
added = 0
|
|
skipped = 0
|
|
|
|
added_items = [] # Track items for thumbnail caching
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
for item in data.items:
|
|
try:
|
|
cursor.execute('''
|
|
INSERT INTO video_download_queue (
|
|
platform, video_id, url, title, custom_title, channel_name,
|
|
thumbnail, duration, upload_date, custom_date, view_count,
|
|
description, source_type, source_id, source_name, priority, metadata
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (
|
|
item.platform, item.video_id, item.url, item.title, item.custom_title,
|
|
item.channel_name, item.thumbnail, item.duration, item.upload_date,
|
|
item.custom_date, item.view_count, item.description, item.source_type,
|
|
item.source_id, item.source_name, item.priority,
|
|
json.dumps(item.metadata) if item.metadata else None
|
|
))
|
|
added += 1
|
|
if item.thumbnail:
|
|
added_items.append((item.platform, item.video_id, item.thumbnail))
|
|
except Exception:
|
|
skipped += 1
|
|
|
|
conn.commit()
|
|
|
|
# Pre-cache thumbnails for faster page loading (async, non-blocking)
|
|
for platform, video_id, thumbnail_url in added_items:
|
|
await cache_queue_thumbnail(platform, video_id, thumbnail_url, app_state.db)
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Added {added} videos, {skipped} already in queue",
|
|
"added": added,
|
|
"skipped": skipped
|
|
}
|
|
|
|
|
|
@router.get("/{queue_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_queue_item(
|
|
request: Request,
|
|
queue_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get a single queue item."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT * FROM video_download_queue WHERE id = ?', (queue_id,))
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
raise RecordNotFoundError("Queue item not found")
|
|
|
|
return {
|
|
"success": True,
|
|
"item": format_queue_item(row)
|
|
}
|
|
|
|
|
|
@router.get("/{queue_id}/stream")
|
|
async def stream_downloaded_video(
|
|
request: Request,
|
|
queue_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Stream a downloaded video file.
|
|
|
|
Returns the video file with proper Range support for seeking.
|
|
Only works for completed downloads that have a file_path.
|
|
"""
|
|
import os
|
|
from starlette.responses import StreamingResponse
|
|
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT file_path, status FROM video_download_queue WHERE id = ?', (queue_id,))
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
raise RecordNotFoundError("Queue item not found")
|
|
|
|
file_path = row['file_path']
|
|
status = row['status']
|
|
|
|
if status != 'completed':
|
|
raise ValidationError("Video has not been downloaded yet")
|
|
|
|
if not file_path or not os.path.exists(file_path):
|
|
raise RecordNotFoundError("Downloaded file not found")
|
|
|
|
file_size = os.path.getsize(file_path)
|
|
|
|
# Determine content type
|
|
ext = os.path.splitext(file_path)[1].lower()
|
|
content_type = {
|
|
'.mp4': 'video/mp4',
|
|
'.webm': 'video/webm',
|
|
'.mkv': 'video/x-matroska',
|
|
'.mov': 'video/quicktime',
|
|
'.avi': 'video/x-msvideo',
|
|
}.get(ext, 'video/mp4')
|
|
|
|
# Handle Range requests for seeking
|
|
range_header = request.headers.get("Range")
|
|
start = 0
|
|
end = file_size - 1
|
|
|
|
if range_header:
|
|
range_match = range_header.replace("bytes=", "").split("-")
|
|
if range_match[0]:
|
|
start = int(range_match[0])
|
|
if len(range_match) > 1 and range_match[1]:
|
|
end = min(int(range_match[1]), file_size - 1)
|
|
|
|
content_length = end - start + 1
|
|
|
|
def file_stream_generator():
|
|
with open(file_path, 'rb') as f:
|
|
f.seek(start)
|
|
remaining = content_length
|
|
while remaining > 0:
|
|
chunk_size = min(65536, remaining)
|
|
chunk = f.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
remaining -= len(chunk)
|
|
yield chunk
|
|
|
|
headers = {
|
|
"Accept-Ranges": "bytes",
|
|
"Content-Length": str(content_length),
|
|
"Cache-Control": "private, max-age=3600",
|
|
}
|
|
|
|
if range_header:
|
|
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
|
return StreamingResponse(
|
|
file_stream_generator(),
|
|
status_code=206,
|
|
media_type=content_type,
|
|
headers=headers
|
|
)
|
|
|
|
return StreamingResponse(
|
|
file_stream_generator(),
|
|
media_type=content_type,
|
|
headers=headers
|
|
)
|
|
|
|
|
|
@router.patch("/{queue_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_queue_item(
|
|
request: Request,
|
|
queue_id: int,
|
|
update: QueueItemUpdate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update a queue item (edit title, date, priority, status)."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Build update query dynamically
|
|
updates = []
|
|
params = []
|
|
|
|
if update.custom_title is not None:
|
|
updates.append('custom_title = ?')
|
|
params.append(update.custom_title if update.custom_title else None)
|
|
|
|
if update.custom_date is not None:
|
|
updates.append('custom_date = ?')
|
|
params.append(update.custom_date if update.custom_date else None)
|
|
|
|
if update.priority is not None:
|
|
updates.append('priority = ?')
|
|
params.append(update.priority)
|
|
|
|
if update.status is not None:
|
|
updates.append('status = ?')
|
|
params.append(update.status)
|
|
|
|
if not updates:
|
|
raise ValidationError("No fields to update")
|
|
|
|
params.append(queue_id)
|
|
query = f"UPDATE video_download_queue SET {', '.join(updates)} WHERE id = ?"
|
|
|
|
cursor.execute(query, params)
|
|
if cursor.rowcount == 0:
|
|
raise RecordNotFoundError("Queue item not found")
|
|
|
|
conn.commit()
|
|
|
|
# Return updated item
|
|
cursor.execute('SELECT * FROM video_download_queue WHERE id = ?', (queue_id,))
|
|
row = cursor.fetchone()
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "Queue item updated",
|
|
"item": format_queue_item(row)
|
|
}
|
|
|
|
|
|
@router.delete("/{queue_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def remove_from_queue(
|
|
request: Request,
|
|
queue_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove a video from the queue."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('DELETE FROM video_download_queue WHERE id = ?', (queue_id,))
|
|
|
|
if cursor.rowcount == 0:
|
|
raise RecordNotFoundError("Queue item not found")
|
|
|
|
conn.commit()
|
|
|
|
return {"success": True, "message": "Removed from queue"}
|
|
|
|
|
|
@router.post("/bulk-action")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def bulk_queue_action(
|
|
request: Request,
|
|
data: BulkQueueAction,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Perform bulk actions on queue items."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
affected = 0
|
|
placeholders = ','.join(['?' for _ in data.ids])
|
|
|
|
if data.action == 'remove':
|
|
cursor.execute(f'DELETE FROM video_download_queue WHERE id IN ({placeholders})', data.ids)
|
|
affected = cursor.rowcount
|
|
|
|
elif data.action == 'pause':
|
|
cursor.execute(f'''
|
|
UPDATE video_download_queue
|
|
SET status = 'paused'
|
|
WHERE id IN ({placeholders}) AND status = 'pending'
|
|
''', data.ids)
|
|
affected = cursor.rowcount
|
|
|
|
elif data.action == 'resume':
|
|
cursor.execute(f'''
|
|
UPDATE video_download_queue
|
|
SET status = 'pending'
|
|
WHERE id IN ({placeholders}) AND status = 'paused'
|
|
''', data.ids)
|
|
affected = cursor.rowcount
|
|
|
|
elif data.action == 'retry':
|
|
cursor.execute(f'''
|
|
UPDATE video_download_queue
|
|
SET status = 'pending', attempts = 0, error_message = NULL
|
|
WHERE id IN ({placeholders}) AND status = 'failed'
|
|
''', data.ids)
|
|
affected = cursor.rowcount
|
|
|
|
elif data.action == 'prioritize':
|
|
if data.priority is not None:
|
|
cursor.execute(f'''
|
|
UPDATE video_download_queue
|
|
SET priority = ?
|
|
WHERE id IN ({placeholders})
|
|
''', [data.priority] + data.ids)
|
|
affected = cursor.rowcount
|
|
|
|
conn.commit()
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"{data.action} applied to {affected} items",
|
|
"affected": affected
|
|
}
|
|
|
|
|
|
@router.post("/clear-completed")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def clear_completed(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Clear all completed downloads from queue."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM video_download_queue WHERE status = 'completed'")
|
|
deleted = cursor.rowcount
|
|
conn.commit()
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Cleared {deleted} completed items",
|
|
"deleted": deleted
|
|
}
|
|
|
|
|
|
@router.post("/clear-failed")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def clear_failed(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Clear all failed downloads from queue."""
|
|
app_state = get_app_state()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM video_download_queue WHERE status = 'failed'")
|
|
deleted = cursor.rowcount
|
|
conn.commit()
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Cleared {deleted} failed items",
|
|
"deleted": deleted
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# CELEBRITY DISCOVERY INTEGRATION
|
|
# ============================================================================
|
|
|
|
@router.post("/add-from-celebrity")
|
|
@limiter.limit("20/minute")
|
|
@handle_exceptions
|
|
async def add_from_celebrity_discovery(
|
|
request: Request,
|
|
video_ids: List[int],
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add videos from celebrity discovery to the download queue."""
|
|
app_state = get_app_state()
|
|
|
|
added = 0
|
|
skipped = 0
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Get the celebrity videos
|
|
placeholders = ','.join(['?' for _ in video_ids])
|
|
cursor.execute(f'''
|
|
SELECT v.*, cp.name as celebrity_name, cp.id as celeb_id
|
|
FROM celebrity_discovered_videos v
|
|
JOIN celebrity_profiles cp ON v.celebrity_id = cp.id
|
|
WHERE v.id IN ({placeholders})
|
|
''', video_ids)
|
|
|
|
for row in cursor.fetchall():
|
|
try:
|
|
cursor.execute('''
|
|
INSERT INTO video_download_queue (
|
|
platform, video_id, url, title, channel_name, thumbnail,
|
|
duration, upload_date, view_count, description,
|
|
source_type, source_id, source_name, priority
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (
|
|
row['platform'], row['video_id'], row['url'], row['title'],
|
|
row['channel_name'], row['thumbnail'], row['duration'],
|
|
row['upload_date'], row['view_count'], row['description'],
|
|
'celebrity', row['celeb_id'], row['celebrity_name'], 5
|
|
))
|
|
added += 1
|
|
|
|
# Update celebrity video status to 'queued'
|
|
cursor.execute('''
|
|
UPDATE celebrity_discovered_videos
|
|
SET status = 'queued', status_updated_at = ?
|
|
WHERE id = ?
|
|
''', (datetime.now().isoformat(), row['id']))
|
|
|
|
except Exception:
|
|
skipped += 1
|
|
|
|
conn.commit()
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Added {added} videos to queue, {skipped} already in queue",
|
|
"added": added,
|
|
"skipped": skipped
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# QUEUE PROCESSOR ENDPOINTS
|
|
# ============================================================================
|
|
|
|
async def ensure_video_thumbnail(app_state, platform: str, video_id: str, thumbnail_url: str = None):
|
|
"""
|
|
Ensure video has a thumbnail stored in video_downloads and thumbnails.db cache.
|
|
Fetches from YouTube URL if not present.
|
|
"""
|
|
import httpx
|
|
import hashlib
|
|
import sqlite3
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
try:
|
|
# Check if thumbnail already exists in video_downloads
|
|
file_path = None
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute(
|
|
"SELECT thumbnail_data, file_path FROM video_downloads WHERE platform = ? AND video_id = ?",
|
|
(platform, video_id)
|
|
)
|
|
row = cursor.fetchone()
|
|
if row:
|
|
file_path = row[1]
|
|
if row[0] and len(row[0]) > 1000:
|
|
# Already has thumbnail in video_downloads, but sync to thumbnails.db
|
|
_sync_to_thumbnail_cache(file_path, row[0])
|
|
return
|
|
|
|
# Determine thumbnail URL - ALWAYS use standardized URL for known platforms
|
|
# YouTube metadata often returns weird URLs with query params that don't work
|
|
if platform == 'youtube':
|
|
# Try maxresdefault first (1280x720, no black bars), fallback to hqdefault
|
|
thumbnail_data = None
|
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
for quality in ['maxresdefault', 'hqdefault']:
|
|
thumbnail_url = f"https://i.ytimg.com/vi/{video_id}/{quality}.jpg"
|
|
response = await client.get(thumbnail_url)
|
|
if response.status_code == 200 and len(response.content) > 1000:
|
|
thumbnail_data = response.content
|
|
break
|
|
elif platform == 'dailymotion':
|
|
thumbnail_url = f"https://www.dailymotion.com/thumbnail/video/{video_id}"
|
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
response = await client.get(thumbnail_url)
|
|
if response.status_code == 200 and len(response.content) > 1000:
|
|
thumbnail_data = response.content
|
|
elif thumbnail_url:
|
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
response = await client.get(thumbnail_url)
|
|
if response.status_code == 200 and len(response.content) > 1000:
|
|
thumbnail_data = response.content
|
|
else:
|
|
return # No known thumbnail URL pattern
|
|
|
|
if thumbnail_data:
|
|
# Store in video_downloads
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute(
|
|
"UPDATE video_downloads SET thumbnail_data = ? WHERE platform = ? AND video_id = ?",
|
|
(thumbnail_data, platform, video_id)
|
|
)
|
|
conn.commit()
|
|
|
|
# Also sync to thumbnails.db cache for Downloads/Media pages
|
|
if file_path:
|
|
_sync_to_thumbnail_cache(file_path, thumbnail_data)
|
|
|
|
logger.debug(f"Stored thumbnail for {platform}/{video_id}", module="VideoQueue")
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Failed to fetch thumbnail for {video_id}: {e}", module="VideoQueue")
|
|
|
|
|
|
def _sync_to_thumbnail_cache(file_path: str, thumbnail_data: bytes):
|
|
"""Sync a thumbnail to the thumbnails.db cache used by Downloads/Media pages."""
|
|
import hashlib
|
|
import sqlite3
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
try:
|
|
thumb_db_path = settings.PROJECT_ROOT / 'database' / 'thumbnails.db'
|
|
# MUST use SHA256 to match get_or_create_thumbnail() in media.py
|
|
file_hash = hashlib.sha256(file_path.encode()).hexdigest()
|
|
|
|
# Get file mtime
|
|
try:
|
|
file_mtime = Path(file_path).stat().st_mtime
|
|
except OSError:
|
|
file_mtime = 0
|
|
|
|
with sqlite3.connect(str(thumb_db_path), timeout=10.0) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT OR REPLACE INTO thumbnails
|
|
(file_hash, file_path, thumbnail_data, created_at, file_mtime)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""", (file_hash, file_path, thumbnail_data, datetime.now().isoformat(), file_mtime))
|
|
conn.commit()
|
|
except Exception as e:
|
|
logger.debug(f"Failed to sync thumbnail to cache: {e}", module="VideoQueue")
|
|
|
|
|
|
async def process_queue_item(item: Dict, app_state) -> bool:
|
|
"""Process a single queue item. Returns True on success, False on failure."""
|
|
import tempfile
|
|
import os
|
|
from modules.universal_video_downloader import UniversalVideoDownloader
|
|
|
|
item_id = item['id']
|
|
video_id = item['video_id']
|
|
url = item['url']
|
|
platform = item['platform']
|
|
title = item['title']
|
|
|
|
cookies_file = None
|
|
|
|
try:
|
|
# Update status to downloading
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET status = 'downloading', started_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
''', (item_id,))
|
|
conn.commit()
|
|
|
|
queue_processor.set_current(item_id, video_id, title)
|
|
logger.info(f"Queue: Starting download of {title[:50]}...", module="VideoQueue")
|
|
|
|
# Determine which scraper to get cookies from based on platform
|
|
# gallery-dl sites: erome, bunkr, cyberdrop, etc.
|
|
gallery_dl_platforms = ['erome', 'bunkr', 'cyberdrop', 'coomer', 'kemono', 'fapello']
|
|
if platform in gallery_dl_platforms:
|
|
scraper_id = 'gallerydl'
|
|
scraper_name = 'gallery-dl'
|
|
else:
|
|
scraper_id = 'ytdlp'
|
|
scraper_name = 'yt-dlp'
|
|
|
|
# Get cookies from scraper settings if available
|
|
cookies_file = None
|
|
try:
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT cookies_json FROM scrapers WHERE id = ?", (scraper_id,))
|
|
row = cursor.fetchone()
|
|
if row and row[0]:
|
|
data = json.loads(row[0])
|
|
# Support both {"cookies": [...]} and [...] formats
|
|
if isinstance(data, dict) and 'cookies' in data:
|
|
cookies_list = data['cookies']
|
|
elif isinstance(data, list):
|
|
cookies_list = data
|
|
else:
|
|
cookies_list = []
|
|
|
|
if cookies_list:
|
|
# Write cookies to temp file in Netscape format
|
|
fd, cookies_file = tempfile.mkstemp(suffix='.txt', prefix=f'{scraper_id}_cookies_')
|
|
with os.fdopen(fd, 'w') as f:
|
|
f.write("# Netscape HTTP Cookie File\n")
|
|
for cookie in cookies_list:
|
|
# Format: domain, include_subdomains, path, secure, expiry, name, value
|
|
domain = cookie.get('domain', '')
|
|
include_subdomains = 'TRUE' if domain.startswith('.') else 'FALSE'
|
|
path = cookie.get('path', '/')
|
|
secure = 'TRUE' if cookie.get('secure', False) else 'FALSE'
|
|
expiry = str(int(cookie.get('expirationDate', 0)))
|
|
name = cookie.get('name', '')
|
|
value = cookie.get('value', '')
|
|
f.write(f"{domain}\t{include_subdomains}\t{path}\t{secure}\t{expiry}\t{name}\t{value}\n")
|
|
logger.debug(f"Queue: Using {len(cookies_list)} cookies from {scraper_name} scraper", module="VideoQueue")
|
|
except Exception as e:
|
|
logger.debug(f"Queue: Could not load cookies: {e}", module="VideoQueue")
|
|
|
|
# Create downloader and download
|
|
downloader = UniversalVideoDownloader(platform=platform, unified_db=app_state.db, cookies_file=cookies_file)
|
|
|
|
# Progress callback to update queue item
|
|
def progress_callback(message, percentage, speed=None, eta=None):
|
|
try:
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET progress = ?
|
|
WHERE id = ?
|
|
''', (int(percentage), item_id))
|
|
conn.commit()
|
|
except Exception:
|
|
pass # Don't fail download due to progress update issues
|
|
|
|
# Execute download in thread pool to avoid blocking the event loop
|
|
import asyncio
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
loop = asyncio.get_event_loop()
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
success, file_path, metadata = await loop.run_in_executor(
|
|
executor,
|
|
lambda: downloader.download_video(url, progress_callback=progress_callback, update_activity=False)
|
|
)
|
|
|
|
if success:
|
|
# Update status to completed
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET status = 'completed', progress = 100, file_path = ?,
|
|
completed_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
''', (file_path, item_id))
|
|
|
|
# Also update celebrity_discovered_videos if this came from there
|
|
if item.get('source_type') == 'celebrity':
|
|
cursor.execute('''
|
|
UPDATE celebrity_discovered_videos
|
|
SET status = 'downloaded', status_updated_at = CURRENT_TIMESTAMP,
|
|
downloaded_path = ?
|
|
WHERE video_id = ?
|
|
''', (file_path, video_id))
|
|
|
|
# Mark in file_inventory so it doesn't show on Media dashboard card
|
|
# (user already actioned this via Internet Discovery card)
|
|
cursor.execute('''
|
|
UPDATE file_inventory
|
|
SET from_discovery = 1
|
|
WHERE file_path = ?
|
|
''', (file_path,))
|
|
|
|
conn.commit()
|
|
|
|
logger.info(f"Queue: Completed download of {title[:50]}", module="VideoQueue")
|
|
|
|
# Fetch and store thumbnail if not already present
|
|
await ensure_video_thumbnail(app_state, platform, video_id, item.get('thumbnail'))
|
|
|
|
# Trigger Immich scan if configured
|
|
trigger_immich_scan(app_state)
|
|
|
|
return {'success': True}
|
|
else:
|
|
# Check if this is a cookie error
|
|
if metadata and metadata.get('cookie_error'):
|
|
# Requeue the item (set back to pending)
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET status = 'pending', progress = 0, error_message = 'Cookie expired - requeued'
|
|
WHERE id = ?
|
|
''', (item_id,))
|
|
conn.commit()
|
|
|
|
logger.warning(f"Queue: Cookie error detected for {title[:50]}, requeued", module="VideoQueue")
|
|
return {'success': False, 'cookie_error': True, 'platform': platform}
|
|
else:
|
|
raise Exception(metadata.get('error', 'Download returned failure') if metadata else 'Download returned failure')
|
|
|
|
except Exception as e:
|
|
error_msg = str(e)[:500]
|
|
logger.error(f"Queue: Failed to download {title[:50]}: {error_msg}", module="VideoQueue")
|
|
|
|
# Update status to failed
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
UPDATE video_download_queue
|
|
SET status = 'failed', error_message = ?, attempts = attempts + 1
|
|
WHERE id = ?
|
|
''', (error_msg, item_id))
|
|
conn.commit()
|
|
|
|
return {'success': False}
|
|
|
|
finally:
|
|
# Clean up temp cookies file
|
|
if cookies_file:
|
|
try:
|
|
import os
|
|
os.unlink(cookies_file)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _get_queue_settings_from_db(app_state) -> dict:
|
|
"""Get queue settings from database."""
|
|
try:
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'video_queue'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
return json.loads(row[0])
|
|
except Exception:
|
|
pass
|
|
return {
|
|
'download_delay_seconds': 15,
|
|
'stop_on_cookie_error': True,
|
|
'send_cookie_notification': True
|
|
}
|
|
|
|
|
|
async def send_cookie_expired_notification(platform: str, app_state):
|
|
"""Send push notification about expired cookies."""
|
|
# Check if notifications are enabled in settings
|
|
settings = _get_queue_settings_from_db(app_state)
|
|
if not settings.get('send_cookie_notification', True):
|
|
logger.debug("Cookie notification disabled in settings, skipping", module="VideoQueue")
|
|
return
|
|
|
|
try:
|
|
from modules.pushover_notifier import PushoverNotifier
|
|
|
|
# Get pushover config from settings
|
|
pushover_config = app_state.settings.get('pushover', {})
|
|
if not pushover_config.get('enabled'):
|
|
logger.debug("Pushover notifications disabled", module="VideoQueue")
|
|
return
|
|
|
|
user_key = pushover_config.get('user_key')
|
|
api_token = pushover_config.get('api_token')
|
|
|
|
if not user_key or not api_token:
|
|
logger.warning("Pushover credentials not configured", module="VideoQueue")
|
|
return
|
|
|
|
notifier = PushoverNotifier(
|
|
user_key=user_key,
|
|
api_token=api_token,
|
|
enabled=True,
|
|
default_priority=pushover_config.get('priority', 0),
|
|
device=pushover_config.get('device'),
|
|
include_image=pushover_config.get('include_image', True)
|
|
)
|
|
|
|
notifier.send_notification(
|
|
title="⚠️ Download Queue Stopped",
|
|
message=f"Cookies expired for {platform}. Queue paused - please update cookies and restart.",
|
|
priority=1, # High priority
|
|
sound="siren"
|
|
)
|
|
logger.info(f"Sent cookie expiration notification for {platform}", module="VideoQueue")
|
|
except Exception as e:
|
|
logger.warning(f"Failed to send cookie notification: {e}", module="VideoQueue")
|
|
|
|
|
|
async def run_queue_processor(app_state):
|
|
"""Main queue processing loop."""
|
|
logger.info("Queue processor started", module="VideoQueue")
|
|
|
|
while queue_processor.is_running:
|
|
# Check if paused
|
|
if queue_processor.is_paused:
|
|
await asyncio.sleep(1)
|
|
continue
|
|
|
|
# Get next pending item (ordered by priority, then added_at)
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
SELECT * FROM video_download_queue
|
|
WHERE status = 'pending'
|
|
ORDER BY priority ASC, added_at ASC
|
|
LIMIT 1
|
|
''')
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
# No more pending items
|
|
logger.info("Queue processor: No more pending items", module="VideoQueue")
|
|
queue_processor.stop()
|
|
break
|
|
|
|
item = dict(row)
|
|
|
|
# Process the item
|
|
result = await process_queue_item(item, app_state)
|
|
|
|
# Handle result (now returns dict instead of bool)
|
|
if isinstance(result, dict):
|
|
if result.get('success'):
|
|
queue_processor.increment_processed()
|
|
elif result.get('cookie_error'):
|
|
# Cookie expired - check settings for how to handle
|
|
platform = result.get('platform', 'unknown')
|
|
settings = _get_queue_settings_from_db(app_state)
|
|
|
|
if settings.get('stop_on_cookie_error', True):
|
|
logger.warning(f"Queue: Cookie expired for {platform}, stopping queue", module="VideoQueue")
|
|
|
|
# Stop the processor
|
|
queue_processor.stop()
|
|
|
|
# Send push notification
|
|
await send_cookie_expired_notification(platform, app_state)
|
|
|
|
break
|
|
else:
|
|
# Just log and continue with next item
|
|
logger.warning(f"Queue: Cookie expired for {platform}, continuing (stop_on_cookie_error=False)", module="VideoQueue")
|
|
queue_processor.increment_failed()
|
|
else:
|
|
queue_processor.increment_failed()
|
|
elif result:
|
|
queue_processor.increment_processed()
|
|
else:
|
|
queue_processor.increment_failed()
|
|
|
|
# Get configurable delay from settings (default 15 seconds to avoid rate limiting)
|
|
delay_seconds = 15
|
|
try:
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'video_queue'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
settings = json.loads(row[0])
|
|
delay_seconds = settings.get('download_delay_seconds', 15)
|
|
except Exception:
|
|
pass # Use default on error
|
|
|
|
logger.debug(f"Queue: Waiting {delay_seconds}s before next download", module="VideoQueue")
|
|
await asyncio.sleep(delay_seconds)
|
|
|
|
logger.info(f"Queue processor stopped. Processed: {queue_processor.processed_count}, Failed: {queue_processor.failed_count}",
|
|
module="VideoQueue")
|
|
|
|
|
|
@router.get("/processor/status")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_processor_status(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get the current queue processor status."""
|
|
app_state = get_app_state()
|
|
|
|
# Get pending count
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT COUNT(*) FROM video_download_queue WHERE status = 'pending'")
|
|
pending_count = cursor.fetchone()[0]
|
|
|
|
status = queue_processor.get_status()
|
|
status["pending_count"] = pending_count
|
|
|
|
return {
|
|
"success": True,
|
|
"processor": status
|
|
}
|
|
|
|
|
|
@router.post("/processor/start")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def start_processor(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start processing the download queue."""
|
|
if queue_processor.is_running and not queue_processor.is_paused:
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor is already running"
|
|
}
|
|
|
|
if queue_processor.is_paused:
|
|
# Resume from pause
|
|
queue_processor.resume()
|
|
logger.info("Queue processor resumed", module="VideoQueue")
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor resumed"
|
|
}
|
|
|
|
app_state = get_app_state()
|
|
|
|
# Check if there are pending items
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT COUNT(*) FROM video_download_queue WHERE status = 'pending'")
|
|
pending_count = cursor.fetchone()[0]
|
|
|
|
if pending_count == 0:
|
|
return {
|
|
"success": False,
|
|
"message": "No pending items in queue"
|
|
}
|
|
|
|
# Start the processor
|
|
queue_processor.start()
|
|
queue_processor.reset_counts()
|
|
|
|
# Run in background
|
|
asyncio.create_task(run_queue_processor(app_state))
|
|
|
|
logger.info(f"Queue processor started with {pending_count} pending items", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Queue processor started with {pending_count} pending items"
|
|
}
|
|
|
|
|
|
@router.post("/processor/pause")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def pause_processor(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Pause the queue processor (finishes current download first)."""
|
|
if not queue_processor.is_running:
|
|
return {
|
|
"success": False,
|
|
"message": "Queue processor is not running"
|
|
}
|
|
|
|
if queue_processor.is_paused:
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor is already paused"
|
|
}
|
|
|
|
queue_processor.pause()
|
|
logger.info("Queue processor paused", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor paused (current download will complete)"
|
|
}
|
|
|
|
|
|
@router.post("/processor/stop")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def stop_processor(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Stop the queue processor completely."""
|
|
if not queue_processor.is_running:
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor is not running"
|
|
}
|
|
|
|
queue_processor.stop()
|
|
logger.info("Queue processor stopped", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "Queue processor stopped"
|
|
}
|
|
|
|
|
|
class DownloadSelectedRequest(BaseModel):
|
|
ids: List[int]
|
|
|
|
|
|
# Store selected IDs for the processor to use
|
|
_selected_ids_to_process: List[int] = []
|
|
|
|
|
|
async def run_selected_queue_processor(app_state, selected_ids: List[int]):
|
|
"""Process only selected queue items."""
|
|
logger.info(f"Queue processor started for {len(selected_ids)} selected items", module="VideoQueue")
|
|
|
|
for item_id in selected_ids:
|
|
if not queue_processor.is_running:
|
|
break
|
|
|
|
# Check if paused
|
|
while queue_processor.is_paused and queue_processor.is_running:
|
|
await asyncio.sleep(1)
|
|
|
|
if not queue_processor.is_running:
|
|
break
|
|
|
|
# Get the specific item
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
SELECT * FROM video_download_queue
|
|
WHERE id = ? AND status = 'pending'
|
|
''', (item_id,))
|
|
row = cursor.fetchone()
|
|
|
|
if not row:
|
|
# Item not found or not pending, skip
|
|
continue
|
|
|
|
item = dict(row)
|
|
|
|
# Process the item
|
|
result = await process_queue_item(item, app_state)
|
|
|
|
# Handle result (now returns dict instead of bool)
|
|
if isinstance(result, dict):
|
|
if result.get('success'):
|
|
queue_processor.increment_processed()
|
|
elif result.get('cookie_error'):
|
|
# Cookie expired - check settings for how to handle
|
|
platform = result.get('platform', 'unknown')
|
|
settings = _get_queue_settings_from_db(app_state)
|
|
|
|
if settings.get('stop_on_cookie_error', True):
|
|
logger.warning(f"Queue: Cookie expired for {platform}, stopping queue", module="VideoQueue")
|
|
|
|
# Stop the processor
|
|
queue_processor.stop()
|
|
|
|
# Send push notification
|
|
await send_cookie_expired_notification(platform, app_state)
|
|
|
|
break
|
|
else:
|
|
# Just log and continue with next item
|
|
logger.warning(f"Queue: Cookie expired for {platform}, continuing (stop_on_cookie_error=False)", module="VideoQueue")
|
|
queue_processor.increment_failed()
|
|
else:
|
|
queue_processor.increment_failed()
|
|
elif result:
|
|
queue_processor.increment_processed()
|
|
else:
|
|
queue_processor.increment_failed()
|
|
|
|
# Get configurable delay from settings
|
|
delay_seconds = 15
|
|
try:
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT value FROM settings WHERE key = 'video_queue'")
|
|
row = cursor.fetchone()
|
|
if row:
|
|
settings = json.loads(row[0])
|
|
delay_seconds = settings.get('download_delay_seconds', 15)
|
|
except Exception:
|
|
pass
|
|
|
|
# Only delay if there are more items to process
|
|
remaining = [i for i in selected_ids if i > item_id or selected_ids.index(i) > selected_ids.index(item_id)]
|
|
if remaining and queue_processor.is_running:
|
|
logger.debug(f"Queue: Waiting {delay_seconds}s before next download", module="VideoQueue")
|
|
await asyncio.sleep(delay_seconds)
|
|
|
|
queue_processor.stop()
|
|
logger.info(f"Queue processor stopped. Processed: {queue_processor.processed_count}, Failed: {queue_processor.failed_count}",
|
|
module="VideoQueue")
|
|
|
|
|
|
@router.post("/processor/start-selected")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def start_processor_selected(
|
|
request: Request,
|
|
data: DownloadSelectedRequest,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start processing only selected queue items."""
|
|
if queue_processor.is_running:
|
|
return {
|
|
"success": False,
|
|
"message": "Queue processor is already running. Stop it first to start a new selection."
|
|
}
|
|
|
|
if not data.ids:
|
|
return {
|
|
"success": False,
|
|
"message": "No items selected"
|
|
}
|
|
|
|
app_state = get_app_state()
|
|
|
|
# Verify selected items exist and are pending
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
placeholders = ','.join(['?' for _ in data.ids])
|
|
cursor.execute(f'''
|
|
SELECT COUNT(*) FROM video_download_queue
|
|
WHERE id IN ({placeholders}) AND status = 'pending'
|
|
''', data.ids)
|
|
pending_count = cursor.fetchone()[0]
|
|
|
|
if pending_count == 0:
|
|
return {
|
|
"success": False,
|
|
"message": "No pending items in selection"
|
|
}
|
|
|
|
# Start the processor for selected items
|
|
queue_processor.start()
|
|
queue_processor.reset_counts()
|
|
|
|
# Run in background with selected IDs
|
|
asyncio.create_task(run_selected_queue_processor(app_state, data.ids))
|
|
|
|
logger.info(f"Queue processor started for {pending_count} selected items", module="VideoQueue")
|
|
|
|
return {
|
|
"success": True,
|
|
"message": f"Downloading {pending_count} selected items"
|
|
}
|