Initial commit

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Todd
2026-03-29 22:42:55 -04:00
commit 0d7b2b1aab
389 changed files with 280296 additions and 0 deletions

674
scripts/install.sh Executable file
View File

@@ -0,0 +1,674 @@
#!/bin/bash
# Media Downloader Installer Script
# Version: 13.13.1
# Installs to /opt/media-downloader
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Installation directory
INSTALL_DIR="/opt/media-downloader"
SERVICE_NAME="media-downloader"
CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
echo -e "${GREEN}╔════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Media Downloader Installer v13.13.1 ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════╝${NC}"
echo ""
# Check if running as root
if [[ $EUID -ne 0 ]]; then
echo -e "${RED}This script must be run as root (use sudo)${NC}"
exit 1
fi
# Get the actual user who ran sudo
ACTUAL_USER="${SUDO_USER:-$USER}"
ACTUAL_HOME=$(getent passwd "$ACTUAL_USER" | cut -d: -f6)
echo -e "${YELLOW}Installation Settings:${NC}"
echo " Install directory: $INSTALL_DIR"
echo " Service name: $SERVICE_NAME"
echo " User: $ACTUAL_USER"
echo " Source: $CURRENT_DIR"
echo ""
read -p "Continue with installation? (y/n) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Installation cancelled"
exit 1
fi
# Stop services if they exist
echo -e "${YELLOW}Stopping existing services...${NC}"
systemctl stop $SERVICE_NAME 2>/dev/null || true
systemctl stop media-downloader-api 2>/dev/null || true
systemctl stop media-downloader-frontend 2>/dev/null || true
systemctl stop xvfb-media-downloader 2>/dev/null || true
# Create installation directory
echo -e "${GREEN}Creating installation directory...${NC}"
mkdir -p "$INSTALL_DIR"
# Copy files
echo -e "${GREEN}Copying files...${NC}"
rsync -a --exclude='.git' --exclude='node_modules' --exclude='venv' --exclude='__pycache__' \
--exclude='.playwright' --exclude='dist' --exclude='*.pyc' \
"$CURRENT_DIR/" "$INSTALL_DIR/"
# Create required directories
echo -e "${GREEN}Creating required directories...${NC}"
mkdir -p "$INSTALL_DIR/logs"
mkdir -p "$INSTALL_DIR/database"
mkdir -p "$INSTALL_DIR/cookies"
mkdir -p "$INSTALL_DIR/sessions"
mkdir -p "$INSTALL_DIR/config"
mkdir -p "$INSTALL_DIR/data"
mkdir -p "$INSTALL_DIR/data/face_references" # Face recognition reference images
mkdir -p "$INSTALL_DIR/data/cache/profile_images" # Cached creator avatars/banners
mkdir -p "/opt/immich/review" # Face recognition review queue
mkdir -p "/opt/immich/recycle" # Recycle bin
mkdir -p "/var/log/media-downloader" # System log directory
# Set permissions
echo -e "${GREEN}Setting permissions...${NC}"
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$INSTALL_DIR"
chmod +x "$INSTALL_DIR/media-downloader.py"
chmod +x "$INSTALL_DIR/scripts/"*.sh
chmod +x "$INSTALL_DIR/scripts/"*.py 2>/dev/null || true
# Install system dependencies
echo -e "${GREEN}Installing system dependencies...${NC}"
apt-get update > /dev/null 2>&1
apt-get install -y cmake build-essential libopenblas-dev liblapack-dev \
ffmpeg redis-server xvfb nodejs npm \
libheif-examples imagemagick \
postgresql postgresql-contrib libpq-dev > /dev/null 2>&1
# Start Redis if not running
systemctl enable redis-server
systemctl start redis-server
# Setup PostgreSQL
echo -e "${GREEN}Setting up PostgreSQL...${NC}"
systemctl enable postgresql
systemctl start postgresql
sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname='media_downloader'" | grep -q 1 || \
sudo -u postgres psql -c "CREATE USER media_downloader WITH PASSWORD 'changeme';"
sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname='media_downloader'" | grep -q 1 || \
sudo -u postgres createdb -O media_downloader media_downloader
echo -e "${GREEN}✓ PostgreSQL configured (user: media_downloader, db: media_downloader)${NC}"
echo -e "${YELLOW}⚠ Remember to update DATABASE_URL in .env with the correct password${NC}"
# Create virtual environment
echo -e "${GREEN}Creating Python virtual environment...${NC}"
rm -rf "$INSTALL_DIR/venv" 2>/dev/null || true
python3 -m venv "$INSTALL_DIR/venv"
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$INSTALL_DIR/venv"
# Install Python dependencies
echo -e "${GREEN}Installing Python dependencies from requirements.txt...${NC}"
sudo -u "$ACTUAL_USER" "$INSTALL_DIR/venv/bin/python" -m pip install --upgrade pip
sudo -u "$ACTUAL_USER" "$INSTALL_DIR/venv/bin/python" -m pip install -r "$INSTALL_DIR/requirements.txt"
# Install playwright browsers
echo -e "${GREEN}Installing Playwright browsers...${NC}"
sudo -u "$ACTUAL_USER" bash -c "cd '$INSTALL_DIR' && '$INSTALL_DIR/venv/bin/python' -m playwright install chromium firefox"
# Install frontend dependencies
echo -e "${GREEN}Installing frontend dependencies...${NC}"
cd "$INSTALL_DIR/web/frontend"
sudo -u "$ACTUAL_USER" npm install
sudo -u "$ACTUAL_USER" npm run build
PYTHON_BIN="$INSTALL_DIR/venv/bin/python"
# ============================================================================
# CHECK DEPENDENCIES
# ============================================================================
echo ""
echo -e "${BLUE}Checking Dependencies...${NC}"
# Check for FlareSolverr
if command -v docker &> /dev/null; then
if docker ps | grep -q flaresolverr; then
echo -e "${GREEN}✓ FlareSolverr container is running${NC}"
else
echo -e "${YELLOW}⚠ FlareSolverr container not found${NC}"
read -p "Install FlareSolverr Docker container now? (recommended) (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
docker run -d \
--name flaresolverr \
-p 8191:8191 \
-e LOG_LEVEL=info \
--restart unless-stopped \
ghcr.io/flaresolverr/flaresolverr:latest
echo -e "${GREEN}✓ FlareSolverr installed on port 8191${NC}"
fi
fi
else
echo -e "${YELLOW}⚠ Docker not found. FlareSolverr requires Docker for Cloudflare bypass.${NC}"
fi
# ============================================================================
# CREATE SYSTEMD SERVICES
# ============================================================================
echo ""
echo -e "${GREEN}Creating systemd services...${NC}"
# 1. Xvfb Virtual Display Service
cat > "/etc/systemd/system/xvfb-media-downloader.service" << EOF
[Unit]
Description=Xvfb Virtual Display for Media Downloader
After=network.target
[Service]
Type=simple
User=root
ExecStart=/usr/bin/Xvfb :100 -screen 0 1920x1080x24 -nolisten tcp
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
# 2. Main Scheduler Service
cat > "/etc/systemd/system/$SERVICE_NAME.service" << EOF
[Unit]
Description=Media Downloader Scheduler Service
After=network.target xvfb-media-downloader.service redis-server.service
Wants=xvfb-media-downloader.service
[Service]
Type=simple
User=$ACTUAL_USER
Group=$ACTUAL_USER
WorkingDirectory=$INSTALL_DIR
ExecStart=$PYTHON_BIN $INSTALL_DIR/media-downloader.py --scheduler
Restart=on-failure
RestartSec=30
StandardOutput=append:$INSTALL_DIR/logs/service.log
StandardError=append:$INSTALL_DIR/logs/service.log
Environment="PYTHONUNBUFFERED=1"
Environment="PYTHONDONTWRITEBYTECODE=1"
Environment="DISPLAY=:100"
LimitNOFILE=65536
Nice=10
[Install]
WantedBy=multi-user.target
EOF
# 3. Web API Service
cat > "/etc/systemd/system/media-downloader-api.service" << EOF
[Unit]
Description=Media Downloader Web API
After=network.target redis-server.service
Wants=redis-server.service
[Service]
Type=simple
User=root
Group=root
WorkingDirectory=$INSTALL_DIR/web/backend
ExecStart=$PYTHON_BIN $INSTALL_DIR/web/backend/api.py
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
Environment="PYTHONUNBUFFERED=1"
Environment="PYTHONDONTWRITEBYTECODE=1"
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 4. Web Frontend Service (Production - serves pre-built static files)
# Note: For development with hot-reload, run: cd web/frontend && npm run dev
cat > "/etc/systemd/system/media-downloader-frontend.service" << EOF
[Unit]
Description=Media Downloader Web Frontend (Production)
After=network.target media-downloader-api.service
Wants=media-downloader-api.service
[Service]
Type=simple
User=root
Group=root
WorkingDirectory=$INSTALL_DIR/web/frontend
ExecStart=/usr/bin/npm run preview -- --host 0.0.0.0 --port 5173
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
Environment="NODE_ENV=production"
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 4b. Development Frontend Service (optional - hot reload)
cat > "/etc/systemd/system/media-downloader-frontend-dev.service" << EOF
[Unit]
Description=Media Downloader Web Frontend (Development - Hot Reload)
After=network.target media-downloader-api.service
Wants=media-downloader-api.service
[Service]
Type=simple
User=$ACTUAL_USER
Group=$ACTUAL_USER
WorkingDirectory=$INSTALL_DIR/web/frontend
ExecStart=/usr/bin/npm run dev -- --host 0.0.0.0 --port 5173
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
Environment="NODE_ENV=development"
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 5. Thumbnail Cache Builder Service + Timer
cat > "/etc/systemd/system/media-cache-builder.service" << EOF
[Unit]
Description=Media Thumbnail and Metadata Cache Builder
After=network.target
[Service]
Type=oneshot
User=root
WorkingDirectory=$INSTALL_DIR
ExecStart=$PYTHON_BIN $INSTALL_DIR/modules/thumbnail_cache_builder.py
StandardOutput=journal
StandardError=journal
TimeoutStartSec=3600
Nice=19
IOSchedulingClass=idle
CPUQuota=50%
[Install]
WantedBy=multi-user.target
EOF
cat > "/etc/systemd/system/media-cache-builder.timer" << EOF
[Unit]
Description=Daily Media Cache Builder Timer
Requires=media-cache-builder.service
[Timer]
OnCalendar=*-*-* 03:00:00
Persistent=true
RandomizedDelaySec=30min
[Install]
WantedBy=timers.target
EOF
# 6. Embedding Generator Service + Timer
cat > "/etc/systemd/system/media-embedding-generator.service" << EOF
[Unit]
Description=Media Downloader Embedding Generator (CLIP)
After=network.target
[Service]
Type=oneshot
User=root
WorkingDirectory=$INSTALL_DIR
ExecStart=$PYTHON_BIN $INSTALL_DIR/scripts/generate-embeddings.py
StandardOutput=journal
StandardError=journal
TimeoutStartSec=3600
[Install]
WantedBy=multi-user.target
EOF
cat > "/etc/systemd/system/media-embedding-generator.timer" << EOF
[Unit]
Description=Nightly Media Embedding Generation Timer
Requires=media-embedding-generator.service
[Timer]
OnCalendar=*-*-* 03:00:00
RandomizedDelaySec=1800
Persistent=true
[Install]
WantedBy=timers.target
EOF
# 7. Celebrity Enrichment Service + Timer
cat > "/etc/systemd/system/media-celebrity-enrichment.service" << EOF
[Unit]
Description=Media Downloader Celebrity Metadata Enrichment
After=network.target
[Service]
Type=oneshot
User=root
WorkingDirectory=$INSTALL_DIR
ExecStart=$PYTHON_BIN $INSTALL_DIR/scripts/enrich_celebrity_metadata.py
StandardOutput=journal
StandardError=journal
TimeoutStartSec=3600
[Install]
WantedBy=multi-user.target
EOF
cat > "/etc/systemd/system/media-celebrity-enrichment.timer" << EOF
[Unit]
Description=Nightly Celebrity Metadata Enrichment Timer
Requires=media-celebrity-enrichment.service
[Timer]
OnCalendar=*-*-* 04:00:00
RandomizedDelaySec=300
Persistent=true
[Install]
WantedBy=timers.target
EOF
# 8. Plex Matching Service + Timer
cat > "/etc/systemd/system/plex-match.service" << EOF
[Unit]
Description=Match appearances to Plex library
After=media-downloader-api.service
Requires=media-downloader-api.service
[Service]
Type=oneshot
ExecStart=$INSTALL_DIR/scripts/plex-match.sh
User=root
EOF
cat > "/etc/systemd/system/plex-match.timer" << EOF
[Unit]
Description=Run Plex matching twice daily
[Timer]
OnCalendar=*-*-* 06:00:00
OnCalendar=*-*-* 18:00:00
Persistent=true
[Install]
WantedBy=timers.target
EOF
# 9. Database Cleanup Service + Timer
cat > "/etc/systemd/system/media-downloader-db-cleanup.service" << EOF
[Unit]
Description=Media Downloader Database Cleanup
After=network.target media-downloader-api.service
[Service]
Type=oneshot
User=root
WorkingDirectory=$INSTALL_DIR
ExecStart=$INSTALL_DIR/scripts/db-cleanup.sh
StandardOutput=append:$INSTALL_DIR/logs/db-cleanup.log
StandardError=append:$INSTALL_DIR/logs/db-cleanup.log
MemoryMax=512M
CPUQuota=50%
Restart=no
[Install]
WantedBy=multi-user.target
EOF
cat > "/etc/systemd/system/media-downloader-db-cleanup.timer" << EOF
[Unit]
Description=Media Downloader Database Cleanup Timer
Requires=media-downloader-db-cleanup.service
[Timer]
OnCalendar=*-*-* 03:00:00
Persistent=true
OnBootSec=5min
RandomizedDelaySec=10min
[Install]
WantedBy=timers.target
EOF
# Create command-line wrapper
echo -e "${GREEN}Creating command-line wrapper...${NC}"
cat > "/usr/local/bin/media-downloader" << EOF
#!/bin/bash
cd $INSTALL_DIR
export DISPLAY=:100
$PYTHON_BIN $INSTALL_DIR/media-downloader.py "\$@"
EOF
chmod +x "/usr/local/bin/media-downloader"
# Copy config if it doesn't exist
if [ ! -f "$INSTALL_DIR/config/settings.json" ]; then
echo -e "${GREEN}Copying default configuration...${NC}"
if [ -f "$INSTALL_DIR/config/settings.example.json" ]; then
cp "$INSTALL_DIR/config/settings.example.json" "$INSTALL_DIR/config/settings.json"
fi
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$INSTALL_DIR/config"
chmod 600 "$INSTALL_DIR/config/settings.json" 2>/dev/null || true
fi
# Reload systemd
echo -e "${GREEN}Reloading systemd...${NC}"
systemctl daemon-reload
# ============================================================================
# ENABLE AND START SERVICES
# ============================================================================
echo ""
echo -e "${BLUE}Service Configuration${NC}"
read -p "Enable and start all services? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo -e "${GREEN}Enabling services...${NC}"
# Core services
systemctl enable xvfb-media-downloader.service
systemctl enable $SERVICE_NAME.service
systemctl enable media-downloader-api.service
# Timers
systemctl enable media-cache-builder.timer
systemctl enable media-embedding-generator.timer
systemctl enable media-celebrity-enrichment.timer
systemctl enable plex-match.timer
systemctl enable media-downloader-db-cleanup.timer
echo -e "${GREEN}Starting services...${NC}"
# Start in order
systemctl start xvfb-media-downloader.service
sleep 2
systemctl start media-downloader-api.service
sleep 2
systemctl start $SERVICE_NAME.service
# Start timers
systemctl start media-cache-builder.timer
systemctl start media-embedding-generator.timer
systemctl start media-celebrity-enrichment.timer
systemctl start plex-match.timer
systemctl start media-downloader-db-cleanup.timer
echo -e "${GREEN}✓ All services started${NC}"
fi
# ============================================================================
# OPTIONAL: NGINX REVERSE PROXY
# ============================================================================
echo ""
if command -v nginx &> /dev/null; then
read -p "Configure nginx reverse proxy? (recommended for production) (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo -e "${GREEN}Creating nginx configuration...${NC}"
cat > "/etc/nginx/sites-available/media-downloader" << 'NGINX_EOF'
# Media Downloader Nginx Configuration
# Reverse proxy for API (8000) and Frontend (5173)
server {
listen 80;
server_name _; # Change to your domain
# Frontend (Vite)
location / {
proxy_pass http://127.0.0.1:5173;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_bypass $http_upgrade;
}
# API Backend
location /api/ {
proxy_pass http://127.0.0.1:8000/api/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket support
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Timeouts for long-running requests
proxy_connect_timeout 60s;
proxy_send_timeout 300s;
proxy_read_timeout 300s;
}
# WebSocket endpoint
location /ws {
proxy_pass http://127.0.0.1:8000/ws;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_read_timeout 86400;
}
# Media files (if serving directly)
location /media/ {
alias /opt/immich/media/;
autoindex off;
}
# Thumbnails
location /thumbnails/ {
proxy_pass http://127.0.0.1:8000/api/thumbnails/;
proxy_cache_valid 200 1d;
}
# Increase max upload size for imports
client_max_body_size 500M;
}
NGINX_EOF
# Enable site
ln -sf /etc/nginx/sites-available/media-downloader /etc/nginx/sites-enabled/ 2>/dev/null || true
# Test and reload nginx
if nginx -t 2>/dev/null; then
systemctl reload nginx
echo -e "${GREEN}✓ Nginx configured and reloaded${NC}"
else
echo -e "${YELLOW}⚠ Nginx config has errors - please check manually${NC}"
fi
fi
else
echo -e "${YELLOW}Note: nginx not installed. For production, consider:${NC}"
echo " sudo apt install nginx"
echo " Then re-run installer or manually configure reverse proxy"
fi
# ============================================================================
# COMPLETION MESSAGE
# ============================================================================
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Complete! ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "${BLUE}Installation location:${NC} $INSTALL_DIR"
echo -e "${BLUE}Configuration file:${NC} $INSTALL_DIR/config/settings.json"
echo -e "${BLUE}Database directory:${NC} $INSTALL_DIR/database"
echo -e "${BLUE}Logs directory:${NC} $INSTALL_DIR/logs"
echo ""
echo -e "${YELLOW}Services:${NC}"
echo " media-downloader - Main scheduler service"
echo " media-downloader-api - Web API (port 8000)"
echo " media-downloader-frontend - Web UI production (port 5173)"
echo " media-downloader-frontend-dev - Web UI development with hot-reload"
echo " xvfb-media-downloader - Virtual display for browser automation"
echo ""
echo -e "${YELLOW}Scheduled Tasks (timers):${NC}"
echo " media-cache-builder - Thumbnail cache (daily 3 AM)"
echo " media-embedding-generator - CLIP embeddings (daily 3 AM)"
echo " media-downloader-db-cleanup- Database cleanup (daily 3 AM)"
echo " media-celebrity-enrichment - Celebrity metadata (daily 4 AM)"
echo " plex-match - Plex library matching (6 AM, 6 PM)"
echo ""
echo -e "${YELLOW}Commands:${NC}"
echo " media-downloader - Run manual download"
echo " media-downloader --scheduler - Run scheduler"
echo " media-downloader --scheduler-status - Check scheduler status"
echo " media-downloader --platform instagram - Download specific platform"
echo ""
echo -e "${YELLOW}Service Management:${NC}"
echo " sudo systemctl status media-downloader - Check status"
echo " sudo systemctl restart media-downloader - Restart scheduler"
echo " sudo systemctl restart media-downloader-api- Restart API"
echo " sudo journalctl -u media-downloader -f - View logs"
echo ""
echo -e "${YELLOW}Web Interface:${NC}"
echo " API: http://localhost:8000"
echo " Frontend: http://localhost:5173"
echo ""
echo -e "${YELLOW}Development Mode:${NC}"
echo " # Switch to development frontend (with hot-reload):"
echo " sudo systemctl stop media-downloader-frontend"
echo " sudo systemctl start media-downloader-frontend-dev"
echo ""
echo -e "${YELLOW}To uninstall:${NC}"
echo " sudo $INSTALL_DIR/scripts/uninstall.sh"