6257 lines
216 KiB
Python
6257 lines
216 KiB
Python
"""
|
|
Paid Content Router
|
|
|
|
API endpoints for the Paid Content feature:
|
|
- Dashboard stats and failed downloads
|
|
- Services management (Coomer/Kemono)
|
|
- Creators CRUD and sync
|
|
- Identities (creator linking)
|
|
- Feed browsing with filters
|
|
- Notifications
|
|
- Settings
|
|
- Import from file hosts
|
|
- Recycle bin
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
from datetime import datetime
|
|
from threading import Lock
|
|
from typing import Dict, List, Optional
|
|
|
|
from pathlib import Path
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, Query, Request, Response, UploadFile, File, Form
|
|
from fastapi.responses import FileResponse
|
|
from pydantic import BaseModel, ConfigDict
|
|
from slowapi import Limiter
|
|
from slowapi.util import get_remote_address
|
|
|
|
from ..core.dependencies import get_current_user, get_app_state
|
|
from ..core.exceptions import handle_exceptions, NotFoundError, ValidationError
|
|
from ..core.responses import message_response, now_iso8601
|
|
from modules.universal_logger import get_logger
|
|
|
|
logger = get_logger('API')
|
|
|
|
router = APIRouter(prefix="/api/paid-content", tags=["Paid Content"])
|
|
limiter = Limiter(key_func=get_remote_address)
|
|
|
|
# Auto health check throttle - prevents repeated checks within cooldown period
|
|
_auto_health_check_running = False
|
|
_last_auto_health_check: Optional[float] = None
|
|
_AUTO_HEALTH_CHECK_INTERVAL = 300 # 5 minutes
|
|
|
|
# Face scan job tracking (transient, same pattern as manual_import.py)
|
|
_face_scan_status: Dict[str, Dict] = {}
|
|
_face_scan_lock = Lock()
|
|
|
|
|
|
# ============================================================================
|
|
# PYDANTIC MODELS
|
|
# ============================================================================
|
|
|
|
class AddCreatorRequest(BaseModel):
|
|
service_id: str # 'coomer' or 'kemono'
|
|
platform: str # 'onlyfans', 'patreon', etc.
|
|
creator_id: str # Platform-specific ID
|
|
auto_download: bool = True
|
|
download_embeds: bool = True
|
|
|
|
|
|
class UpdateCreatorRequest(BaseModel):
|
|
enabled: Optional[bool] = None
|
|
auto_download: Optional[bool] = None
|
|
download_embeds: Optional[bool] = None
|
|
identity_id: Optional[int] = None
|
|
sync_posts: Optional[bool] = None
|
|
sync_stories: Optional[bool] = None
|
|
sync_highlights: Optional[bool] = None
|
|
use_authenticated_api: Optional[bool] = None
|
|
filter_tagged_users: Optional[List[str]] = None
|
|
|
|
|
|
class UpdateSessionRequest(BaseModel):
|
|
session_cookie: str
|
|
|
|
|
|
class CreateIdentityRequest(BaseModel):
|
|
name: str
|
|
notes: Optional[str] = None
|
|
|
|
|
|
class UpdateIdentityRequest(BaseModel):
|
|
name: Optional[str] = None
|
|
notes: Optional[str] = None
|
|
profile_image_url: Optional[str] = None
|
|
|
|
|
|
class LinkCreatorRequest(BaseModel):
|
|
creator_id: int
|
|
|
|
|
|
class ImportUrlRequest(BaseModel):
|
|
url: str
|
|
creator_id: Optional[int] = None
|
|
|
|
|
|
class UpdatePostRequest(BaseModel):
|
|
title: Optional[str] = None
|
|
content: Optional[str] = None
|
|
published_at: Optional[str] = None # Date in ISO format or YYYY-MM-DD
|
|
|
|
|
|
class RetryDownloadsRequest(BaseModel):
|
|
attachment_ids: List[int]
|
|
|
|
|
|
class RemoveFromQueueRequest(BaseModel):
|
|
attachment_ids: List[int]
|
|
|
|
|
|
class UpdateQueueItemRequest(BaseModel):
|
|
auto_requeue: Optional[bool] = None
|
|
|
|
|
|
class MarkNotificationsReadRequest(BaseModel):
|
|
notification_ids: Optional[List[int]] = None
|
|
mark_all: bool = False
|
|
|
|
|
|
class ImportToAttachmentRequest(BaseModel):
|
|
"""Request to import a file from URL and link it to an existing attachment"""
|
|
url: str
|
|
|
|
|
|
class ParseFilenamesRequest(BaseModel):
|
|
"""Request to parse filenames and extract dates"""
|
|
filenames: List[str]
|
|
|
|
|
|
class CreateTagRequest(BaseModel):
|
|
"""Request to create a new tag"""
|
|
name: str
|
|
color: Optional[str] = "#6b7280"
|
|
description: Optional[str] = None
|
|
|
|
|
|
class UpdateTagRequest(BaseModel):
|
|
"""Request to update a tag"""
|
|
name: Optional[str] = None
|
|
color: Optional[str] = None
|
|
description: Optional[str] = None
|
|
|
|
|
|
class TagPostsRequest(BaseModel):
|
|
"""Request to add/remove tags from posts"""
|
|
post_ids: List[int]
|
|
tag_ids: List[int]
|
|
|
|
|
|
class SetPostTagsRequest(BaseModel):
|
|
"""Request to set tags for a single post"""
|
|
tag_ids: List[int]
|
|
|
|
|
|
class CreateAutoTagRuleRequest(BaseModel):
|
|
"""Request to create an auto-tag rule"""
|
|
name: str
|
|
conditions: Dict
|
|
tag_ids: List[int]
|
|
priority: int = 0
|
|
|
|
class UpdateAutoTagRuleRequest(BaseModel):
|
|
"""Request to update an auto-tag rule"""
|
|
name: Optional[str] = None
|
|
enabled: Optional[int] = None
|
|
conditions: Optional[Dict] = None
|
|
tag_ids: Optional[List[int]] = None
|
|
priority: Optional[int] = None
|
|
|
|
class BulkDeletePostsRequest(BaseModel):
|
|
"""Request to bulk soft-delete posts"""
|
|
post_ids: List[int]
|
|
|
|
|
|
class WatchLaterAddRequest(BaseModel):
|
|
"""Request to add attachment to watch later"""
|
|
attachment_id: int
|
|
|
|
class WatchLaterBulkAddRequest(BaseModel):
|
|
"""Request to add multiple attachments to watch later"""
|
|
attachment_ids: List[int]
|
|
|
|
class WatchLaterRemoveRequest(BaseModel):
|
|
"""Request to remove multiple attachments from watch later"""
|
|
attachment_ids: List[int]
|
|
|
|
class WatchLaterReorderRequest(BaseModel):
|
|
"""Request to reorder watch later items"""
|
|
ordered_ids: List[int]
|
|
|
|
class FaceAddReferenceRequest(BaseModel):
|
|
person_name: str
|
|
file_path: str # local_path of an existing attachment
|
|
|
|
|
|
class FaceScanCreatorRequest(BaseModel):
|
|
creator_id: int
|
|
person_name: str # e.g. "India Reynolds"
|
|
tolerance: float = 0.35
|
|
|
|
|
|
class PaidContentSettings(BaseModel):
|
|
"""Settings model that ignores extra fields like id, created_at, updated_at from the database"""
|
|
model_config = ConfigDict(extra='ignore')
|
|
|
|
base_download_path: Optional[str] = None
|
|
organize_by_date: Optional[bool] = None
|
|
organize_by_post: Optional[bool] = None
|
|
check_interval_hours: Optional[int] = None
|
|
max_concurrent_downloads: Optional[int] = None
|
|
download_embeds: Optional[bool] = None
|
|
embed_quality: Optional[str] = None
|
|
notifications_enabled: Optional[bool] = None
|
|
push_notifications_enabled: Optional[bool] = None
|
|
perceptual_duplicate_detection: Optional[bool] = None
|
|
perceptual_threshold: Optional[int] = None
|
|
auto_retry_failed: Optional[bool] = None
|
|
retry_max_attempts: Optional[int] = None
|
|
|
|
|
|
# ============================================================================
|
|
# HELPER FUNCTIONS
|
|
# ============================================================================
|
|
|
|
def _get_db_adapter():
|
|
"""Get PaidContentDBAdapter instance"""
|
|
from modules.paid_content import PaidContentDBAdapter
|
|
app_state = get_app_state()
|
|
return PaidContentDBAdapter(app_state.db)
|
|
|
|
|
|
def _get_scraper():
|
|
"""Get PaidContentScraper instance"""
|
|
from modules.paid_content import PaidContentScraper
|
|
app_state = get_app_state()
|
|
return PaidContentScraper(
|
|
unified_db=app_state.db,
|
|
notifier=getattr(app_state, 'notifier', None),
|
|
websocket_manager=getattr(app_state, 'websocket_manager', None),
|
|
app_state=app_state
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# DASHBOARD ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/dashboard/stats")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_dashboard_stats(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get dashboard statistics"""
|
|
db = _get_db_adapter()
|
|
stats = db.get_dashboard_stats()
|
|
return {"stats": stats, "timestamp": now_iso8601()}
|
|
|
|
|
|
@router.get("/dashboard/active-syncs")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_active_syncs(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get currently active sync tasks (for polling-based real-time updates).
|
|
|
|
Uses activity_manager for database-backed tracking that works across processes
|
|
(scheduler, API, etc.)
|
|
"""
|
|
from modules.activity_status import get_activity_manager
|
|
app_state = get_app_state()
|
|
|
|
# Get active syncs from activity_manager (database-backed, works across processes)
|
|
activity_manager = get_activity_manager(app_state.db if app_state else None)
|
|
|
|
# Get all active background tasks of type 'paid_content_sync'
|
|
all_tasks = activity_manager.get_active_background_tasks()
|
|
paid_content_syncs = [
|
|
task for task in all_tasks
|
|
if task.get('task_type') == 'paid_content_sync'
|
|
]
|
|
|
|
# Convert to the format expected by the frontend
|
|
syncs = []
|
|
for task in paid_content_syncs:
|
|
extra_data = task.get('extra_data', {}) or {}
|
|
sync_data = {
|
|
'creator_id': extra_data.get('creator_id'),
|
|
'creator': extra_data.get('creator'),
|
|
'platform': extra_data.get('platform'),
|
|
'phase': extra_data.get('phase', 'syncing'),
|
|
'status': task.get('detailed_status') or task.get('status', 'Running'),
|
|
'started_at': task.get('start_time'),
|
|
'updated_at': task.get('updated_at'),
|
|
**extra_data
|
|
}
|
|
if task.get('progress'):
|
|
sync_data['progress'] = task['progress'].get('current')
|
|
sync_data['total_files'] = task['progress'].get('total')
|
|
syncs.append(sync_data)
|
|
|
|
return {
|
|
"syncs": syncs,
|
|
"count": len(syncs)
|
|
}
|
|
|
|
|
|
@router.get("/dashboard/failed-downloads")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_failed_downloads(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get failed downloads for retry"""
|
|
db = _get_db_adapter()
|
|
failed = db.get_failed_downloads()
|
|
return {"downloads": failed, "count": len(failed)}
|
|
|
|
|
|
@router.post("/dashboard/retry-failed")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def retry_failed_downloads(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
body: RetryDownloadsRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Retry failed downloads"""
|
|
background_tasks.add_task(_retry_downloads_background, body.attachment_ids)
|
|
return {"status": "queued", "count": len(body.attachment_ids)}
|
|
|
|
|
|
@router.post("/dashboard/scan-missing")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def scan_missing_files(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Scan all completed attachments and check if files exist on disk.
|
|
Updates status to 'missing' for files that don't exist.
|
|
"""
|
|
db = _get_db_adapter()
|
|
result = db.scan_missing_files()
|
|
return {
|
|
"status": "completed",
|
|
"scanned": result['scanned'],
|
|
"missing": result['missing'],
|
|
"found": result['found'],
|
|
"missing_files": result['missing_files'][:50] # Limit response size
|
|
}
|
|
|
|
|
|
@router.get("/dashboard/missing-files")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_missing_files(
|
|
request: Request,
|
|
limit: int = Query(default=100, le=500),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get list of attachments with missing files"""
|
|
db = _get_db_adapter()
|
|
missing = db.get_missing_attachments(limit=limit)
|
|
return {"files": missing, "count": len(missing)}
|
|
|
|
|
|
class ResetMissingRequest(BaseModel):
|
|
attachment_ids: Optional[List[int]] = None
|
|
reset_all: bool = False
|
|
|
|
|
|
@router.post("/dashboard/reset-missing")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def reset_missing_files(
|
|
request: Request,
|
|
body: ResetMissingRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Reset missing attachments to pending for re-download"""
|
|
db = _get_db_adapter()
|
|
|
|
if body.reset_all:
|
|
count = db.reset_missing_to_pending()
|
|
elif body.attachment_ids:
|
|
count = db.reset_missing_to_pending(body.attachment_ids)
|
|
else:
|
|
return {"status": "error", "message": "Provide attachment_ids or set reset_all=true"}
|
|
|
|
return {"status": "success", "reset_count": count}
|
|
|
|
|
|
class QualityRecheckRequest(BaseModel):
|
|
attachment_ids: List[int]
|
|
|
|
|
|
@router.post("/attachments/recheck-quality")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def recheck_attachment_quality(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
body: QualityRecheckRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Re-check Fansly attachments for higher quality variants"""
|
|
background_tasks.add_task(_recheck_quality_background, body.attachment_ids)
|
|
return {"status": "queued", "count": len(body.attachment_ids)}
|
|
|
|
|
|
async def _recheck_quality_background(attachment_ids: List[int]):
|
|
"""Background task to recheck quality for specified attachments"""
|
|
from modules.paid_content.fansly_direct_client import FanslyDirectClient
|
|
|
|
db = _get_db_adapter()
|
|
|
|
# Get Fansly auth token
|
|
service = db.get_service('fansly_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
logger.error("No Fansly auth token configured for quality recheck", module="PaidContent")
|
|
return
|
|
|
|
auth_token = service['session_cookie']
|
|
results = []
|
|
upgraded_creator_ids = set()
|
|
|
|
async with FanslyDirectClient(auth_token) as client:
|
|
for att_id in attachment_ids:
|
|
try:
|
|
result = await client.recheck_attachment_quality(att_id, db)
|
|
results.append({'attachment_id': att_id, **result})
|
|
if result.get('upgraded') and result.get('creator_id'):
|
|
upgraded_creator_ids.add(result['creator_id'])
|
|
except Exception as e:
|
|
logger.error(f"Quality recheck failed for attachment {att_id}: {e}", module="PaidContent")
|
|
results.append({'attachment_id': att_id, 'upgraded': False, 'error': str(e)})
|
|
|
|
# Delete upgraded posts and trigger re-sync to get fresh 4K versions
|
|
if upgraded_creator_ids:
|
|
for att_id in attachment_ids:
|
|
att = db.get_attachment(att_id)
|
|
if not att or not att.get('post_id'):
|
|
continue
|
|
post = db.get_post(att['post_id'])
|
|
if not post or post.get('creator_id') not in upgraded_creator_ids:
|
|
continue
|
|
# Only delete if this attachment was actually upgraded
|
|
result_for_att = next((r for r in results if r.get('attachment_id') == att_id and r.get('upgraded')), None)
|
|
if not result_for_att:
|
|
continue
|
|
# Delete the old file
|
|
local_path = att.get('local_path')
|
|
if local_path:
|
|
import os
|
|
try:
|
|
if os.path.exists(local_path):
|
|
os.unlink(local_path)
|
|
logger.info(f"Deleted old file for quality upgrade: {local_path}", module="PaidContent")
|
|
except Exception as e:
|
|
logger.warning(f"Failed to delete old file {local_path}: {e}", module="PaidContent")
|
|
# Delete attachment and post from DB
|
|
post_id = att['post_id']
|
|
with db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM paid_content_attachments WHERE id = ?", (att_id,))
|
|
remaining = cursor.execute("SELECT COUNT(*) FROM paid_content_attachments WHERE post_id = ?", (post_id,)).fetchone()[0]
|
|
if remaining == 0:
|
|
cursor.execute("DELETE FROM paid_content_posts WHERE id = ?", (post_id,))
|
|
conn.commit()
|
|
logger.info(f"Deleted attachment {att_id} and post {post_id} for quality upgrade re-sync", module="PaidContent")
|
|
|
|
# Trigger a full sync for each creator to re-discover posts at 4K
|
|
for creator_id in upgraded_creator_ids:
|
|
try:
|
|
scraper = _get_scraper()
|
|
await scraper.sync_creator(creator_id, download=True)
|
|
await scraper.close()
|
|
logger.info(f"Quality upgrade re-sync completed for creator {creator_id}", module="PaidContent")
|
|
except Exception as e:
|
|
logger.error(f"Quality upgrade re-sync failed for creator {creator_id}: {e}", module="PaidContent")
|
|
|
|
# Broadcast WebSocket event
|
|
try:
|
|
app_state = get_app_state()
|
|
ws = getattr(app_state, 'websocket_manager', None)
|
|
if ws:
|
|
ws.broadcast_sync({
|
|
'type': 'quality_recheck_complete',
|
|
'results': results,
|
|
'upgraded_count': sum(1 for r in results if r.get('upgraded')),
|
|
'total_checked': len(results),
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"WebSocket broadcast failed for quality recheck: {e}", module="PaidContent")
|
|
|
|
|
|
async def _auto_quality_recheck_background(unified_db=None):
|
|
"""Automatically recheck quality for flagged attachments after Fansly sync"""
|
|
from modules.paid_content.fansly_direct_client import FanslyDirectClient
|
|
from modules.paid_content import PaidContentDBAdapter
|
|
|
|
if unified_db:
|
|
db = PaidContentDBAdapter(unified_db)
|
|
else:
|
|
db = _get_db_adapter()
|
|
candidates = db.get_quality_recheck_candidates(max_attempts=24)
|
|
|
|
if not candidates:
|
|
return
|
|
|
|
logger.info(f"Auto quality recheck: {len(candidates)} candidates found", module="PaidContent")
|
|
|
|
# Get Fansly auth token
|
|
service = db.get_service('fansly_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
return
|
|
|
|
auth_token = service['session_cookie']
|
|
upgraded_creator_ids = set()
|
|
upgraded_attachments = []
|
|
|
|
async with FanslyDirectClient(auth_token) as client:
|
|
for candidate in candidates:
|
|
try:
|
|
result = await client.recheck_attachment_quality(candidate['id'], db)
|
|
if result.get('upgraded') and result.get('creator_id'):
|
|
upgraded_creator_ids.add(result['creator_id'])
|
|
upgraded_attachments.append(candidate['id'])
|
|
logger.info(
|
|
f"Auto recheck upgraded attachment {candidate['id']}: "
|
|
f"{result.get('old_width')}x{result.get('old_height')} -> "
|
|
f"{result.get('new_width')}x{result.get('new_height')}",
|
|
module="PaidContent"
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Auto quality recheck failed for {candidate['id']}: {e}", module="PaidContent")
|
|
|
|
# Delete upgraded posts and trigger re-sync to get fresh 4K versions
|
|
if upgraded_creator_ids:
|
|
import os
|
|
for att_id in upgraded_attachments:
|
|
att = db.get_attachment(att_id)
|
|
if not att or not att.get('post_id'):
|
|
continue
|
|
# Delete the old file
|
|
local_path = att.get('local_path')
|
|
if local_path:
|
|
try:
|
|
if os.path.exists(local_path):
|
|
os.unlink(local_path)
|
|
logger.info(f"Deleted old file for quality upgrade: {local_path}", module="PaidContent")
|
|
except Exception as e:
|
|
logger.warning(f"Failed to delete old file {local_path}: {e}", module="PaidContent")
|
|
# Delete attachment and post from DB
|
|
post_id = att['post_id']
|
|
with db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM paid_content_attachments WHERE id = ?", (att_id,))
|
|
remaining = cursor.execute("SELECT COUNT(*) FROM paid_content_attachments WHERE post_id = ?", (post_id,)).fetchone()[0]
|
|
if remaining == 0:
|
|
cursor.execute("DELETE FROM paid_content_posts WHERE id = ?", (post_id,))
|
|
conn.commit()
|
|
logger.info(f"Deleted attachment {att_id} and post {post_id} for quality upgrade re-sync", module="PaidContent")
|
|
|
|
# Trigger a full sync for each creator to re-discover posts at 4K
|
|
for creator_id in upgraded_creator_ids:
|
|
try:
|
|
scraper = _get_scraper()
|
|
await scraper.sync_creator(creator_id, download=True)
|
|
await scraper.close()
|
|
logger.info(f"Quality upgrade re-sync completed for creator {creator_id}", module="PaidContent")
|
|
except Exception as e:
|
|
logger.error(f"Quality upgrade re-sync failed for creator {creator_id}: {e}", module="PaidContent")
|
|
|
|
|
|
# ============================================================================
|
|
# DOWNLOAD QUEUE ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/queue")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_download_queue(
|
|
request: Request,
|
|
status: Optional[str] = Query(None, description="Filter by status: pending, downloading, failed"),
|
|
creator_id: Optional[int] = None,
|
|
limit: int = Query(100, description="Max items per status category (0 = all)"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get download queue with pending, downloading, and failed items"""
|
|
from modules.activity_status import get_activity_manager
|
|
|
|
app_state = get_app_state()
|
|
db = _get_db_adapter()
|
|
activity_manager = get_activity_manager(app_state.db if app_state else None)
|
|
|
|
# Auto-cleanup: if no active syncs, reset any "downloading" items to "pending"
|
|
# This handles cases where the server was restarted mid-download
|
|
# Check both activity_manager (works across processes) and app_state (local)
|
|
all_tasks = activity_manager.get_active_background_tasks()
|
|
has_active_syncs = any(t.get('task_type') == 'paid_content_sync' for t in all_tasks)
|
|
if not has_active_syncs and hasattr(app_state, 'active_paid_content_syncs'):
|
|
has_active_syncs = len(app_state.active_paid_content_syncs) > 0
|
|
|
|
if not has_active_syncs:
|
|
reset_count = db.reset_downloading_to_pending()
|
|
if reset_count > 0:
|
|
logger.info(f"Auto-reset {reset_count} stale downloading items to pending")
|
|
|
|
item_limit = limit if limit > 0 else None
|
|
|
|
# Get counts efficiently (always), items only up to limit
|
|
result = {
|
|
'pending': [],
|
|
'downloading': [],
|
|
'failed': [],
|
|
'counts': {'pending': 0, 'downloading': 0, 'failed': 0}
|
|
}
|
|
|
|
if status is None or status == 'pending':
|
|
result['counts']['pending'] = db.get_pending_attachment_count(creator_id=creator_id)
|
|
result['pending'] = db.get_pending_attachments(creator_id=creator_id, limit=item_limit)
|
|
|
|
if status is None or status == 'downloading':
|
|
result['counts']['downloading'] = db.get_downloading_attachment_count(creator_id=creator_id)
|
|
result['downloading'] = db.get_downloading_attachments(creator_id=creator_id, limit=item_limit)
|
|
|
|
if status is None or status == 'failed':
|
|
result['counts']['failed'] = db.get_failed_attachment_count()
|
|
result['failed'] = db.get_failed_downloads(limit=item_limit)
|
|
|
|
return result
|
|
|
|
|
|
@router.delete("/queue/{attachment_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def remove_from_queue(
|
|
request: Request,
|
|
attachment_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove a single item from the download queue"""
|
|
db = _get_db_adapter()
|
|
success = db.update_attachment_status(attachment_id, 'skipped',
|
|
error_message='Removed from queue by user')
|
|
if not success:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
return {"status": "removed", "id": attachment_id}
|
|
|
|
|
|
@router.post("/queue/remove")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def remove_multiple_from_queue(
|
|
request: Request,
|
|
body: RemoveFromQueueRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove multiple items from the download queue"""
|
|
db = _get_db_adapter()
|
|
removed = 0
|
|
for att_id in body.attachment_ids:
|
|
if db.update_attachment_status(att_id, 'skipped',
|
|
error_message='Removed from queue by user'):
|
|
removed += 1
|
|
return {"status": "removed", "count": removed}
|
|
|
|
|
|
@router.put("/queue/{attachment_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_queue_item(
|
|
request: Request,
|
|
attachment_id: int,
|
|
body: UpdateQueueItemRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update queue item settings (e.g., auto_requeue)"""
|
|
db = _get_db_adapter()
|
|
updates = {}
|
|
if body.auto_requeue is not None:
|
|
updates['auto_requeue'] = 1 if body.auto_requeue else 0
|
|
|
|
if updates:
|
|
success = db.update_attachment(attachment_id, updates)
|
|
if not success:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
|
|
return {"status": "updated", "id": attachment_id}
|
|
|
|
|
|
@router.post("/queue/stop-all")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def stop_all_downloads(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Stop all active downloads and clear the queue"""
|
|
from modules.activity_status import get_activity_manager
|
|
|
|
app_state = get_app_state()
|
|
db = _get_db_adapter()
|
|
activity_manager = get_activity_manager(app_state.db if app_state else None)
|
|
|
|
# Cancel active syncs via activity_manager (database-backed, works across processes)
|
|
stopped_syncs = 0
|
|
all_tasks = activity_manager.get_active_background_tasks()
|
|
for task in all_tasks:
|
|
if task.get('task_type') == 'paid_content_sync':
|
|
activity_manager.stop_background_task(task['task_id'])
|
|
stopped_syncs += 1
|
|
|
|
# Also clear from app_state for backwards compatibility
|
|
if hasattr(app_state, 'active_paid_content_syncs'):
|
|
sync_ids = list(app_state.active_paid_content_syncs.keys())
|
|
for sync_id in sync_ids:
|
|
app_state.active_paid_content_syncs.pop(sync_id, None)
|
|
|
|
# Mark downloading items as pending (so they can be resumed later)
|
|
downloading_reset = db.reset_downloading_to_pending()
|
|
|
|
return {
|
|
"status": "stopped",
|
|
"syncs_cancelled": stopped_syncs,
|
|
"downloads_paused": downloading_reset
|
|
}
|
|
|
|
|
|
@router.post("/queue/clear-failed")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def clear_failed_downloads(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Clear all failed downloads (mark as skipped)"""
|
|
db = _get_db_adapter()
|
|
count = db.clear_failed_downloads()
|
|
return {"status": "cleared", "count": count}
|
|
|
|
|
|
# ============================================================================
|
|
# SERVICES ENDPOINTS
|
|
# ============================================================================
|
|
|
|
async def _check_single_service_health(service: dict, app_state) -> dict:
|
|
"""Check health for a single service. Returns health dict."""
|
|
service_id = service['id']
|
|
|
|
if service_id == 'youtube':
|
|
from modules.paid_content import YouTubeClient
|
|
youtube = YouTubeClient()
|
|
return {'status': 'healthy', 'message': 'yt-dlp is available'} if youtube.is_available() \
|
|
else {'status': 'down', 'message': 'yt-dlp not found'}
|
|
|
|
elif service_id == 'twitch':
|
|
from modules.paid_content import TwitchClient
|
|
twitch = TwitchClient()
|
|
return {'status': 'healthy', 'message': 'yt-dlp is available for Twitch clips'} if twitch.is_available() \
|
|
else {'status': 'down', 'message': 'yt-dlp not found'}
|
|
|
|
elif service_id == 'fansly_direct':
|
|
from modules.paid_content import FanslyDirectClient
|
|
auth_token = service.get('session_cookie')
|
|
if not auth_token:
|
|
return {'status': 'down', 'message': 'Auth token not configured'}
|
|
client = FanslyDirectClient(auth_token=auth_token)
|
|
# Skip rate limiting for health checks
|
|
client._init_rate_limiter(min_delay=0, max_delay=0, batch_delay_min=0, batch_delay_max=0)
|
|
try:
|
|
result = await client.check_auth()
|
|
if result.get('valid'):
|
|
return {'status': 'healthy', 'message': f"Connected as {result.get('username', 'unknown')}"}
|
|
return {'status': 'down', 'message': result.get('error', 'Auth failed')}
|
|
finally:
|
|
await client.close()
|
|
|
|
elif service_id == 'onlyfans_direct':
|
|
from modules.paid_content import OnlyFansClient
|
|
import json as _json
|
|
raw = service.get('session_cookie')
|
|
if not raw:
|
|
return {'status': 'down', 'message': 'Credentials not configured'}
|
|
try:
|
|
auth_config = _json.loads(raw)
|
|
except (_json.JSONDecodeError, TypeError):
|
|
auth_config = {}
|
|
if not auth_config.get('sess'):
|
|
return {'status': 'down', 'message': 'Credentials not configured'}
|
|
client = OnlyFansClient(auth_config=auth_config, signing_url=auth_config.get('signing_url'))
|
|
# Skip rate limiting for health checks
|
|
client._init_rate_limiter(min_delay=0, max_delay=0, batch_delay_min=0, batch_delay_max=0)
|
|
try:
|
|
result = await client.check_auth()
|
|
if result.get('valid'):
|
|
return {'status': 'healthy', 'message': f"Connected as {result.get('username', 'unknown')}"}
|
|
return {'status': 'down', 'message': result.get('error', 'Auth failed')}
|
|
finally:
|
|
await client.close()
|
|
|
|
elif service_id == 'pornhub':
|
|
from modules.paid_content.pornhub_client import PornhubClient
|
|
pornhub = PornhubClient(unified_db=app_state.db)
|
|
if not pornhub.is_available():
|
|
return {'status': 'down', 'message': 'yt-dlp not found'}
|
|
# Check cookie health from scrapers table
|
|
try:
|
|
scraper = app_state.db.get_scraper('pornhub')
|
|
if scraper:
|
|
if scraper.get('last_test_status') == 'failed':
|
|
msg = scraper.get('last_test_message') or 'Cookies expired'
|
|
return {'status': 'degraded', 'message': f'yt-dlp available but cookies bad: {msg}'}
|
|
if scraper.get('cookies_json'):
|
|
return {'status': 'healthy', 'message': 'yt-dlp available with cookies'}
|
|
except Exception:
|
|
pass
|
|
return {'status': 'healthy', 'message': 'yt-dlp available (no cookies — some content may be restricted)'}
|
|
|
|
elif service_id == 'xhamster':
|
|
from modules.paid_content.xhamster_client import XHamsterClient
|
|
xhamster = XHamsterClient(unified_db=app_state.db)
|
|
if xhamster.is_available():
|
|
return {'status': 'healthy', 'message': 'yt-dlp is available for XHamster'}
|
|
return {'status': 'down', 'message': 'yt-dlp not found'}
|
|
|
|
elif service_id == 'tiktok':
|
|
from modules.paid_content.tiktok_client import TikTokClient
|
|
tiktok = TikTokClient(unified_db=app_state.db)
|
|
if not tiktok.ytdlp_path:
|
|
return {'status': 'down', 'message': 'yt-dlp not found'}
|
|
if not tiktok.is_available():
|
|
return {'status': 'degraded', 'message': 'gallery-dl not found (downloads may fail)'}
|
|
# Check cookie health from scrapers table
|
|
try:
|
|
scraper = app_state.db.get_scraper('tiktok')
|
|
if scraper:
|
|
if scraper.get('last_test_status') == 'failed':
|
|
msg = scraper.get('last_test_message') or 'Cookies expired or invalid'
|
|
return {'status': 'degraded', 'message': f'Tools available but cookies bad: {msg}'}
|
|
cookies_json = scraper.get('cookies_json')
|
|
if not cookies_json:
|
|
return {'status': 'degraded', 'message': 'Tools available but no cookies configured'}
|
|
except Exception:
|
|
pass
|
|
return {'status': 'healthy', 'message': 'yt-dlp and gallery-dl available with cookies'}
|
|
|
|
elif service_id == 'instagram':
|
|
# Instagram on paid content uses ImgInn API (no login cookies required)
|
|
# Just verify imginn.com is reachable and FlareSolverr handles Cloudflare
|
|
import httpx
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get('https://imginn.com', headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
# 403 from Cloudflare is normal — FlareSolverr handles bypass at runtime
|
|
if resp.status_code in (200, 403):
|
|
return {'status': 'healthy', 'message': 'ImgInn API available (via FlareSolverr)'}
|
|
return {'status': 'degraded', 'message': f'ImgInn returned HTTP {resp.status_code}'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach ImgInn: {str(e)}'}
|
|
|
|
elif service_id == 'soundgasm':
|
|
# Soundgasm requires no cookies — just check the site is reachable
|
|
import httpx
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get('https://soundgasm.net', headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
if resp.status_code == 200:
|
|
return {'status': 'healthy', 'message': 'Soundgasm reachable (no cookies required)'}
|
|
return {'status': 'degraded', 'message': f'HTTP {resp.status_code}'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach Soundgasm: {str(e)}'}
|
|
|
|
elif service_id == 'bellazon':
|
|
# Bellazon works with or without cookies — cookies unlock restricted content
|
|
import httpx
|
|
try:
|
|
session_cookie = service.get('session_cookie')
|
|
cookie_dict = {}
|
|
if session_cookie:
|
|
import json as _json
|
|
try:
|
|
parsed = _json.loads(session_cookie)
|
|
if isinstance(parsed, dict):
|
|
cookie_dict = parsed
|
|
elif isinstance(parsed, list):
|
|
cookie_dict = {c['name']: c['value'] for c in parsed if c.get('name') and c.get('value')}
|
|
except (_json.JSONDecodeError, TypeError):
|
|
cookie_dict = {}
|
|
|
|
base_url = service.get('base_url', 'https://www.bellazon.com/main')
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
}
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get(base_url, headers=headers, cookies=cookie_dict)
|
|
if resp.status_code != 200:
|
|
return {'status': 'degraded', 'message': f'HTTP {resp.status_code}'}
|
|
body = resp.text
|
|
# Check if logged in by looking for sign-out link or member menu
|
|
logged_in = 'sign-out' in body.lower() or 'signout' in body.lower() or 'ipsUserPhoto' in body
|
|
if cookie_dict:
|
|
if logged_in:
|
|
return {'status': 'healthy', 'message': 'Logged in with cookies'}
|
|
else:
|
|
return {'status': 'degraded', 'message': 'Cookies set but not logged in — session may be expired'}
|
|
else:
|
|
return {'status': 'healthy', 'message': 'Reachable (no cookies — limited access)'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach Bellazon: {str(e)}'}
|
|
|
|
elif service_id in ('hqcelebcorner', 'picturepub'):
|
|
# XenForo forums — check if the site is reachable (via FlareSolverr if needed)
|
|
import httpx
|
|
base_url = service.get('base_url', '')
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get(base_url, headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
if resp.status_code in (200, 403):
|
|
# 403 is expected from Cloudflare — FlareSolverr handles bypass at runtime
|
|
return {'status': 'healthy', 'message': f'Forum reachable (HTTP {resp.status_code}, FlareSolverr used at runtime)'}
|
|
return {'status': 'degraded', 'message': f'HTTP {resp.status_code}'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach forum: {str(e)}'}
|
|
|
|
elif service_id == 'coppermine':
|
|
# Coppermine is a pure HTTP scraper — no base_url needed (per-creator gallery URLs)
|
|
import httpx
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
# Test reachability with a known Coppermine gallery host
|
|
resp = await client.get('https://www.google.com', headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
return {'status': 'healthy', 'message': 'HTTP client available (per-creator gallery URLs)'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'HTTP client error: {str(e)}'}
|
|
|
|
elif service_id == 'besteyecandy':
|
|
# BestEyeCandy — check if the site is reachable
|
|
import httpx
|
|
base_url = service.get('base_url', 'https://besteyecandy.com')
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get(base_url, headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
})
|
|
if resp.status_code == 200:
|
|
return {'status': 'healthy', 'message': 'BestEyeCandy reachable'}
|
|
elif resp.status_code == 403:
|
|
return {'status': 'healthy', 'message': 'BestEyeCandy reachable (Cloudflare, FlareSolverr used at runtime)'}
|
|
return {'status': 'degraded', 'message': f'HTTP {resp.status_code}'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach BestEyeCandy: {str(e)}'}
|
|
|
|
elif service_id == 'reddit':
|
|
# Reddit uses gallery-dl — check that gallery-dl is available
|
|
import shutil as _shutil
|
|
import os as _os
|
|
gdl = _shutil.which('gallery-dl') or '/opt/media-downloader/venv/bin/gallery-dl'
|
|
if _os.path.isfile(gdl):
|
|
# Also check cookie health from scrapers table
|
|
try:
|
|
scraper = app_state.db.get_scraper('gallerydl')
|
|
if scraper and scraper.get('last_test_status') == 'failed':
|
|
msg = scraper.get('last_test_message') or 'gallery-dl test failed'
|
|
return {'status': 'degraded', 'message': f'gallery-dl available but: {msg}'}
|
|
except Exception:
|
|
pass
|
|
return {'status': 'healthy', 'message': 'gallery-dl available for Reddit'}
|
|
return {'status': 'down', 'message': 'gallery-dl not found'}
|
|
|
|
elif service_id == 'snapchat':
|
|
# Snapchat paid content uses HTTP client (no cookies needed for public profiles)
|
|
import httpx
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
|
resp = await client.get('https://story.snapchat.com', headers={
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
})
|
|
if resp.status_code in (200, 301, 302):
|
|
return {'status': 'healthy', 'message': 'Snapchat story endpoint reachable'}
|
|
return {'status': 'degraded', 'message': f'HTTP {resp.status_code}'}
|
|
except Exception as e:
|
|
return {'status': 'down', 'message': f'Cannot reach Snapchat: {str(e)}'}
|
|
|
|
else:
|
|
from modules.paid_content import PaidContentAPIClient
|
|
client = PaidContentAPIClient(
|
|
service_id,
|
|
session_cookie=service.get('session_cookie'),
|
|
base_url=service.get('base_url')
|
|
)
|
|
try:
|
|
return await client.check_health()
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
async def _run_auto_health_checks():
|
|
"""Run health checks for all services in the background with per-service timeout."""
|
|
global _auto_health_check_running, _last_auto_health_check
|
|
import time
|
|
try:
|
|
_auto_health_check_running = True
|
|
db = _get_db_adapter()
|
|
app_state = get_app_state()
|
|
services = db.get_services()
|
|
|
|
for service in services:
|
|
service_id = service['id']
|
|
try:
|
|
# 30s timeout per service to prevent blocking
|
|
health = await asyncio.wait_for(
|
|
_check_single_service_health(service, app_state),
|
|
timeout=30.0
|
|
)
|
|
db.update_service(service_id, {
|
|
'health_status': health.get('status', 'unknown'),
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
except asyncio.TimeoutError:
|
|
logger.warning(f"Auto health check timed out for {service_id}", module="PaidContent")
|
|
db.update_service(service_id, {
|
|
'health_status': 'degraded',
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"Auto health check failed for {service_id}: {e}", module="PaidContent")
|
|
db.update_service(service_id, {
|
|
'health_status': 'down',
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
|
|
_last_auto_health_check = time.time()
|
|
logger.info("Auto health checks completed", module="PaidContent")
|
|
except Exception as e:
|
|
logger.error(f"Auto health check error: {e}", module="PaidContent")
|
|
finally:
|
|
_auto_health_check_running = False
|
|
|
|
|
|
@router.get("/services")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_services(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all services with health status"""
|
|
global _last_auto_health_check
|
|
import time
|
|
|
|
db = _get_db_adapter()
|
|
services = db.get_services()
|
|
|
|
# Auto-trigger health checks if stale (not checked in last 5 minutes)
|
|
if not _auto_health_check_running:
|
|
now = time.time()
|
|
needs_check = _last_auto_health_check is None or (now - _last_auto_health_check) > _AUTO_HEALTH_CHECK_INTERVAL
|
|
if needs_check:
|
|
_last_auto_health_check = now # Set immediately to prevent duplicate triggers
|
|
if any(not s.get('last_health_check') or s.get('health_status') in ('unknown', None, '') for s in services):
|
|
# First check or stale data — run synchronously so response has fresh data
|
|
await _run_auto_health_checks()
|
|
services = db.get_services()
|
|
else:
|
|
asyncio.create_task(_run_auto_health_checks())
|
|
|
|
return {"services": services}
|
|
|
|
|
|
@router.get("/services/{service_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_service(
|
|
request: Request,
|
|
service_id: str,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get single service details"""
|
|
db = _get_db_adapter()
|
|
service = db.get_service(service_id)
|
|
if not service:
|
|
raise NotFoundError(f"Service '{service_id}' not found")
|
|
return service
|
|
|
|
|
|
@router.get("/services/{service_id}/health")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def check_service_health(
|
|
request: Request,
|
|
service_id: str,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Check service health"""
|
|
db = _get_db_adapter()
|
|
app_state = get_app_state()
|
|
service = db.get_service(service_id)
|
|
if not service:
|
|
raise NotFoundError(f"Service '{service_id}' not found")
|
|
|
|
try:
|
|
health = await asyncio.wait_for(
|
|
_check_single_service_health(service, app_state),
|
|
timeout=30.0
|
|
)
|
|
except asyncio.TimeoutError:
|
|
health = {'status': 'degraded', 'message': 'Health check timed out'}
|
|
except Exception as e:
|
|
health = {'status': 'down', 'message': str(e)}
|
|
|
|
db.update_service(service_id, {
|
|
'health_status': health.get('status', 'unknown'),
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
|
|
return health
|
|
|
|
|
|
@router.post("/services/check-all-health")
|
|
@limiter.limit("6/minute")
|
|
@handle_exceptions
|
|
async def check_all_services_health(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Check health of all services"""
|
|
db = _get_db_adapter()
|
|
app_state = get_app_state()
|
|
services = db.get_services()
|
|
results = {}
|
|
|
|
for service in services:
|
|
service_id = service['id']
|
|
try:
|
|
health = await asyncio.wait_for(
|
|
_check_single_service_health(service, app_state),
|
|
timeout=30.0
|
|
)
|
|
db.update_service(service_id, {
|
|
'health_status': health.get('status', 'unknown'),
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
results[service_id] = health
|
|
except asyncio.TimeoutError:
|
|
logger.warning(f"Health check timed out for {service_id}", module="PaidContent")
|
|
db.update_service(service_id, {
|
|
'health_status': 'degraded',
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
results[service_id] = {'status': 'degraded', 'message': 'Health check timed out'}
|
|
except Exception as e:
|
|
logger.error(f"Health check failed for {service_id}: {e}", module="PaidContent")
|
|
db.update_service(service_id, {
|
|
'health_status': 'down',
|
|
'last_health_check': datetime.now().isoformat()
|
|
})
|
|
results[service_id] = {'status': 'down', 'message': str(e)}
|
|
|
|
return {'services': results, 'checked_at': datetime.now().isoformat()}
|
|
|
|
|
|
@router.post("/services/{service_id}/session")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def update_session(
|
|
request: Request,
|
|
service_id: str,
|
|
body: UpdateSessionRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update session cookie for service"""
|
|
db = _get_db_adapter()
|
|
success = db.update_service(service_id, {
|
|
'session_cookie': body.session_cookie,
|
|
'session_updated_at': datetime.now().isoformat()
|
|
})
|
|
|
|
if not success:
|
|
raise NotFoundError(f"Service '{service_id}' not found")
|
|
|
|
return message_response("Session updated successfully")
|
|
|
|
|
|
class UpdateServiceUrlRequest(BaseModel):
|
|
base_url: str
|
|
|
|
|
|
@router.put("/services/{service_id}/url")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def update_service_url(
|
|
request: Request,
|
|
service_id: str,
|
|
body: UpdateServiceUrlRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update base URL for service (when domains change)"""
|
|
db = _get_db_adapter()
|
|
|
|
# Validate URL format
|
|
url = body.base_url.strip().rstrip('/')
|
|
if not url.startswith('https://') and not url.startswith('http://'):
|
|
raise ValidationError("URL must start with http:// or https://")
|
|
|
|
# Remove /api/v1 suffix if present (we add it automatically)
|
|
if '/api/v1' in url:
|
|
url = url.replace('/api/v1', '').rstrip('/')
|
|
|
|
success = db.update_service(service_id, {
|
|
'base_url': url,
|
|
'updated_at': datetime.now().isoformat(),
|
|
'health_status': 'unknown' # Reset health status since URL changed
|
|
})
|
|
|
|
if not success:
|
|
raise NotFoundError(f"Service '{service_id}' not found")
|
|
|
|
logger.info(f"Updated {service_id} base URL to: {url}", module="PaidContent")
|
|
return message_response("Service URL updated successfully")
|
|
|
|
|
|
# ============================================================================
|
|
# FANSLY DIRECT ENDPOINTS
|
|
# ============================================================================
|
|
|
|
class FanslyVerifyAuthRequest(BaseModel):
|
|
auth_token: str
|
|
|
|
|
|
class FanslyDirectSyncRequest(BaseModel):
|
|
username: str
|
|
date_from: Optional[str] = None
|
|
date_to: Optional[str] = None
|
|
days_back: Optional[int] = None
|
|
download: bool = True
|
|
|
|
|
|
@router.post("/fansly-direct/verify-auth")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def verify_fansly_auth(
|
|
request: Request,
|
|
body: FanslyVerifyAuthRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Verify Fansly auth token and return account info"""
|
|
from modules.paid_content import FanslyDirectClient
|
|
|
|
client = FanslyDirectClient(auth_token=body.auth_token)
|
|
try:
|
|
result = await client.check_auth()
|
|
return result
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
@router.post("/fansly-direct/sync")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def sync_fansly_direct(
|
|
request: Request,
|
|
body: FanslyDirectSyncRequest,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Sync a Fansly creator via direct API.
|
|
Auto-adds the creator if not already tracked.
|
|
"""
|
|
db = _get_db_adapter()
|
|
|
|
# Check if auth token is configured
|
|
service = db.get_service('fansly_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
raise ValidationError("Fansly auth token not configured. Please set it in Settings.")
|
|
|
|
# Check if creator exists, if not add them
|
|
# Try by creator_id first, then by username (creator_id may be numeric Fansly ID)
|
|
existing = db.get_creator_by_api_id('fansly_direct', 'fansly', body.username)
|
|
if not existing:
|
|
existing = db.get_creator_by_username('fansly_direct', 'fansly', body.username)
|
|
if existing:
|
|
creator_id = existing['id']
|
|
else:
|
|
# Add new creator
|
|
scraper = _get_scraper()
|
|
try:
|
|
result = await scraper.add_creator(
|
|
service_id='fansly_direct',
|
|
platform='fansly',
|
|
creator_id=body.username,
|
|
auto_download=body.download
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
if not result.get('success'):
|
|
raise ValidationError(result.get('error', 'Failed to add creator'))
|
|
creator_id = result['creator']['id']
|
|
|
|
# Build date filter info for response
|
|
date_filter = None
|
|
if body.date_from or body.date_to:
|
|
date_filter = {'from': body.date_from, 'to': body.date_to}
|
|
elif body.days_back:
|
|
date_filter = {'days_back': body.days_back}
|
|
|
|
# Queue background sync with date filters
|
|
background_tasks.add_task(
|
|
_sync_fansly_direct_background,
|
|
creator_id,
|
|
body.download,
|
|
body.date_from,
|
|
body.date_to,
|
|
body.days_back
|
|
)
|
|
|
|
return {
|
|
"status": "queued",
|
|
"creator_id": creator_id,
|
|
"username": body.username,
|
|
"date_filter": date_filter
|
|
}
|
|
|
|
|
|
async def _sync_fansly_direct_background(
|
|
creator_id: int,
|
|
download: bool,
|
|
date_from: str = None,
|
|
date_to: str = None,
|
|
days_back: int = None
|
|
):
|
|
"""Background task for Fansly Direct sync with date filters"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
creator = scraper.db.get_creator(creator_id)
|
|
if creator and creator.get('service_id') == 'fansly_direct':
|
|
await scraper._sync_fansly_direct_creator(
|
|
creator,
|
|
download=download,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
days_back=days_back
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
# Auto-recheck quality for flagged attachments after sync
|
|
try:
|
|
await _auto_quality_recheck_background()
|
|
except Exception as e:
|
|
logger.error(f"Auto quality recheck after sync failed: {e}", module="PaidContent")
|
|
|
|
|
|
# ============================================================================
|
|
# ONLYFANS DIRECT ENDPOINTS
|
|
# ============================================================================
|
|
|
|
class OnlyFansVerifyAuthRequest(BaseModel):
|
|
sess: str
|
|
auth_id: str
|
|
auth_uid: Optional[str] = None
|
|
x_bc: Optional[str] = None
|
|
user_agent: Optional[str] = None
|
|
signing_url: Optional[str] = None
|
|
|
|
|
|
class OnlyFansDirectSyncRequest(BaseModel):
|
|
username: str
|
|
date_from: Optional[str] = None
|
|
date_to: Optional[str] = None
|
|
days_back: Optional[int] = None
|
|
download: bool = True
|
|
|
|
|
|
@router.post("/onlyfans-direct/verify-auth")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def verify_onlyfans_auth(
|
|
request: Request,
|
|
body: OnlyFansVerifyAuthRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Verify OnlyFans credentials and return account info.
|
|
Also saves/updates the credentials in the database on success."""
|
|
from modules.paid_content import OnlyFansClient
|
|
import json as _json
|
|
|
|
# Default user-agent and x-bc if not provided (same approach as Coomer importers)
|
|
default_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
|
|
auth_config = {
|
|
'sess': body.sess,
|
|
'auth_id': body.auth_id,
|
|
'auth_uid': body.auth_uid or '',
|
|
'x_bc': body.x_bc or '',
|
|
'user_agent': body.user_agent or default_ua,
|
|
}
|
|
|
|
client = OnlyFansClient(
|
|
auth_config=auth_config,
|
|
signing_url=body.signing_url,
|
|
)
|
|
try:
|
|
result = await client.check_auth()
|
|
|
|
# If valid, save credentials to database
|
|
if result.get('valid'):
|
|
db = _get_db_adapter()
|
|
cred_blob = _json.dumps({
|
|
**auth_config,
|
|
'signing_url': body.signing_url or '',
|
|
})
|
|
db.update_service('onlyfans_direct', {'session_cookie': cred_blob})
|
|
|
|
return result
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
@router.post("/onlyfans-direct/sync")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def sync_onlyfans_direct(
|
|
request: Request,
|
|
body: OnlyFansDirectSyncRequest,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Sync an OnlyFans creator via direct API.
|
|
Auto-adds the creator if not already tracked.
|
|
"""
|
|
db = _get_db_adapter()
|
|
|
|
# Check if credentials are configured
|
|
service = db.get_service('onlyfans_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
raise ValidationError("OnlyFans credentials not configured. Please set them in Settings.")
|
|
|
|
# Check if creator exists, if not add them
|
|
existing = db.get_creator_by_api_id('onlyfans_direct', 'onlyfans', body.username)
|
|
if existing:
|
|
creator_id = existing['id']
|
|
else:
|
|
scraper = _get_scraper()
|
|
try:
|
|
result = await scraper.add_creator(
|
|
service_id='onlyfans_direct',
|
|
platform='onlyfans',
|
|
creator_id=body.username,
|
|
auto_download=body.download,
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
if not result.get('success'):
|
|
raise ValidationError(result.get('error', 'Failed to add creator'))
|
|
creator_id = result['creator']['id']
|
|
|
|
# Build date filter info for response
|
|
date_filter = None
|
|
if body.date_from or body.date_to:
|
|
date_filter = {'from': body.date_from, 'to': body.date_to}
|
|
elif body.days_back:
|
|
date_filter = {'days_back': body.days_back}
|
|
|
|
# Queue background sync
|
|
background_tasks.add_task(
|
|
_sync_onlyfans_direct_background,
|
|
creator_id,
|
|
body.download,
|
|
body.date_from,
|
|
body.date_to,
|
|
body.days_back,
|
|
)
|
|
|
|
return {
|
|
"status": "queued",
|
|
"creator_id": creator_id,
|
|
"username": body.username,
|
|
"date_filter": date_filter,
|
|
}
|
|
|
|
|
|
async def _sync_onlyfans_direct_background(
|
|
creator_id: int,
|
|
download: bool,
|
|
date_from: str = None,
|
|
date_to: str = None,
|
|
days_back: int = None,
|
|
):
|
|
"""Background task for OnlyFans Direct sync with date filters"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
creator = scraper.db.get_creator(creator_id)
|
|
if creator and creator.get('service_id') == 'onlyfans_direct':
|
|
await scraper._sync_onlyfans_direct_creator(
|
|
creator,
|
|
download=download,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
days_back=days_back,
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
|
|
# ============================================================================
|
|
# ATTACHMENT MANAGEMENT ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.post("/posts/{post_id}/attachments/upload")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def upload_attachment(
|
|
request: Request,
|
|
post_id: int,
|
|
file: UploadFile = File(...),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Upload a new attachment to a post.
|
|
Saves file to post's directory and creates attachment record.
|
|
"""
|
|
import shutil
|
|
|
|
db = _get_db_adapter()
|
|
|
|
# Get post and creator info
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
creator = db.get_creator(post['creator_id'])
|
|
if not creator:
|
|
raise NotFoundError("Creator not found")
|
|
|
|
# Build output path
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/paid-content'))
|
|
|
|
# Organize by platform/username/date
|
|
published_at = post.get('published_at') or ''
|
|
post_date = published_at[:10] if published_at else 'unknown-date'
|
|
output_dir = base_path / creator['platform'] / creator['username'] / post_date
|
|
|
|
# Create directory if needed
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Generate unique filename if file exists
|
|
original_name = file.filename or 'uploaded_file'
|
|
output_path = output_dir / original_name
|
|
counter = 1
|
|
while output_path.exists():
|
|
name_parts = original_name.rsplit('.', 1)
|
|
if len(name_parts) == 2:
|
|
output_path = output_dir / f"{name_parts[0]}_{counter}.{name_parts[1]}"
|
|
else:
|
|
output_path = output_dir / f"{original_name}_{counter}"
|
|
counter += 1
|
|
|
|
# Save file
|
|
try:
|
|
with open(output_path, 'wb') as f:
|
|
shutil.copyfileobj(file.file, f)
|
|
except Exception as e:
|
|
raise ValidationError(f"Failed to save file: {str(e)}")
|
|
|
|
# Get file info
|
|
file_size = output_path.stat().st_size
|
|
file_type = 'unknown'
|
|
ext = output_path.suffix.lower().lstrip('.')
|
|
|
|
image_exts = {'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff', 'heic', 'heif', 'avif'}
|
|
video_exts = {'mp4', 'mov', 'avi', 'mkv', 'webm', 'm4v', 'wmv', 'flv'}
|
|
audio_exts = {'mp3', 'wav', 'flac', 'aac', 'm4a', 'ogg'}
|
|
|
|
if ext in image_exts:
|
|
file_type = 'image'
|
|
elif ext in video_exts:
|
|
file_type = 'video'
|
|
elif ext in audio_exts:
|
|
file_type = 'audio'
|
|
|
|
# Get next attachment index
|
|
existing_attachments = db.get_post_attachments(post_id)
|
|
next_index = max((a.get('attachment_index', 0) for a in existing_attachments), default=-1) + 1
|
|
|
|
# Extract dimensions/duration for images and videos
|
|
width, height, duration = None, None, None
|
|
if file_type == 'image':
|
|
try:
|
|
from PIL import Image
|
|
with Image.open(output_path) as img:
|
|
width, height = img.size
|
|
except Exception:
|
|
pass
|
|
elif file_type == 'video':
|
|
try:
|
|
import subprocess as _sp
|
|
result = _sp.run(
|
|
['ffprobe', '-v', 'quiet', '-print_format', 'json',
|
|
'-show_streams', '-select_streams', 'v:0', str(output_path)],
|
|
capture_output=True, text=True, timeout=10
|
|
)
|
|
if result.returncode == 0:
|
|
probe = json.loads(result.stdout)
|
|
if probe.get('streams'):
|
|
stream = probe['streams'][0]
|
|
width = stream.get('width')
|
|
height = stream.get('height')
|
|
duration_str = stream.get('duration')
|
|
if duration_str:
|
|
duration = int(float(duration_str))
|
|
except Exception:
|
|
pass
|
|
|
|
# Create attachment record
|
|
attachment_data = {
|
|
'name': output_path.name,
|
|
'file_type': file_type,
|
|
'extension': ext,
|
|
'server_path': f'uploaded/{output_path.name}',
|
|
'file_size': file_size,
|
|
'width': width,
|
|
'height': height,
|
|
'duration': duration,
|
|
'status': 'completed',
|
|
'local_path': str(output_path),
|
|
'local_filename': output_path.name,
|
|
'attachment_index': next_index,
|
|
'downloaded_at': datetime.now().isoformat()
|
|
}
|
|
|
|
attachment_id = db.upsert_attachment(post_id, attachment_data)
|
|
|
|
# Invalidate stale thumbnail caches so they regenerate from the new file
|
|
db.update_attachment(attachment_id, {'thumbnail_data': None})
|
|
for thumb_size in ('small', 'medium', 'large', 'native'):
|
|
stale_thumb = Path(f"/opt/media-downloader/cache/thumbnails/{thumb_size}/{attachment_id}.jpg")
|
|
if stale_thumb.exists():
|
|
stale_thumb.unlink()
|
|
|
|
return {
|
|
"attachment_id": attachment_id,
|
|
"filename": output_path.name,
|
|
"file_type": file_type,
|
|
"file_size": file_size,
|
|
"width": width,
|
|
"height": height,
|
|
"duration": duration,
|
|
"local_path": str(output_path)
|
|
}
|
|
|
|
|
|
@router.delete("/posts/{post_id}/attachments/{attachment_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_attachment(
|
|
request: Request,
|
|
post_id: int,
|
|
attachment_id: int,
|
|
permanent: bool = False,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Delete an attachment from a post.
|
|
If permanent=True, deletes file and DB record.
|
|
If permanent=False, moves file to recycle bin.
|
|
"""
|
|
db = _get_db_adapter()
|
|
|
|
# Get attachment
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
|
|
if attachment['post_id'] != post_id:
|
|
raise ValidationError("Attachment does not belong to this post")
|
|
|
|
local_path = attachment.get('local_path')
|
|
|
|
if permanent:
|
|
# Delete file from disk
|
|
if local_path and Path(local_path).exists():
|
|
try:
|
|
Path(local_path).unlink()
|
|
except Exception as e:
|
|
logger.warning(f"Failed to delete file {local_path}: {e}", module="PaidContent")
|
|
|
|
# Delete from database
|
|
db.delete_attachment(attachment_id)
|
|
return {"success": True, "action": "deleted"}
|
|
else:
|
|
# Move to recycle bin
|
|
if local_path and Path(local_path).exists():
|
|
config = db.get_config()
|
|
recycle_path = Path(config.get('base_download_path', '/paid-content')) / '.recycle'
|
|
recycle_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
dest_path = recycle_path / Path(local_path).name
|
|
counter = 1
|
|
while dest_path.exists():
|
|
name_parts = Path(local_path).name.rsplit('.', 1)
|
|
if len(name_parts) == 2:
|
|
dest_path = recycle_path / f"{name_parts[0]}_{counter}.{name_parts[1]}"
|
|
else:
|
|
dest_path = recycle_path / f"{Path(local_path).name}_{counter}"
|
|
counter += 1
|
|
|
|
try:
|
|
Path(local_path).rename(dest_path)
|
|
# Update attachment with recycle location
|
|
db.update_attachment(attachment_id, {
|
|
'status': 'deleted',
|
|
'error_message': f'Moved to recycle: {dest_path}'
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"Failed to move file to recycle: {e}", module="PaidContent")
|
|
raise ValidationError(f"Failed to move file: {str(e)}")
|
|
|
|
return {"success": True, "action": "recycled"}
|
|
|
|
|
|
@router.delete("/feed/{post_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_post(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete a post and all its attachments (files + DB records)."""
|
|
db = _get_db_adapter()
|
|
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
# Delete attachment files from disk
|
|
with db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT * FROM paid_content_attachments WHERE post_id = ?', (post_id,))
|
|
attachments = [dict(row) for row in cursor.fetchall()]
|
|
for att in attachments:
|
|
local_path = att.get('local_path')
|
|
if local_path:
|
|
try:
|
|
Path(local_path).unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
except OSError as e:
|
|
logger.warning(f"Could not delete {local_path}: {e}", module="PaidContent")
|
|
|
|
# Soft delete: set deleted_at timestamp (keeps record for sync dedup)
|
|
# Attachment DB records are kept so upsert_attachment finds them → no re-download
|
|
with db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("UPDATE paid_content_posts SET deleted_at = datetime('now') WHERE id = ?", (post_id,))
|
|
conn.commit()
|
|
|
|
return {"success": True, "message": "Post deleted"}
|
|
|
|
|
|
@router.post("/feed/bulk-delete")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def bulk_delete_posts(
|
|
request: Request,
|
|
body: BulkDeletePostsRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Bulk soft-delete posts and their attachment files."""
|
|
db = _get_db_adapter()
|
|
deleted_count = 0
|
|
|
|
for post_id in body.post_ids:
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
continue
|
|
|
|
# Delete attachment files from disk
|
|
with db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT * FROM paid_content_attachments WHERE post_id = ?', (post_id,))
|
|
attachments = [dict(row) for row in cursor.fetchall()]
|
|
for att in attachments:
|
|
local_path = att.get('local_path')
|
|
if local_path:
|
|
try:
|
|
Path(local_path).unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
except OSError as e:
|
|
logger.warning(f"Could not delete {local_path}: {e}", module="PaidContent")
|
|
|
|
# Soft delete: set deleted_at timestamp
|
|
with db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("UPDATE paid_content_posts SET deleted_at = datetime('now') WHERE id = ?", (post_id,))
|
|
conn.commit()
|
|
deleted_count += 1
|
|
|
|
return {"success": True, "deleted_count": deleted_count}
|
|
|
|
|
|
# ============================================================================
|
|
# CREATORS ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/creators")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_creators(
|
|
request: Request,
|
|
service: Optional[str] = None,
|
|
platform: Optional[str] = None,
|
|
identity_id: Optional[int] = None,
|
|
search: Optional[str] = None,
|
|
enabled_only: bool = False,
|
|
limit: int = Query(default=100, le=500),
|
|
offset: int = 0,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get tracked creators"""
|
|
db = _get_db_adapter()
|
|
creators = db.get_creators(
|
|
service_id=service,
|
|
platform=platform,
|
|
identity_id=identity_id,
|
|
search=search,
|
|
enabled_only=enabled_only,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
return {"creators": creators, "count": len(creators)}
|
|
|
|
|
|
@router.get("/creators/search")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def search_creators(
|
|
request: Request,
|
|
service_id: str,
|
|
query: str,
|
|
platform: Optional[str] = None,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Search for creators on a service"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
results = await scraper.search_creators(service_id, query, platform)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
return {"results": results, "count": len(results)}
|
|
|
|
|
|
@router.get("/creators/{creator_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_creator(
|
|
request: Request,
|
|
creator_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get single creator details"""
|
|
db = _get_db_adapter()
|
|
creator = db.get_creator(creator_id)
|
|
if not creator:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
# Add stats
|
|
creator['stats'] = db.get_creator_stats(creator_id)
|
|
return creator
|
|
|
|
|
|
@router.post("/creators")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def add_creator(
|
|
request: Request,
|
|
body: AddCreatorRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add new creator to track"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
result = await scraper.add_creator(
|
|
service_id=body.service_id,
|
|
platform=body.platform,
|
|
creator_id=body.creator_id,
|
|
auto_download=body.auto_download,
|
|
download_embeds=body.download_embeds
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
if not result.get('success'):
|
|
raise ValidationError(result.get('error', 'Failed to add creator'))
|
|
|
|
return {"creator": result['creator'], "message": "Creator added successfully"}
|
|
|
|
|
|
class AddCreatorByUrlRequest(BaseModel):
|
|
url: str
|
|
auto_download: bool = True
|
|
download_embeds: bool = True
|
|
|
|
|
|
@router.post("/creators/by-url")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def add_creator_by_url(
|
|
request: Request,
|
|
body: AddCreatorByUrlRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add new creator by parsing a Coomer/Kemono URL"""
|
|
from modules.paid_content.utils import parse_creator_url
|
|
|
|
parsed = parse_creator_url(body.url)
|
|
if not parsed:
|
|
raise ValidationError("Invalid URL. Expected format: https://onlyfans.com/username, https://coomer.party/onlyfans/user/creatorid, https://www.youtube.com/@channelhandle, https://www.pornhub.com/pornstar/name, https://xhamster.com/creators/name, https://www.tiktok.com/@username, https://www.instagram.com/username, https://besteyecandy.com/section/celeb-photogallery/cid-XXX/.../Name.html, or https://example.com/gallery/index.php (Coppermine gallery)")
|
|
|
|
service_id, platform, creator_id = parsed
|
|
|
|
scraper = _get_scraper()
|
|
try:
|
|
result = await scraper.add_creator(
|
|
service_id=service_id,
|
|
platform=platform,
|
|
creator_id=creator_id,
|
|
auto_download=body.auto_download,
|
|
download_embeds=body.download_embeds
|
|
)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
if not result.get('success'):
|
|
raise ValidationError(result.get('error', 'Failed to add creator'))
|
|
|
|
return {"creator": result['creator'], "message": "Creator added successfully"}
|
|
|
|
|
|
@router.put("/creators/{creator_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_creator(
|
|
request: Request,
|
|
creator_id: int,
|
|
body: UpdateCreatorRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update creator settings"""
|
|
db = _get_db_adapter()
|
|
|
|
updates = body.dict(exclude_none=True)
|
|
if not updates:
|
|
raise ValidationError("No updates provided")
|
|
|
|
# Convert booleans to integers for database storage
|
|
for key in ['enabled', 'auto_download', 'download_embeds', 'sync_posts', 'sync_stories', 'sync_highlights', 'use_authenticated_api']:
|
|
if key in updates:
|
|
updates[key] = 1 if updates[key] else 0
|
|
|
|
# Convert filter_tagged_users list to JSON string
|
|
if 'filter_tagged_users' in updates:
|
|
updates['filter_tagged_users'] = json.dumps(updates['filter_tagged_users'])
|
|
|
|
success = db.update_creator(creator_id, updates)
|
|
if not success:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
return message_response("Creator updated successfully")
|
|
|
|
|
|
@router.get("/creators/{creator_id}/tagged-users")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_creator_tagged_users(
|
|
request: Request,
|
|
creator_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get distinct tagged usernames for a creator's posts"""
|
|
db = _get_db_adapter()
|
|
return db.get_creator_tagged_usernames(creator_id)
|
|
|
|
|
|
@router.delete("/creators/{creator_id}")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def delete_creator(
|
|
request: Request,
|
|
creator_id: int,
|
|
delete_files: bool = False,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove creator and optionally delete their files"""
|
|
db = _get_db_adapter()
|
|
|
|
# Get creator info before deletion
|
|
creator = db.get_creator(creator_id)
|
|
if not creator:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
files_deleted = 0
|
|
|
|
# Optionally delete files from filesystem
|
|
if delete_files:
|
|
import shutil
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/paid-content'))
|
|
creator_path = base_path / creator['platform'] / creator['username']
|
|
|
|
if creator_path.exists() and creator_path.is_dir():
|
|
try:
|
|
# Count files before deletion
|
|
files_deleted = sum(1 for _ in creator_path.rglob('*') if _.is_file())
|
|
shutil.rmtree(creator_path)
|
|
except Exception as e:
|
|
# Log but don't fail - still delete from DB
|
|
logger.warning(f"Could not delete creator files: {e}", module="PaidContent")
|
|
|
|
success = db.delete_creator(creator_id)
|
|
if not success:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
if delete_files and files_deleted > 0:
|
|
return message_response(f"Creator deleted successfully. Removed {files_deleted} files.")
|
|
return message_response("Creator deleted successfully")
|
|
|
|
|
|
@router.post("/creators/{creator_id}/sync")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def sync_creator(
|
|
request: Request,
|
|
creator_id: int,
|
|
background_tasks: BackgroundTasks,
|
|
download: bool = True,
|
|
force_backfill: bool = False,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Manually trigger sync for creator"""
|
|
db = _get_db_adapter()
|
|
creator = db.get_creator(creator_id)
|
|
if not creator:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
background_tasks.add_task(_sync_creator_background, creator_id, download, force_backfill)
|
|
return {"status": "queued", "creator": creator['username']}
|
|
|
|
|
|
@router.post("/creators/sync-service/{service_id}")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def sync_service_creators(
|
|
request: Request,
|
|
service_id: str,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Sync all enabled creators for a specific service"""
|
|
db = _get_db_adapter()
|
|
creators = db.get_creators(service_id=service_id, enabled_only=True, limit=1000)
|
|
if not creators:
|
|
raise NotFoundError(f"No enabled creators found for service '{service_id}'")
|
|
|
|
usernames = [c['username'] for c in creators]
|
|
creator_ids = [c['id'] for c in creators]
|
|
background_tasks.add_task(_sync_service_background, creator_ids)
|
|
return {"status": "queued", "count": len(creators), "creators": usernames}
|
|
|
|
|
|
@router.post("/creators/sync-all")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def sync_all_creators(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Sync all enabled creators"""
|
|
background_tasks.add_task(_sync_all_creators_background)
|
|
return {"status": "queued"}
|
|
|
|
|
|
@router.post("/download-queue/start")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def start_download_queue(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Download all pending attachments without syncing metadata"""
|
|
async def _download_all_background():
|
|
try:
|
|
scraper = _get_scraper()
|
|
scraper.app_state = None # Disable app_state cancellation check for download-only
|
|
await scraper.download_all_pending()
|
|
await scraper.close()
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Background download all failed: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
background_tasks.add_task(_download_all_background)
|
|
return {"status": "queued", "message": "Downloading pending files"}
|
|
|
|
|
|
# ============================================================================
|
|
# IDENTITIES ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/identities")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_identities(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all identities"""
|
|
db = _get_db_adapter()
|
|
identities = db.get_identities()
|
|
return {"identities": identities}
|
|
|
|
|
|
@router.get("/identities/{identity_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_identity(
|
|
request: Request,
|
|
identity_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get single identity with linked creators"""
|
|
db = _get_db_adapter()
|
|
identity = db.get_identity(identity_id)
|
|
if not identity:
|
|
raise NotFoundError(f"Identity {identity_id} not found")
|
|
|
|
# Get linked creators
|
|
identity['creators'] = db.get_creators(identity_id=identity_id)
|
|
return identity
|
|
|
|
|
|
@router.post("/identities")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def create_identity(
|
|
request: Request,
|
|
body: CreateIdentityRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Create new identity"""
|
|
db = _get_db_adapter()
|
|
identity_id = db.create_identity(body.name, body.notes)
|
|
return {"id": identity_id, "name": body.name, "message": "Identity created"}
|
|
|
|
|
|
@router.put("/identities/{identity_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_identity(
|
|
request: Request,
|
|
identity_id: int,
|
|
body: UpdateIdentityRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update identity"""
|
|
db = _get_db_adapter()
|
|
updates = body.dict(exclude_none=True)
|
|
if not updates:
|
|
raise ValidationError("No updates provided")
|
|
|
|
success = db.update_identity(identity_id, updates)
|
|
if not success:
|
|
raise NotFoundError(f"Identity {identity_id} not found")
|
|
|
|
return message_response("Identity updated successfully")
|
|
|
|
|
|
@router.delete("/identities/{identity_id}")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def delete_identity(
|
|
request: Request,
|
|
identity_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete identity (unlinks creators but doesn't delete them)"""
|
|
db = _get_db_adapter()
|
|
success = db.delete_identity(identity_id)
|
|
if not success:
|
|
raise NotFoundError(f"Identity {identity_id} not found")
|
|
|
|
return message_response("Identity deleted successfully")
|
|
|
|
|
|
@router.post("/identities/{identity_id}/link")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def link_creator(
|
|
request: Request,
|
|
identity_id: int,
|
|
body: LinkCreatorRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Link creator to identity"""
|
|
db = _get_db_adapter()
|
|
success = db.link_creator_to_identity(body.creator_id, identity_id)
|
|
if not success:
|
|
raise NotFoundError("Creator or identity not found")
|
|
|
|
return message_response("Creator linked to identity")
|
|
|
|
|
|
@router.post("/identities/{identity_id}/unlink/{creator_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def unlink_creator(
|
|
request: Request,
|
|
identity_id: int,
|
|
creator_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Unlink creator from identity"""
|
|
db = _get_db_adapter()
|
|
success = db.unlink_creator_from_identity(creator_id)
|
|
return message_response("Creator unlinked from identity")
|
|
|
|
|
|
# ============================================================================
|
|
# TAG ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/tags")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_tags(
|
|
request: Request,
|
|
creator_ids: Optional[str] = Query(None, description="Comma-separated creator IDs to scope tags"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all tags, optionally scoped to specific creators"""
|
|
db = _get_db_adapter()
|
|
parsed_ids = [int(x) for x in creator_ids.split(',') if x.strip().isdigit()] if creator_ids else None
|
|
tags = db.get_tags(creator_ids=parsed_ids if parsed_ids else None)
|
|
return {"tags": tags}
|
|
|
|
|
|
@router.post("/tags")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def create_tag(
|
|
request: Request,
|
|
body: CreateTagRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Create a new tag"""
|
|
db = _get_db_adapter()
|
|
tag_id = db.create_tag(body.name, body.color, body.description)
|
|
if not tag_id:
|
|
raise ValidationError("Failed to create tag - name may already exist")
|
|
tag = db.get_tag(tag_id)
|
|
return {"tag": tag}
|
|
|
|
|
|
@router.put("/tags/{tag_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_tag(
|
|
request: Request,
|
|
tag_id: int,
|
|
body: UpdateTagRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update a tag"""
|
|
db = _get_db_adapter()
|
|
updates = body.model_dump(exclude_none=True)
|
|
if not updates:
|
|
raise ValidationError("No updates provided")
|
|
success = db.update_tag(tag_id, updates)
|
|
if not success:
|
|
raise NotFoundError(f"Tag {tag_id} not found")
|
|
tag = db.get_tag(tag_id)
|
|
return {"tag": tag}
|
|
|
|
|
|
@router.delete("/tags/{tag_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_tag(
|
|
request: Request,
|
|
tag_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete a tag"""
|
|
db = _get_db_adapter()
|
|
success = db.delete_tag(tag_id)
|
|
if not success:
|
|
raise NotFoundError(f"Tag {tag_id} not found")
|
|
return message_response("Tag deleted")
|
|
|
|
|
|
@router.post("/tags/add-to-posts")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def add_tags_to_posts(
|
|
request: Request,
|
|
body: TagPostsRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add tags to multiple posts"""
|
|
db = _get_db_adapter()
|
|
added = 0
|
|
for post_id in body.post_ids:
|
|
for tag_id in body.tag_ids:
|
|
if db.add_tag_to_post(post_id, tag_id):
|
|
added += 1
|
|
return {"added": added}
|
|
|
|
|
|
@router.post("/tags/remove-from-posts")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def remove_tags_from_posts(
|
|
request: Request,
|
|
body: TagPostsRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove tags from multiple posts"""
|
|
db = _get_db_adapter()
|
|
removed = 0
|
|
for post_id in body.post_ids:
|
|
for tag_id in body.tag_ids:
|
|
if db.remove_tag_from_post(post_id, tag_id):
|
|
removed += 1
|
|
return {"removed": removed}
|
|
|
|
|
|
@router.get("/posts/{post_id}/tags")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_post_tags(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get tags for a specific post"""
|
|
db = _get_db_adapter()
|
|
tags = db.get_post_tags(post_id)
|
|
return {"tags": tags}
|
|
|
|
|
|
@router.put("/posts/{post_id}/tags")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def set_post_tags(
|
|
request: Request,
|
|
post_id: int,
|
|
body: SetPostTagsRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Set tags for a post (replaces existing tags)"""
|
|
db = _get_db_adapter()
|
|
db.set_post_tags(post_id, body.tag_ids)
|
|
tags = db.get_post_tags(post_id)
|
|
return {"tags": tags}
|
|
|
|
|
|
class SetPostTaggedUsersRequest(BaseModel):
|
|
usernames: List[str]
|
|
|
|
|
|
@router.put("/posts/{post_id}/tagged-users")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def set_post_tagged_users(
|
|
request: Request,
|
|
post_id: int,
|
|
body: SetPostTaggedUsersRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Set tagged users for a post (replaces existing tagged users)"""
|
|
db = _get_db_adapter()
|
|
db.set_post_tagged_users(post_id, body.usernames)
|
|
tagged_users = db.get_post_tagged_users(post_id)
|
|
return {"tagged_users": tagged_users}
|
|
|
|
|
|
# ============================================================================
|
|
# CREATOR GROUPS
|
|
# ============================================================================
|
|
|
|
class CreatorGroupCreate(BaseModel):
|
|
name: str
|
|
description: Optional[str] = None
|
|
|
|
class CreatorGroupUpdate(BaseModel):
|
|
name: Optional[str] = None
|
|
description: Optional[str] = None
|
|
|
|
class CreatorGroupMemberAdd(BaseModel):
|
|
creator_id: int
|
|
|
|
class GroupMemberFilterUpdate(BaseModel):
|
|
filter_tagged_users: Optional[List[str]] = None
|
|
filter_tag_ids: Optional[List[int]] = None
|
|
|
|
|
|
@router.get("/creator-groups")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def list_creator_groups(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""List all creator groups"""
|
|
db = _get_db_adapter()
|
|
groups = db.get_creator_groups()
|
|
return {"groups": groups}
|
|
|
|
|
|
@router.get("/creator-groups/{group_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_creator_group(
|
|
request: Request,
|
|
group_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get a single creator group with members"""
|
|
db = _get_db_adapter()
|
|
group = db.get_creator_group(group_id)
|
|
if not group:
|
|
raise NotFoundError(f"Creator group {group_id} not found")
|
|
return group
|
|
|
|
|
|
@router.post("/creator-groups")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def create_creator_group(
|
|
request: Request,
|
|
body: CreatorGroupCreate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Create a new creator group"""
|
|
db = _get_db_adapter()
|
|
group = db.create_creator_group(body.name, body.description)
|
|
return {"group": group, "message": "Group created"}
|
|
|
|
|
|
@router.put("/creator-groups/{group_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_creator_group(
|
|
request: Request,
|
|
group_id: int,
|
|
body: CreatorGroupUpdate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update a creator group"""
|
|
db = _get_db_adapter()
|
|
success = db.update_creator_group(group_id, name=body.name, description=body.description)
|
|
if not success:
|
|
raise NotFoundError(f"Creator group {group_id} not found")
|
|
return {"success": True, "message": "Group updated"}
|
|
|
|
|
|
@router.delete("/creator-groups/{group_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_creator_group(
|
|
request: Request,
|
|
group_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete a creator group"""
|
|
db = _get_db_adapter()
|
|
success = db.delete_creator_group(group_id)
|
|
if not success:
|
|
raise NotFoundError(f"Creator group {group_id} not found")
|
|
return {"success": True, "message": "Group deleted"}
|
|
|
|
|
|
@router.post("/creator-groups/{group_id}/members")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def add_creator_to_group(
|
|
request: Request,
|
|
group_id: int,
|
|
body: CreatorGroupMemberAdd,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add a creator to a group"""
|
|
db = _get_db_adapter()
|
|
# Verify group exists
|
|
group = db.get_creator_group(group_id)
|
|
if not group:
|
|
raise NotFoundError(f"Creator group {group_id} not found")
|
|
db.add_creator_to_group(group_id, body.creator_id)
|
|
return {"success": True, "message": "Creator added to group"}
|
|
|
|
|
|
@router.delete("/creator-groups/{group_id}/members/{creator_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def remove_creator_from_group(
|
|
request: Request,
|
|
group_id: int,
|
|
creator_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove a creator from a group"""
|
|
db = _get_db_adapter()
|
|
db.remove_creator_from_group(group_id, creator_id)
|
|
return {"success": True, "message": "Creator removed from group"}
|
|
|
|
|
|
@router.put("/creator-groups/{group_id}/members/{creator_id}/filters")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def update_group_member_filters(
|
|
request: Request,
|
|
group_id: int,
|
|
creator_id: int,
|
|
body: GroupMemberFilterUpdate,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update per-member filter overrides for a group member"""
|
|
db = _get_db_adapter()
|
|
filter_tagged_users = json.dumps(body.filter_tagged_users) if body.filter_tagged_users else None
|
|
filter_tag_ids = json.dumps(body.filter_tag_ids) if body.filter_tag_ids else None
|
|
success = db.update_group_member_filters(group_id, creator_id, filter_tagged_users, filter_tag_ids)
|
|
if not success:
|
|
raise NotFoundError(f"Member not found in group")
|
|
return {"success": True, "message": "Member filters updated"}
|
|
|
|
|
|
# ============================================================================
|
|
# FEED ENDPOINTS
|
|
# ============================================================================
|
|
|
|
|
|
@router.get("/unviewed-count")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_unviewed_count(request: Request, current_user: Dict = Depends(get_current_user)):
|
|
"""Get count of unviewed posts."""
|
|
db = _get_db_adapter()
|
|
count = db.get_unviewed_posts_count()
|
|
return {"count": count}
|
|
|
|
|
|
@router.post("/mark-all-viewed")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def mark_all_viewed(request: Request, current_user: Dict = Depends(get_current_user)):
|
|
"""Mark all unviewed posts as viewed."""
|
|
db = _get_db_adapter()
|
|
updated = db.mark_all_posts_viewed()
|
|
return {"updated": updated}
|
|
|
|
|
|
@router.get("/tagged-users")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_tagged_users(
|
|
request: Request,
|
|
creator_ids: Optional[str] = Query(None, description="Comma-separated creator IDs to scope tagged users"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all distinct tagged usernames with post counts, optionally scoped to specific creators."""
|
|
db = _get_db_adapter()
|
|
parsed_ids = [int(x) for x in creator_ids.split(',') if x.strip().isdigit()] if creator_ids else None
|
|
return db.get_all_tagged_usernames(creator_ids=parsed_ids if parsed_ids else None)
|
|
|
|
|
|
@router.get("/content-types")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_content_types(
|
|
request: Request,
|
|
creator_ids: Optional[str] = Query(None, description="Comma-separated creator IDs to scope content types"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get distinct content types, optionally scoped to specific creators."""
|
|
db = _get_db_adapter()
|
|
parsed_ids = [int(x) for x in creator_ids.split(',') if x.strip().isdigit()] if creator_ids else None
|
|
return db.get_content_types(creator_ids=parsed_ids if parsed_ids else None)
|
|
|
|
|
|
@router.get("/feed")
|
|
@limiter.limit("300/minute")
|
|
@handle_exceptions
|
|
async def get_feed(
|
|
request: Request,
|
|
creator_id: Optional[int] = None,
|
|
creator_ids: Optional[str] = None, # Comma-separated creator IDs
|
|
creator_group_id: Optional[int] = None,
|
|
identity_id: Optional[int] = None,
|
|
service: Optional[str] = None,
|
|
platform: Optional[str] = None,
|
|
content_type: Optional[str] = None,
|
|
min_resolution: Optional[str] = None, # 720p, 1080p, 1440p, 4k
|
|
date_from: Optional[str] = None,
|
|
date_to: Optional[str] = None,
|
|
search: Optional[str] = None,
|
|
tag_ids: Optional[str] = None, # Comma-separated tag IDs
|
|
tagged_user: Optional[str] = None,
|
|
favorites_only: bool = False,
|
|
unviewed_only: bool = False,
|
|
downloaded_only: bool = False,
|
|
has_missing: bool = False,
|
|
missing_description: bool = False,
|
|
hide_empty: bool = True,
|
|
sort_by: str = "published_at",
|
|
sort_order: str = "desc",
|
|
shuffle: bool = False,
|
|
shuffle_seed: Optional[int] = None,
|
|
limit: int = Query(default=50, le=500),
|
|
offset: int = 0,
|
|
pinned_first: bool = True,
|
|
skip_pinned: bool = False,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get feed with filters"""
|
|
db = _get_db_adapter()
|
|
|
|
# Parse tag_ids if provided
|
|
parsed_tag_ids = None
|
|
if tag_ids:
|
|
try:
|
|
parsed_tag_ids = [int(t.strip()) for t in tag_ids.split(',') if t.strip()]
|
|
except ValueError:
|
|
pass
|
|
|
|
# Parse creator_ids if provided
|
|
parsed_creator_ids = None
|
|
if creator_ids:
|
|
try:
|
|
parsed_creator_ids = [int(c.strip()) for c in creator_ids.split(',') if c.strip()]
|
|
except ValueError:
|
|
pass
|
|
|
|
posts = db.get_posts(
|
|
creator_id=creator_id,
|
|
creator_ids=parsed_creator_ids,
|
|
creator_group_id=creator_group_id,
|
|
identity_id=identity_id,
|
|
service=service,
|
|
platform=platform,
|
|
content_type=content_type,
|
|
min_resolution=min_resolution,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
search=search,
|
|
tag_ids=parsed_tag_ids,
|
|
tagged_user=tagged_user,
|
|
favorites_only=favorites_only,
|
|
unviewed_only=unviewed_only,
|
|
downloaded_only=downloaded_only,
|
|
has_missing=has_missing,
|
|
missing_description=missing_description,
|
|
hide_empty=hide_empty,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
shuffle=shuffle,
|
|
shuffle_seed=shuffle_seed,
|
|
limit=limit,
|
|
offset=offset,
|
|
pinned_first=pinned_first,
|
|
skip_pinned=skip_pinned
|
|
)
|
|
|
|
# Get total count
|
|
filter_kwargs = dict(
|
|
creator_id=creator_id,
|
|
creator_ids=parsed_creator_ids,
|
|
creator_group_id=creator_group_id,
|
|
identity_id=identity_id,
|
|
service=service,
|
|
platform=platform,
|
|
content_type=content_type,
|
|
min_resolution=min_resolution,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
search=search,
|
|
tag_ids=parsed_tag_ids,
|
|
tagged_user=tagged_user,
|
|
favorites_only=favorites_only,
|
|
unviewed_only=unviewed_only,
|
|
downloaded_only=downloaded_only,
|
|
has_missing=has_missing,
|
|
missing_description=missing_description,
|
|
hide_empty=hide_empty,
|
|
skip_pinned=skip_pinned
|
|
)
|
|
total = db.get_posts_count(**filter_kwargs)
|
|
total_media = db.get_media_count(**filter_kwargs)
|
|
|
|
return {
|
|
"posts": posts,
|
|
"count": len(posts),
|
|
"total": total,
|
|
"total_media": total_media,
|
|
"limit": limit,
|
|
"offset": offset
|
|
}
|
|
|
|
|
|
@router.get("/feed/{post_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_post(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get single post with attachments"""
|
|
db = _get_db_adapter()
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
# Mark as viewed
|
|
db.mark_post_viewed(post_id)
|
|
|
|
return post
|
|
|
|
|
|
@router.post("/feed/{post_id}/viewed")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def toggle_viewed(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Toggle viewed status"""
|
|
db = _get_db_adapter()
|
|
is_viewed = db.toggle_post_viewed(post_id)
|
|
return {"is_viewed": is_viewed}
|
|
|
|
|
|
@router.post("/feed/{post_id}/favorite")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def toggle_favorite(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Toggle favorite status"""
|
|
db = _get_db_adapter()
|
|
is_favorited = db.toggle_post_favorite(post_id)
|
|
return {"is_favorited": is_favorited}
|
|
|
|
|
|
@router.post("/feed/{post_id}/download")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def download_post(
|
|
request: Request,
|
|
post_id: int,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Download all attachments for a post"""
|
|
db = _get_db_adapter()
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
background_tasks.add_task(_download_post_background, post_id)
|
|
return {"status": "queued", "post_id": post_id}
|
|
|
|
|
|
@router.put("/feed/{post_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_post(
|
|
request: Request,
|
|
post_id: int,
|
|
body: UpdatePostRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update post title, description, and/or date (with file relocation)"""
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
db = _get_db_adapter()
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
updates = body.model_dump(exclude_none=True)
|
|
if not updates:
|
|
raise ValidationError("No updates provided")
|
|
|
|
files_moved = False
|
|
new_local_path = None
|
|
|
|
# If date is changing and we have a local_path, move the files
|
|
if 'published_at' in updates and post.get('local_path'):
|
|
old_date = post.get('published_at', '')[:10] if post.get('published_at') else None
|
|
new_date = updates['published_at'][:10] if updates['published_at'] else None
|
|
|
|
if old_date and new_date and old_date != new_date:
|
|
old_path = Path(post['local_path'])
|
|
|
|
if old_path.exists():
|
|
# Build new path by replacing the date directory
|
|
# Path structure: /base/platform/creator/DATE/post_id/
|
|
path_parts = list(old_path.parts)
|
|
|
|
# Find and replace the date part (YYYY-MM-DD format)
|
|
for i, part in enumerate(path_parts):
|
|
if len(part) == 10 and part[4] == '-' and part[7] == '-':
|
|
try:
|
|
# Verify it's a date
|
|
int(part[:4]) # year
|
|
int(part[5:7]) # month
|
|
int(part[8:10]) # day
|
|
path_parts[i] = new_date
|
|
break
|
|
except ValueError:
|
|
pass
|
|
|
|
new_path = Path(*path_parts)
|
|
|
|
if new_path != old_path:
|
|
# Create parent directory and move
|
|
new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.move(str(old_path), str(new_path))
|
|
|
|
# Remove old empty parent directories
|
|
try:
|
|
old_path.parent.rmdir()
|
|
except OSError:
|
|
pass # Not empty, that's fine
|
|
|
|
new_local_path = str(new_path)
|
|
files_moved = True
|
|
|
|
# Update attachment paths
|
|
for att in post.get('attachments', []):
|
|
if att.get('local_path'):
|
|
old_att_path = att['local_path']
|
|
new_att_path = old_att_path.replace(str(old_path), str(new_path))
|
|
|
|
# Also rename attachment if name contains old date
|
|
if att.get('name') and old_date in att['name']:
|
|
new_name = att['name'].replace(old_date, new_date)
|
|
db.update_attachment(att['id'], {'name': new_name, 'local_path': new_att_path})
|
|
else:
|
|
db.update_attachment(att['id'], {'local_path': new_att_path})
|
|
|
|
# Add local_path update if files were moved
|
|
if new_local_path:
|
|
updates['local_path'] = new_local_path
|
|
|
|
success = db.update_post(post_id, updates)
|
|
if success:
|
|
result = {"status": "success", "updated": list(updates.keys())}
|
|
if files_moved:
|
|
result["files_moved"] = True
|
|
result["new_path"] = new_local_path
|
|
return result
|
|
return {"status": "failed"}
|
|
|
|
|
|
async def _refresh_fansly_post(db, post, post_id, creator):
|
|
"""Refresh a Fansly post by fetching fresh data from the API"""
|
|
from modules.paid_content.fansly_direct_client import FanslyDirectClient
|
|
service = db.get_service('fansly_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
raise ValidationError("Fansly auth token not configured")
|
|
|
|
client = FanslyDirectClient(
|
|
auth_token=service['session_cookie'],
|
|
log_callback=lambda msg, level='info': logger.log(msg, module='FanslyDirect', level=level)
|
|
)
|
|
|
|
try:
|
|
fresh_post = await client.get_single_post(post['post_id'], creator['creator_id'])
|
|
if not fresh_post:
|
|
raise NotFoundError("Post not found on Fansly")
|
|
|
|
# Update post metadata
|
|
post_updates = {}
|
|
if fresh_post.content and fresh_post.content != post.get('content'):
|
|
post_updates['content'] = fresh_post.content
|
|
if post_updates:
|
|
db.update_post(post_id, post_updates)
|
|
|
|
# Upsert attachments (creates missing ones, updates URLs for existing)
|
|
new_attachments = 0
|
|
for idx, attachment in enumerate(fresh_post.attachments):
|
|
att_data = attachment.to_dict()
|
|
att_data['attachment_index'] = idx
|
|
if attachment.download_url is None:
|
|
att_data['status'] = 'unavailable'
|
|
if db.upsert_attachment(post_id, att_data):
|
|
new_attachments += 1
|
|
|
|
# Remove Preview tag and add Full Length tag if content is now unlocked (keep PPV tag)
|
|
has_ppv = any(att.download_url is None for att in fresh_post.attachments)
|
|
if not has_ppv:
|
|
preview_tag = db.get_tag_by_slug('preview')
|
|
if preview_tag:
|
|
db.remove_tag_from_post(post_id, preview_tag['id'])
|
|
full_length_tag = db.get_tag_by_slug('full-length')
|
|
if full_length_tag:
|
|
db.add_tag_to_post(post_id, full_length_tag['id'])
|
|
|
|
# Download any new pending attachments in background
|
|
pending = db.get_pending_attachments_for_post(post_id)
|
|
if pending:
|
|
import asyncio
|
|
creator_id_val = creator['id']
|
|
|
|
async def _download_pending():
|
|
scraper = _get_scraper()
|
|
scraper.app_state = None # Disable app_state cancellation check
|
|
# Register a temporary active sync so activity_manager doesn't cancel
|
|
scraper._register_active_sync(creator_id_val, {
|
|
'username': creator['username'],
|
|
'platform': 'fansly',
|
|
'service': 'fansly_direct',
|
|
'status': 'Downloading refreshed post...',
|
|
'phase': 'downloading',
|
|
})
|
|
try:
|
|
await scraper.download_pending_for_creator(creator_id_val)
|
|
finally:
|
|
scraper._unregister_active_sync(creator_id_val)
|
|
await scraper.close()
|
|
|
|
asyncio.create_task(_download_pending())
|
|
return {
|
|
"status": "success",
|
|
"new_attachments": new_attachments,
|
|
"total_attachments": len(fresh_post.attachments),
|
|
"pending_download": len(pending) if pending else 0,
|
|
"has_ppv": has_ppv,
|
|
"updated": list(post_updates.keys()) if post_updates else []
|
|
}
|
|
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
async def _refresh_onlyfans_post(db, post, post_id, creator):
|
|
"""Refresh an OnlyFans post by fetching fresh data from the API"""
|
|
import json as _json
|
|
from modules.paid_content.onlyfans_client import OnlyFansClient
|
|
service = db.get_service('onlyfans_direct')
|
|
if not service or not service.get('session_cookie'):
|
|
raise ValidationError("OnlyFans credentials not configured")
|
|
|
|
try:
|
|
auth_config = _json.loads(service['session_cookie'])
|
|
except (_json.JSONDecodeError, TypeError):
|
|
raise ValidationError("OnlyFans credentials: invalid JSON")
|
|
|
|
client = OnlyFansClient(
|
|
auth_config=auth_config,
|
|
signing_url=auth_config.get('signing_url'),
|
|
log_callback=lambda msg, level='info': logger.log(msg, module='OnlyFansDirect', level=level)
|
|
)
|
|
|
|
try:
|
|
fresh_post = await client.get_single_post(post['post_id'])
|
|
if not fresh_post:
|
|
raise NotFoundError("Post not found on OnlyFans")
|
|
|
|
post_updates = {}
|
|
if fresh_post.content and fresh_post.content != post.get('content'):
|
|
post_updates['content'] = fresh_post.content
|
|
if post_updates:
|
|
db.update_post(post_id, post_updates)
|
|
|
|
new_attachments = 0
|
|
for idx, attachment in enumerate(fresh_post.attachments):
|
|
att_data = attachment.to_dict()
|
|
att_data['attachment_index'] = idx
|
|
if attachment.download_url is None:
|
|
att_data['status'] = 'unavailable'
|
|
if db.upsert_attachment(post_id, att_data):
|
|
new_attachments += 1
|
|
|
|
# Remove Preview tag and add Full Length tag if content is now unlocked (keep PPV tag)
|
|
has_ppv = any(att.download_url is None for att in fresh_post.attachments)
|
|
if not has_ppv:
|
|
preview_tag = db.get_tag_by_slug('preview')
|
|
if preview_tag:
|
|
db.remove_tag_from_post(post_id, preview_tag['id'])
|
|
full_length_tag = db.get_tag_by_slug('full-length')
|
|
if full_length_tag:
|
|
db.add_tag_to_post(post_id, full_length_tag['id'])
|
|
|
|
pending = db.get_pending_attachments_for_post(post_id)
|
|
if pending:
|
|
import asyncio
|
|
creator_id_val = creator['id']
|
|
|
|
async def _download_pending():
|
|
scraper = _get_scraper()
|
|
scraper.app_state = None
|
|
scraper._register_active_sync(creator_id_val, {
|
|
'username': creator['username'],
|
|
'platform': 'onlyfans',
|
|
'service': 'onlyfans_direct',
|
|
'status': 'Downloading refreshed post...',
|
|
'phase': 'downloading',
|
|
})
|
|
try:
|
|
await scraper.download_pending_for_creator(creator_id_val)
|
|
finally:
|
|
scraper._unregister_active_sync(creator_id_val)
|
|
await scraper.close()
|
|
|
|
asyncio.create_task(_download_pending())
|
|
return {
|
|
"status": "success",
|
|
"new_attachments": new_attachments,
|
|
"total_attachments": len(fresh_post.attachments),
|
|
"pending_download": len(pending) if pending else 0,
|
|
"has_ppv": has_ppv,
|
|
"updated": list(post_updates.keys()) if post_updates else []
|
|
}
|
|
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
@router.post("/feed/{post_id}/refresh")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def refresh_post_metadata(
|
|
request: Request,
|
|
post_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Refresh post metadata and attachments from source"""
|
|
db = _get_db_adapter()
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
raise NotFoundError(f"Post {post_id} not found")
|
|
|
|
creator = db.get_creator(post['creator_id'])
|
|
if not creator:
|
|
raise NotFoundError(f"Creator not found for post {post_id}")
|
|
|
|
# Handle Fansly Direct posts
|
|
if creator['service_id'] == 'fansly_direct':
|
|
return await _refresh_fansly_post(db, post, post_id, creator)
|
|
|
|
# Handle OnlyFans Direct posts
|
|
elif creator['service_id'] == 'onlyfans_direct':
|
|
return await _refresh_onlyfans_post(db, post, post_id, creator)
|
|
|
|
# Handle YouTube posts
|
|
elif creator['service_id'] == 'youtube':
|
|
from modules.paid_content import YouTubeClient
|
|
app_state = get_app_state()
|
|
client = YouTubeClient(unified_db=app_state.db)
|
|
|
|
video_id = post.get('post_id')
|
|
if not video_id:
|
|
raise ValidationError("No video ID found for post")
|
|
|
|
video_url = f"https://www.youtube.com/watch?v={video_id}"
|
|
|
|
# Fetch fresh metadata
|
|
try:
|
|
cmd = client._get_base_cmd() + [
|
|
'--skip-download',
|
|
'--no-warnings',
|
|
'-j',
|
|
video_url
|
|
]
|
|
|
|
import asyncio
|
|
result = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE
|
|
)
|
|
stdout, stderr = await result.communicate()
|
|
|
|
if result.returncode == 0:
|
|
import json
|
|
data = json.loads(stdout.decode('utf-8', errors='replace'))
|
|
updates = {
|
|
'title': data.get('title'),
|
|
'content': data.get('description'),
|
|
}
|
|
# Remove None values
|
|
updates = {k: v for k, v in updates.items() if v is not None}
|
|
|
|
if updates:
|
|
db.update_post(post_id, updates)
|
|
client.cleanup()
|
|
return {"status": "success", "updated": list(updates.keys())}
|
|
else:
|
|
client.cleanup()
|
|
return {"status": "no_changes"}
|
|
else:
|
|
client.cleanup()
|
|
error = stderr.decode('utf-8', errors='replace')[:200]
|
|
raise ValidationError(f"Failed to fetch metadata: {error}")
|
|
|
|
except Exception as e:
|
|
client.cleanup()
|
|
raise ValidationError(f"Failed to refresh: {str(e)}")
|
|
|
|
# Handle Coomer/Kemono posts
|
|
else:
|
|
from modules.paid_content import PaidContentAPIClient
|
|
app_state = get_app_state()
|
|
|
|
service = db.get_service(creator['service_id'])
|
|
client = PaidContentAPIClient(
|
|
creator['service_id'],
|
|
session_cookie=service.get('session_cookie') if service else None
|
|
)
|
|
|
|
try:
|
|
post_data = await client.get_post(
|
|
creator['platform'],
|
|
creator['creator_id'],
|
|
post['post_id']
|
|
)
|
|
await client.close()
|
|
|
|
if post_data:
|
|
updates = {
|
|
'title': post_data.get('title'),
|
|
'content': post_data.get('content'),
|
|
}
|
|
updates = {k: v for k, v in updates.items() if v is not None}
|
|
|
|
if updates:
|
|
db.update_post(post_id, updates)
|
|
return {"status": "success", "updated": list(updates.keys())}
|
|
else:
|
|
return {"status": "no_changes"}
|
|
else:
|
|
raise NotFoundError("Post not found on source")
|
|
|
|
except Exception as e:
|
|
await client.close()
|
|
raise ValidationError(f"Failed to refresh: {str(e)}")
|
|
|
|
|
|
# ============================================================================
|
|
# NOTIFICATIONS ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/notifications")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_notifications(
|
|
request: Request,
|
|
unread_only: bool = False,
|
|
limit: int = Query(default=50, le=100),
|
|
offset: int = 0,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get notification history"""
|
|
db = _get_db_adapter()
|
|
notifications = db.get_notifications(unread_only=unread_only, limit=limit, offset=offset)
|
|
unread_count = db.get_unread_notification_count()
|
|
return {"notifications": notifications, "count": len(notifications), "unread_count": unread_count}
|
|
|
|
|
|
@router.post("/notifications/mark-read")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def mark_notifications_read(
|
|
request: Request,
|
|
body: MarkNotificationsReadRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Mark notifications as read"""
|
|
db = _get_db_adapter()
|
|
|
|
if body.mark_all:
|
|
count = db.mark_all_notifications_read()
|
|
elif body.notification_ids:
|
|
count = db.mark_notifications_read(body.notification_ids)
|
|
else:
|
|
count = 0
|
|
|
|
return {"marked_count": count}
|
|
|
|
|
|
@router.delete("/notifications/{notification_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_notification(
|
|
request: Request,
|
|
notification_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete a notification"""
|
|
db = _get_db_adapter()
|
|
success = db.delete_notification(notification_id)
|
|
if not success:
|
|
raise NotFoundError(f"Notification {notification_id} not found")
|
|
return {"status": "success"}
|
|
|
|
|
|
# ============================================================================
|
|
# SETTINGS ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/settings")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_settings(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get settings"""
|
|
db = _get_db_adapter()
|
|
config = db.get_config()
|
|
return config
|
|
|
|
|
|
@router.put("/settings")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def update_settings(
|
|
request: Request,
|
|
body: PaidContentSettings,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update settings"""
|
|
db = _get_db_adapter()
|
|
|
|
updates = body.dict(exclude_none=True)
|
|
if updates:
|
|
# Convert booleans to integers for database storage
|
|
for key in ['organize_by_date', 'organize_by_post', 'download_embeds',
|
|
'notifications_enabled', 'push_notifications_enabled',
|
|
'perceptual_duplicate_detection', 'auto_retry_failed']:
|
|
if key in updates:
|
|
updates[key] = 1 if updates[key] else 0
|
|
|
|
db.update_config(updates)
|
|
|
|
return message_response("Settings updated successfully")
|
|
|
|
|
|
# ============================================================================
|
|
# IMPORT ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.post("/import/url")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def import_from_url(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
body: ImportUrlRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Import content from external file host URL"""
|
|
from modules.paid_content import FileHostDownloader
|
|
|
|
downloader = FileHostDownloader()
|
|
if not downloader.is_supported_url(body.url):
|
|
raise ValidationError(f"Unsupported URL. Supported hosts: {', '.join(downloader.get_supported_domains())}")
|
|
|
|
background_tasks.add_task(_import_url_background, body.url, body.creator_id)
|
|
return {"status": "queued", "url": body.url}
|
|
|
|
|
|
@router.post("/parse-filenames")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def parse_filenames(
|
|
request: Request,
|
|
body: ParseFilenamesRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Parse filenames to extract dates and metadata.
|
|
Supports Fansly snowflake IDs and embedded date formats.
|
|
"""
|
|
from modules.paid_content.filename_parser import parse_filenames as do_parse
|
|
|
|
analysis = do_parse(body.filenames)
|
|
|
|
# Convert datetimes to ISO strings for JSON
|
|
def dt_to_str(dt):
|
|
return dt.isoformat() if dt else None
|
|
|
|
return {
|
|
"files": [
|
|
{
|
|
**f,
|
|
"detected_date": dt_to_str(f["detected_date"]),
|
|
}
|
|
for f in analysis["files"]
|
|
],
|
|
"earliest_date": dt_to_str(analysis["earliest_date"]),
|
|
"latest_date": dt_to_str(analysis["latest_date"]),
|
|
"suggested_date": dt_to_str(analysis["suggested_date"]),
|
|
"has_dates": analysis["has_dates"],
|
|
}
|
|
|
|
|
|
@router.post("/upload")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def upload_manual_content(
|
|
request: Request,
|
|
files: List[UploadFile] = File(...),
|
|
creator_id: int = Form(...),
|
|
title: str = Form(""),
|
|
description: str = Form(""),
|
|
post_date: str = Form(None),
|
|
tags: str = Form(""), # Comma-separated tag IDs
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Upload files manually and add them to the feed.
|
|
Creates a new post for the specified creator with the uploaded files.
|
|
"""
|
|
import hashlib
|
|
import mimetypes
|
|
import uuid
|
|
|
|
db = _get_db_adapter()
|
|
|
|
# Validate creator exists
|
|
creator = db.get_creator(creator_id)
|
|
if not creator:
|
|
raise NotFoundError(f"Creator {creator_id} not found")
|
|
|
|
# Get config for base path
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/opt/immich/paid'))
|
|
|
|
# Parse post date or use current date
|
|
if post_date:
|
|
try:
|
|
# Validate date format (YYYY-MM-DD)
|
|
datetime.strptime(post_date, '%Y-%m-%d')
|
|
date_str = post_date
|
|
except ValueError:
|
|
date_str = datetime.now().strftime('%Y-%m-%d')
|
|
else:
|
|
date_str = datetime.now().strftime('%Y-%m-%d')
|
|
|
|
# Generate unique post ID
|
|
post_id_str = f"manual_{uuid.uuid4().hex[:12]}"
|
|
|
|
# Build destination directory
|
|
# Format: /base/platform/username/date/post_id/
|
|
dest_dir = base_path / creator['platform'] / creator['username'] / date_str / post_id_str
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# First pass: determine file types for title generation
|
|
image_exts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.heic']
|
|
video_exts = ['.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ts']
|
|
|
|
image_count = 0
|
|
video_count = 0
|
|
for upload_file in files:
|
|
ext = Path(upload_file.filename).suffix.lower() if upload_file.filename else ''
|
|
if not ext and upload_file.content_type:
|
|
ext = mimetypes.guess_extension(upload_file.content_type) or ''
|
|
if ext in image_exts:
|
|
image_count += 1
|
|
elif ext in video_exts:
|
|
video_count += 1
|
|
|
|
# Generate default title if not provided: "Jan 23, 2026 — 3 images, 1 video"
|
|
if not title.strip():
|
|
# Parse date for formatting
|
|
try:
|
|
date_obj = datetime.strptime(date_str, '%Y-%m-%d')
|
|
formatted_date = date_obj.strftime('%b %d, %Y') # "Jan 23, 2026"
|
|
except ValueError:
|
|
formatted_date = date_str
|
|
|
|
# Build content description
|
|
parts = []
|
|
if image_count > 0:
|
|
parts.append(f"{image_count} {'image' if image_count == 1 else 'images'}")
|
|
if video_count > 0:
|
|
parts.append(f"{video_count} {'video' if video_count == 1 else 'videos'}")
|
|
if not parts:
|
|
parts.append(f"{len(files)} {'file' if len(files) == 1 else 'files'}")
|
|
|
|
default_title = f"{formatted_date} — {', '.join(parts)}"
|
|
else:
|
|
default_title = title.strip()
|
|
|
|
# Create the post record
|
|
post_data = {
|
|
'post_id': post_id_str,
|
|
'title': default_title,
|
|
'content': description.strip(),
|
|
'published_at': f"{date_str}T12:00:00",
|
|
'added_at': datetime.now().isoformat(),
|
|
'has_attachments': 1 if files else 0,
|
|
'attachment_count': len(files),
|
|
'downloaded': 1, # Already downloaded since we're uploading
|
|
'download_date': datetime.now().isoformat(),
|
|
'local_path': str(dest_dir),
|
|
'metadata': '{"source": "manual_upload"}'
|
|
}
|
|
|
|
post_db_id = db.upsert_post(creator_id, post_data)
|
|
if not post_db_id:
|
|
raise ValidationError("Failed to create post record")
|
|
|
|
# Process each uploaded file
|
|
saved_files = []
|
|
for idx, upload_file in enumerate(files):
|
|
try:
|
|
# Generate filename: 001.ext, 002.ext, etc.
|
|
original_ext = Path(upload_file.filename).suffix.lower() if upload_file.filename else ''
|
|
if not original_ext:
|
|
# Try to guess from content type
|
|
content_type = upload_file.content_type or 'application/octet-stream'
|
|
original_ext = mimetypes.guess_extension(content_type) or '.bin'
|
|
|
|
filename = f"{(idx + 1):03d}{original_ext}"
|
|
file_path = dest_dir / filename
|
|
|
|
# Reset file position since we may have read it in first pass
|
|
await upload_file.seek(0)
|
|
|
|
# Read and save file
|
|
content = await upload_file.read()
|
|
with open(file_path, 'wb') as f:
|
|
f.write(content)
|
|
|
|
# Compute hash
|
|
file_hash = hashlib.sha256(content).hexdigest()
|
|
|
|
# Determine file type
|
|
file_type = 'other'
|
|
if original_ext in image_exts:
|
|
file_type = 'image'
|
|
elif original_ext in video_exts:
|
|
file_type = 'video'
|
|
|
|
# Get dimensions for images/videos if possible
|
|
width, height, duration = None, None, None
|
|
if file_type == 'image':
|
|
try:
|
|
from PIL import Image
|
|
with Image.open(file_path) as img:
|
|
width, height = img.size
|
|
except Exception:
|
|
pass
|
|
elif file_type == 'video':
|
|
try:
|
|
import subprocess
|
|
result = subprocess.run(
|
|
['ffprobe', '-v', 'quiet', '-print_format', 'json',
|
|
'-show_streams', '-select_streams', 'v:0', str(file_path)],
|
|
capture_output=True, text=True, timeout=10
|
|
)
|
|
if result.returncode == 0:
|
|
import json
|
|
data = json.loads(result.stdout)
|
|
if data.get('streams'):
|
|
stream = data['streams'][0]
|
|
width = stream.get('width')
|
|
height = stream.get('height')
|
|
duration_str = stream.get('duration')
|
|
if duration_str:
|
|
duration = int(float(duration_str))
|
|
except Exception:
|
|
pass
|
|
|
|
# Create attachment record
|
|
attachment_data = {
|
|
'attachment_index': idx,
|
|
'name': upload_file.filename or filename,
|
|
'file_type': file_type,
|
|
'extension': original_ext.lstrip('.'),
|
|
'server_path': f'manual://{filename}',
|
|
'download_url': '',
|
|
'file_size': len(content),
|
|
'width': width,
|
|
'height': height,
|
|
'duration': duration,
|
|
'status': 'completed',
|
|
'local_path': str(file_path),
|
|
'local_filename': filename,
|
|
'file_hash': file_hash,
|
|
'downloaded_at': datetime.now().isoformat()
|
|
}
|
|
|
|
att_id = db.upsert_attachment(post_db_id, attachment_data)
|
|
saved_files.append({
|
|
'id': att_id,
|
|
'filename': filename,
|
|
'original_name': upload_file.filename,
|
|
'size': len(content),
|
|
'file_type': file_type,
|
|
'path': str(file_path)
|
|
})
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to save uploaded file {upload_file.filename}: {e}", module="PaidContent")
|
|
# Continue with other files
|
|
|
|
# Update creator stats
|
|
db.update_creator(creator_id, {
|
|
'post_count': db.get_creator_post_count(creator_id),
|
|
'downloaded_count': db.get_creator_downloaded_count(creator_id)
|
|
})
|
|
|
|
# Apply tags if provided
|
|
applied_tags = []
|
|
if tags.strip():
|
|
try:
|
|
tag_ids = [int(t.strip()) for t in tags.split(',') if t.strip()]
|
|
for tag_id in tag_ids:
|
|
if db.add_tag_to_post(post_db_id, tag_id):
|
|
applied_tags.append(tag_id)
|
|
except ValueError:
|
|
pass # Invalid tag IDs, skip
|
|
|
|
return {
|
|
"status": "success",
|
|
"post_id": post_db_id,
|
|
"post_api_id": post_id_str,
|
|
"creator": creator['username'],
|
|
"files_saved": len(saved_files),
|
|
"files": saved_files,
|
|
"local_path": str(dest_dir),
|
|
"tags_applied": applied_tags
|
|
}
|
|
|
|
|
|
@router.get("/attachments/{attachment_id}")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_attachment(
|
|
request: Request,
|
|
attachment_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get a single attachment by ID"""
|
|
db = _get_db_adapter()
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
# Remove binary fields that can't be JSON serialized
|
|
result = {k: v for k, v in attachment.items() if k != 'thumbnail_data'}
|
|
return result
|
|
|
|
|
|
@router.post("/attachments/{attachment_id}/import-url")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def import_url_to_attachment(
|
|
request: Request,
|
|
attachment_id: int,
|
|
body: ImportToAttachmentRequest,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Import a file from a supported URL and link it to an existing attachment.
|
|
Downloads the file in background, renames it appropriately, and updates the database.
|
|
Returns immediately - poll GET /attachments/{id} for status.
|
|
"""
|
|
db = _get_db_adapter()
|
|
|
|
# Get the attachment details
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
|
|
# Get post and creator info for building the path
|
|
post = db.get_post(attachment['post_id'])
|
|
if not post:
|
|
raise NotFoundError(f"Post for attachment {attachment_id} not found")
|
|
|
|
creator = db.get_creator(post['creator_id'])
|
|
if not creator:
|
|
raise NotFoundError(f"Creator for attachment {attachment_id} not found")
|
|
|
|
# Mark as downloading immediately
|
|
db.update_attachment_status(attachment_id, 'downloading')
|
|
|
|
# Run download in background (ws_manager will be fetched inside the task)
|
|
background_tasks.add_task(
|
|
_import_url_to_attachment_background,
|
|
attachment_id, body.url, attachment, post, creator
|
|
)
|
|
|
|
return {
|
|
"status": "downloading",
|
|
"attachment_id": attachment_id,
|
|
"message": "Download started in background. Poll GET /attachments/{id} for status."
|
|
}
|
|
|
|
|
|
async def _import_url_to_attachment_background(
|
|
attachment_id: int,
|
|
url: str,
|
|
attachment: Dict,
|
|
post: Dict,
|
|
creator: Dict
|
|
):
|
|
"""Background task for importing URL to attachment with progress tracking"""
|
|
import aiohttp
|
|
import hashlib
|
|
from modules.paid_content import FileHostDownloader
|
|
from core.dependencies import get_app_state
|
|
|
|
db = _get_db_adapter()
|
|
|
|
# Helper to get ws_manager (fetched fresh each time to ensure it's initialized)
|
|
def get_ws_manager():
|
|
app_state = get_app_state()
|
|
return getattr(app_state, 'websocket_manager', None)
|
|
|
|
# Build the destination path
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/opt/immich/paid'))
|
|
|
|
# Get post date for path
|
|
post_date = (post.get('published_at') or '')[:10] or 'unknown-date'
|
|
post_id_str = post.get('post_id', str(post['id']))
|
|
|
|
# Build path: /base/platform/username/date/post_id/
|
|
dest_dir = base_path / creator['platform'] / creator['username'] / post_date / post_id_str
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Determine filename - use attachment index + extension
|
|
ext = attachment.get('extension') or Path(attachment.get('name', '')).suffix or '.bin'
|
|
if not ext.startswith('.'):
|
|
ext = '.' + ext
|
|
filename = f"{(attachment.get('attachment_index', 0) + 1):03d}{ext}"
|
|
dest_path = dest_dir / filename
|
|
|
|
try:
|
|
ws_manager = get_ws_manager()
|
|
logger.info(f"Background download started for attachment {attachment_id}, ws_manager={'present' if ws_manager else 'MISSING'}")
|
|
|
|
# Progress callback to update database AND broadcast via WebSocket
|
|
def update_progress(downloaded: int, total: int, filename: str):
|
|
pct = int(downloaded * 100 / total) if total > 0 else 0
|
|
progress_msg = f"Downloading: {pct}% ({downloaded // (1024*1024)}MB / {total // (1024*1024)}MB)"
|
|
|
|
# Update database
|
|
db.update_attachment(attachment_id, {
|
|
'file_size': downloaded,
|
|
'error_message': progress_msg
|
|
})
|
|
|
|
# Broadcast via WebSocket for real-time UI updates
|
|
ws = get_ws_manager()
|
|
if ws:
|
|
try:
|
|
ws.broadcast_sync({
|
|
'type': 'attachment_download_progress',
|
|
'data': {
|
|
'attachment_id': attachment_id,
|
|
'status': 'downloading',
|
|
'progress': pct,
|
|
'downloaded': downloaded,
|
|
'total': total,
|
|
'message': progress_msg
|
|
}
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"WebSocket broadcast failed: {e}")
|
|
|
|
downloader = FileHostDownloader(progress_callback=update_progress)
|
|
|
|
# Download the file
|
|
if downloader.is_supported_url(url):
|
|
# Use file host downloader for supported hosts (handles CDN fallback, etc.)
|
|
result = await downloader.download_url(url, dest_dir)
|
|
if not result['success'] or not result['files']:
|
|
db.update_attachment_status(attachment_id, 'failed', error_message=result.get('error', 'Unknown error'))
|
|
logger.error(f"Import failed for attachment {attachment_id}: {result.get('error')}")
|
|
return
|
|
|
|
# Rename the downloaded file to the correct name
|
|
downloaded_path = Path(result['files'][0])
|
|
if downloaded_path != dest_path:
|
|
if dest_path.exists():
|
|
dest_path.unlink()
|
|
downloaded_path.rename(dest_path)
|
|
else:
|
|
# Direct download for other URLs with progress
|
|
timeout = aiohttp.ClientTimeout(total=3600) # 1 hour for large files
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
}
|
|
async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
|
|
async with session.get(url) as resp:
|
|
if resp.status != 200:
|
|
db.update_attachment_status(attachment_id, 'failed', error_message=f"HTTP {resp.status}")
|
|
logger.error(f"Import failed for attachment {attachment_id}: HTTP {resp.status}")
|
|
return
|
|
|
|
total_size = int(resp.headers.get('content-length', 0))
|
|
downloaded = 0
|
|
last_update_pct = 0
|
|
|
|
with open(dest_path, 'wb') as f:
|
|
async for chunk in resp.content.iter_chunked(65536): # 64KB chunks
|
|
f.write(chunk)
|
|
downloaded += len(chunk)
|
|
|
|
# Update progress in DB every 5%
|
|
if total_size > 0:
|
|
pct = int(downloaded * 100 / total_size)
|
|
if pct >= last_update_pct + 5:
|
|
db.update_attachment(attachment_id, {
|
|
'file_size': downloaded,
|
|
'error_message': f"Downloading: {pct}% ({downloaded // (1024*1024)}MB / {total_size // (1024*1024)}MB)"
|
|
})
|
|
last_update_pct = pct
|
|
|
|
# Compute file hash
|
|
sha256 = hashlib.sha256()
|
|
with open(dest_path, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(65536), b''):
|
|
sha256.update(chunk)
|
|
file_hash = sha256.hexdigest()
|
|
|
|
# Get file size
|
|
file_size = dest_path.stat().st_size
|
|
|
|
# Update database with success
|
|
db.update_attachment_status(
|
|
attachment_id,
|
|
'completed',
|
|
local_path=str(dest_path),
|
|
local_filename=filename,
|
|
file_hash=file_hash,
|
|
file_size=file_size,
|
|
downloaded_at=now_iso8601(),
|
|
error_message=None
|
|
)
|
|
logger.info(f"Import completed for attachment {attachment_id}: {file_size // (1024*1024)}MB")
|
|
|
|
# Broadcast completion via WebSocket
|
|
ws = get_ws_manager()
|
|
if ws:
|
|
ws.broadcast_sync({
|
|
'type': 'attachment_download_progress',
|
|
'data': {
|
|
'attachment_id': attachment_id,
|
|
'status': 'completed',
|
|
'progress': 100,
|
|
'file_size': file_size,
|
|
'message': 'Download complete'
|
|
}
|
|
})
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to import URL to attachment {attachment_id}: {e}")
|
|
db.update_attachment_status(attachment_id, 'failed', error_message=str(e))
|
|
|
|
|
|
@router.post("/attachments/{attachment_id}/upload-file")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def upload_file_to_attachment(
|
|
request: Request,
|
|
attachment_id: int,
|
|
file: UploadFile = File(...),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Upload a file directly to an existing attachment.
|
|
Saves the file to the post's directory and updates the attachment record.
|
|
"""
|
|
import shutil
|
|
import hashlib
|
|
|
|
db = _get_db_adapter()
|
|
|
|
# Get the attachment details
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError(f"Attachment {attachment_id} not found")
|
|
|
|
# Get post and creator info for building the path
|
|
post = db.get_post(attachment['post_id'])
|
|
if not post:
|
|
raise NotFoundError(f"Post for attachment {attachment_id} not found")
|
|
|
|
creator = db.get_creator(post['creator_id'])
|
|
if not creator:
|
|
raise NotFoundError(f"Creator for attachment {attachment_id} not found")
|
|
|
|
# Build the destination path
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/opt/immich/paid'))
|
|
|
|
# Get post date for path
|
|
post_date = (post.get('published_at') or '')[:10] or 'unknown-date'
|
|
post_id_str = post.get('post_id', str(post['id']))
|
|
|
|
# Build path: /base/platform/username/date/post_id/
|
|
dest_dir = base_path / creator['platform'] / creator['username'] / post_date / post_id_str
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Determine filename - use attachment index + extension from uploaded file
|
|
original_name = file.filename or 'uploaded_file'
|
|
ext = Path(original_name).suffix or '.bin'
|
|
if not ext.startswith('.'):
|
|
ext = '.' + ext
|
|
filename = f"{(attachment.get('attachment_index', 0) + 1):03d}{ext}"
|
|
dest_path = dest_dir / filename
|
|
|
|
try:
|
|
# Save file
|
|
with open(dest_path, 'wb') as f:
|
|
shutil.copyfileobj(file.file, f)
|
|
|
|
# Get file info
|
|
file_size = dest_path.stat().st_size
|
|
|
|
# Calculate file hash
|
|
hash_md5 = hashlib.md5()
|
|
with open(dest_path, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(8192), b''):
|
|
hash_md5.update(chunk)
|
|
file_hash = hash_md5.hexdigest()
|
|
|
|
# Determine file type
|
|
file_type = 'unknown'
|
|
ext_lower = ext.lower().lstrip('.')
|
|
image_exts = {'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff', 'heic', 'heif', 'avif'}
|
|
video_exts = {'mp4', 'mov', 'avi', 'mkv', 'webm', 'm4v', 'wmv', 'flv'}
|
|
audio_exts = {'mp3', 'wav', 'flac', 'aac', 'm4a', 'ogg'}
|
|
|
|
if ext_lower in image_exts:
|
|
file_type = 'image'
|
|
elif ext_lower in video_exts:
|
|
file_type = 'video'
|
|
elif ext_lower in audio_exts:
|
|
file_type = 'audio'
|
|
|
|
# Extract dimensions/duration for images and videos
|
|
width, height, duration = None, None, None
|
|
if file_type == 'image':
|
|
try:
|
|
from PIL import Image
|
|
with Image.open(dest_path) as img:
|
|
width, height = img.size
|
|
except Exception:
|
|
pass
|
|
elif file_type == 'video':
|
|
try:
|
|
import subprocess as _sp
|
|
result = _sp.run(
|
|
['ffprobe', '-v', 'quiet', '-print_format', 'json',
|
|
'-show_streams', '-select_streams', 'v:0', str(dest_path)],
|
|
capture_output=True, text=True, timeout=30
|
|
)
|
|
if result.returncode == 0:
|
|
probe = json.loads(result.stdout)
|
|
if probe.get('streams'):
|
|
stream = probe['streams'][0]
|
|
width = stream.get('width')
|
|
height = stream.get('height')
|
|
duration_str = stream.get('duration')
|
|
if duration_str:
|
|
duration = int(float(duration_str))
|
|
except Exception:
|
|
pass
|
|
|
|
# Update attachment record
|
|
db.update_attachment_status(
|
|
attachment_id,
|
|
'completed',
|
|
local_path=str(dest_path),
|
|
local_filename=filename,
|
|
file_hash=file_hash,
|
|
file_size=file_size,
|
|
downloaded_at=now_iso8601(),
|
|
error_message=None
|
|
)
|
|
|
|
# Also update file_type and dimensions if we detected them
|
|
update_fields = {}
|
|
if file_type != 'unknown':
|
|
update_fields['file_type'] = file_type
|
|
update_fields['extension'] = ext_lower
|
|
if width is not None:
|
|
update_fields['width'] = width
|
|
if height is not None:
|
|
update_fields['height'] = height
|
|
if duration is not None:
|
|
update_fields['duration'] = duration
|
|
if update_fields:
|
|
db.update_attachment(attachment_id, update_fields)
|
|
|
|
# Invalidate stale thumbnail caches so they regenerate from the new file
|
|
db.update_attachment(attachment_id, {'thumbnail_data': None})
|
|
for thumb_size in ('small', 'medium', 'large', 'native'):
|
|
stale_thumb = Path(f"/opt/media-downloader/cache/thumbnails/{thumb_size}/{attachment_id}.jpg")
|
|
if stale_thumb.exists():
|
|
stale_thumb.unlink()
|
|
|
|
logger.info(f"File uploaded to attachment {attachment_id}: {filename} ({file_size // (1024*1024)}MB)")
|
|
|
|
return {
|
|
"status": "completed",
|
|
"attachment_id": attachment_id,
|
|
"filename": filename,
|
|
"file_type": file_type,
|
|
"file_size": file_size,
|
|
"width": width,
|
|
"height": height,
|
|
"duration": duration,
|
|
"local_path": str(dest_path)
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to upload file to attachment {attachment_id}: {e}")
|
|
db.update_attachment_status(attachment_id, 'failed', error_message=str(e))
|
|
raise ValidationError(f"Upload failed: {str(e)}")
|
|
|
|
|
|
# ============================================================================
|
|
# RECYCLE BIN ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/recycle-bin")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_recycle_bin(
|
|
request: Request,
|
|
item_type: Optional[str] = None,
|
|
search: Optional[str] = None,
|
|
platform: Optional[str] = None,
|
|
creator: Optional[str] = None,
|
|
content_type: Optional[str] = None,
|
|
limit: int = Query(default=50, le=100),
|
|
offset: int = 0,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get items in recycle bin"""
|
|
db = _get_db_adapter()
|
|
items = db.get_recycle_bin_items(
|
|
item_type=item_type,
|
|
search=search,
|
|
platform=platform,
|
|
creator=creator,
|
|
content_type=content_type,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
return {"items": items, "count": len(items)}
|
|
|
|
|
|
@router.post("/recycle-bin/{item_id}/restore")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def restore_from_recycle_bin(
|
|
request: Request,
|
|
item_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Restore item from recycle bin"""
|
|
db = _get_db_adapter()
|
|
success = db.restore_from_recycle_bin(item_id)
|
|
if not success:
|
|
raise NotFoundError(f"Item {item_id} not found in recycle bin")
|
|
|
|
return message_response("Item restored successfully")
|
|
|
|
|
|
@router.delete("/recycle-bin")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def empty_recycle_bin(
|
|
request: Request,
|
|
item_type: Optional[str] = None,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Empty recycle bin"""
|
|
db = _get_db_adapter()
|
|
count = db.empty_recycle_bin(item_type=item_type)
|
|
return {"deleted_count": count}
|
|
|
|
|
|
class RecycleBinBatchRequest(BaseModel):
|
|
item_ids: List[int]
|
|
|
|
|
|
@router.post("/recycle-bin/restore-batch")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def restore_from_recycle_bin_batch(
|
|
request: Request,
|
|
body: RecycleBinBatchRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Restore multiple items from recycle bin"""
|
|
db = _get_db_adapter()
|
|
restored_count = 0
|
|
for item_id in body.item_ids:
|
|
if db.restore_from_recycle_bin(item_id):
|
|
restored_count += 1
|
|
return {"restored_count": restored_count}
|
|
|
|
|
|
@router.post("/recycle-bin/delete-permanently")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def delete_permanently(
|
|
request: Request,
|
|
body: RecycleBinBatchRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Permanently delete items from recycle bin"""
|
|
db = _get_db_adapter()
|
|
deleted_count = 0
|
|
for item_id in body.item_ids:
|
|
if db.permanently_delete_recycle_item(item_id):
|
|
deleted_count += 1
|
|
return {"deleted_count": deleted_count}
|
|
|
|
|
|
# ============================================================================
|
|
# BACKGROUND TASKS
|
|
# ============================================================================
|
|
|
|
async def _sync_creator_background(creator_id: int, download: bool = True, force_backfill: bool = False):
|
|
"""Background task for creator sync"""
|
|
logger.info(f"Starting background sync for creator {creator_id}", module="PaidContent")
|
|
try:
|
|
scraper = _get_scraper()
|
|
logger.info(f"Scraper created, app_state={scraper.app_state is not None}", module="PaidContent")
|
|
await scraper.sync_creator(creator_id, download=download, force_backfill=force_backfill)
|
|
await scraper.close()
|
|
logger.info(f"Background sync completed for creator {creator_id}", module="PaidContent")
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Background sync failed for creator {creator_id}: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
|
|
|
|
async def _sync_all_creators_background():
|
|
"""Background task for syncing all creators"""
|
|
logger.info("Starting background sync for all creators", module="PaidContent")
|
|
try:
|
|
scraper = _get_scraper()
|
|
logger.info(f"Scraper created for sync_all, app_state={scraper.app_state is not None}", module="PaidContent")
|
|
await scraper.sync_all_creators()
|
|
await scraper.close()
|
|
logger.info("Background sync all completed", module="PaidContent")
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Background sync all failed: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
|
|
|
|
async def _sync_service_background(creator_ids: List[int]):
|
|
"""Background task for syncing all creators in a service"""
|
|
logger.info(f"Starting background sync for {len(creator_ids)} creators in service", module="PaidContent")
|
|
try:
|
|
scraper = _get_scraper()
|
|
for creator_id in creator_ids:
|
|
try:
|
|
await scraper.sync_creator(creator_id)
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Sync failed for creator {creator_id} in service sync: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
await scraper.close()
|
|
logger.info(f"Background service sync completed for {len(creator_ids)} creators", module="PaidContent")
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Background service sync failed: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
|
|
|
|
async def sync_paid_content_all(from_scheduler: bool = False, download: bool = True):
|
|
"""
|
|
Standalone sync function for all paid content creators.
|
|
|
|
This function creates its own database connection and doesn't rely on app_state,
|
|
making it callable from the scheduler (which runs in a separate process).
|
|
|
|
Args:
|
|
from_scheduler: If True, send push notifications for new content.
|
|
Only the scheduler should set this to True.
|
|
download: If True, download attachments after syncing posts.
|
|
"""
|
|
from modules.unified_database import UnifiedDatabase
|
|
from modules.paid_content import PaidContentScraper
|
|
from modules.activity_status import get_activity_manager
|
|
|
|
logger.info(f"Starting paid content sync (from_scheduler={from_scheduler}, download={download})", module="PaidContent")
|
|
|
|
# Create standalone database connection (works across processes)
|
|
db = UnifiedDatabase()
|
|
activity_manager = get_activity_manager(db)
|
|
|
|
# Initialize notifier if this is a scheduled sync
|
|
notifier = None
|
|
if from_scheduler:
|
|
try:
|
|
from modules.settings_manager import SettingsManager
|
|
from modules.pushover_notifier import create_notifier_from_config
|
|
|
|
settings_manager = SettingsManager(str(db.db_path))
|
|
config = settings_manager.get_all()
|
|
notifier = create_notifier_from_config(config, unified_db=db)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to initialize notifier: {e}", module="PaidContent")
|
|
|
|
try:
|
|
# Create scraper without app_state (works across processes)
|
|
scraper = PaidContentScraper(
|
|
unified_db=db,
|
|
notifier=notifier,
|
|
websocket_manager=None, # No websocket in scheduler process
|
|
app_state=None
|
|
)
|
|
|
|
# Run sync
|
|
results = await scraper.sync_all_creators(scheduled=from_scheduler, download=download)
|
|
|
|
# Count results
|
|
total_posts = sum(r.new_posts for r in results.values() if r.success)
|
|
total_files = sum(r.new_attachments for r in results.values() if r.success)
|
|
failed_count = sum(1 for r in results.values() if not r.success)
|
|
|
|
logger.info(f"Paid content sync complete: {total_posts} new posts, {total_files} new files, {failed_count} failures", module="PaidContent")
|
|
|
|
await scraper.close()
|
|
|
|
# Auto-recheck quality for flagged Fansly attachments
|
|
try:
|
|
await _auto_quality_recheck_background(unified_db=db)
|
|
except Exception as e:
|
|
logger.error(f"Auto quality recheck after scheduled sync failed: {e}", module="PaidContent")
|
|
|
|
return {
|
|
'success': True,
|
|
'new_posts': total_posts,
|
|
'new_files': total_files,
|
|
'failed_creators': failed_count
|
|
}
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
logger.error(f"Paid content sync failed: {e}\n{traceback.format_exc()}", module="PaidContent")
|
|
return {
|
|
'success': False,
|
|
'error': str(e)
|
|
}
|
|
|
|
|
|
async def _retry_downloads_background(attachment_ids: List[int]):
|
|
"""Background task for retrying downloads"""
|
|
try:
|
|
scraper = _get_scraper()
|
|
await scraper.retry_failed_downloads(attachment_ids)
|
|
await scraper.close()
|
|
except Exception as e:
|
|
logger.error(f"Background retry failed: {e}", module="PaidContent")
|
|
|
|
|
|
async def _download_post_background(post_id: int):
|
|
"""Background task for downloading a post"""
|
|
try:
|
|
db = _get_db_adapter()
|
|
post = db.get_post(post_id)
|
|
if not post:
|
|
return
|
|
|
|
scraper = _get_scraper()
|
|
await scraper.download_pending_for_creator(post['creator_id'])
|
|
await scraper.close()
|
|
except Exception as e:
|
|
logger.error(f"Background download failed for post {post_id}: {e}", module="PaidContent")
|
|
|
|
|
|
# ============================================================================
|
|
# FILE SERVING
|
|
# ============================================================================
|
|
|
|
@router.get("/files/serve")
|
|
@limiter.limit("1000/minute")
|
|
@handle_exceptions
|
|
async def serve_file(
|
|
request: Request,
|
|
path: str,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Serve a downloaded file with byte-range support for video streaming"""
|
|
file_path = Path(path)
|
|
|
|
# Security: ensure path is within allowed directories
|
|
db = _get_db_adapter()
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/paid-content'))
|
|
|
|
# Resolve to absolute path and check it's within base_path
|
|
try:
|
|
resolved_path = file_path.resolve()
|
|
resolved_base = base_path.resolve()
|
|
|
|
if not str(resolved_path).startswith(str(resolved_base)):
|
|
raise ValidationError("Access denied: path outside allowed directory")
|
|
except Exception:
|
|
raise ValidationError("Invalid path")
|
|
|
|
if not file_path.exists():
|
|
raise NotFoundError(f"File not found")
|
|
|
|
# Determine media type
|
|
suffix = file_path.suffix.lower()
|
|
media_types = {
|
|
'.jpg': 'image/jpeg',
|
|
'.jpeg': 'image/jpeg',
|
|
'.png': 'image/png',
|
|
'.gif': 'image/gif',
|
|
'.webp': 'image/webp',
|
|
'.mp4': 'video/mp4',
|
|
'.webm': 'video/webm',
|
|
'.mov': 'video/quicktime',
|
|
'.avi': 'video/x-msvideo',
|
|
'.mkv': 'video/x-matroska',
|
|
}
|
|
media_type = media_types.get(suffix, 'application/octet-stream')
|
|
|
|
# Get file size
|
|
file_size = file_path.stat().st_size
|
|
|
|
# Check for Range header (for video streaming)
|
|
range_header = request.headers.get('range')
|
|
if range_header and media_type.startswith('video/'):
|
|
# Parse range header: "bytes=start-end"
|
|
try:
|
|
range_spec = range_header.replace('bytes=', '')
|
|
if '-' in range_spec:
|
|
parts = range_spec.split('-')
|
|
start = int(parts[0]) if parts[0] else 0
|
|
end = int(parts[1]) if parts[1] else file_size - 1
|
|
else:
|
|
start = int(range_spec)
|
|
end = file_size - 1
|
|
|
|
# Clamp values
|
|
start = max(0, start)
|
|
end = min(end, file_size - 1)
|
|
content_length = end - start + 1
|
|
|
|
# Stream the requested range (sync generator runs in threadpool,
|
|
# keeping the async event loop free for other requests)
|
|
def stream_range():
|
|
with open(file_path, 'rb') as f:
|
|
f.seek(start)
|
|
remaining = content_length
|
|
chunk_size = 2 * 1024 * 1024 # 2MB chunks
|
|
while remaining > 0:
|
|
read_size = min(chunk_size, remaining)
|
|
data = f.read(read_size)
|
|
if not data:
|
|
break
|
|
remaining -= len(data)
|
|
yield data
|
|
|
|
from starlette.responses import StreamingResponse
|
|
return StreamingResponse(
|
|
stream_range(),
|
|
status_code=206,
|
|
media_type=media_type,
|
|
headers={
|
|
'Content-Range': f'bytes {start}-{end}/{file_size}',
|
|
'Accept-Ranges': 'bytes',
|
|
'Content-Length': str(content_length),
|
|
}
|
|
)
|
|
except (ValueError, IndexError):
|
|
pass # Fall through to regular file response
|
|
|
|
# Regular file response for non-range requests
|
|
return FileResponse(
|
|
path=str(file_path),
|
|
media_type=media_type,
|
|
filename=file_path.name,
|
|
headers={
|
|
'Accept-Ranges': 'bytes',
|
|
'Cache-Control': 'private, max-age=86400', # Cache for 24 hours
|
|
}
|
|
)
|
|
|
|
|
|
# HLS streaming cache directory
|
|
import subprocess
|
|
import hashlib
|
|
import tempfile
|
|
import os
|
|
import aiohttp
|
|
from io import BytesIO
|
|
from PIL import Image
|
|
|
|
|
|
async def _download_youtube_poster(video_id: str, max_size: tuple = None, crop_portrait: bool = False) -> Optional[bytes]:
|
|
"""Download YouTube poster image at the best available resolution.
|
|
If max_size is provided, resize to fit. If None (native), return full resolution.
|
|
If crop_portrait is True, center-crop the 16:9 image to 9:16 for Shorts."""
|
|
thumbnail_urls = [
|
|
f"https://i.ytimg.com/vi/{video_id}/maxresdefault.jpg", # 1280x720
|
|
f"https://i.ytimg.com/vi/{video_id}/sddefault.jpg", # 640x480
|
|
f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg", # 480x360
|
|
]
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
for url in thumbnail_urls:
|
|
try:
|
|
async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp:
|
|
if resp.status == 200:
|
|
image_data = await resp.read()
|
|
if len(image_data) < 1000:
|
|
continue # Placeholder image
|
|
with Image.open(BytesIO(image_data)) as img:
|
|
if img.mode != 'RGB':
|
|
img = img.convert('RGB')
|
|
if crop_portrait:
|
|
# Center-crop 16:9 to 9:16
|
|
w, h = img.size
|
|
target_w = int(h * 9 / 16)
|
|
left = (w - target_w) // 2
|
|
img = img.crop((left, 0, left + target_w, h))
|
|
if max_size:
|
|
img.thumbnail(max_size, Image.Resampling.LANCZOS)
|
|
buffer = BytesIO()
|
|
img.save(buffer, format='JPEG', quality=90, optimize=True)
|
|
return buffer.getvalue()
|
|
except Exception:
|
|
continue
|
|
except Exception as e:
|
|
logger.warning(f"Failed to download YouTube poster for {video_id}: {e}")
|
|
return None
|
|
|
|
HLS_CACHE_DIR = Path(tempfile.gettempdir()) / "paid_content_hls_cache"
|
|
HLS_CACHE_DIR.mkdir(exist_ok=True)
|
|
|
|
|
|
def _get_hls_cache_path(file_path: Path) -> Path:
|
|
"""Get cache directory for HLS segments of a video file."""
|
|
# Use hash of file path + mtime for cache key
|
|
cache_key = hashlib.md5(f"{file_path}:{file_path.stat().st_mtime}".encode()).hexdigest()
|
|
return HLS_CACHE_DIR / cache_key
|
|
|
|
|
|
def _generate_hls_playlist(file_path: Path, cache_path: Path) -> bool:
|
|
"""Generate HLS playlist and segments using FFmpeg."""
|
|
cache_path.mkdir(parents=True, exist_ok=True)
|
|
playlist_path = cache_path / "playlist.m3u8"
|
|
|
|
# If playlist already exists, skip generation
|
|
if playlist_path.exists():
|
|
return True
|
|
|
|
try:
|
|
# Probe video codec to pick segment type
|
|
probe_cmd = ['ffprobe', '-v', 'error', '-select_streams', 'v:0',
|
|
'-show_entries', 'stream=codec_name', '-of', 'csv=p=0',
|
|
str(file_path)]
|
|
probe = subprocess.run(probe_cmd, capture_output=True, timeout=30)
|
|
video_codec = probe.stdout.decode().strip()
|
|
|
|
# MPEG-TS only supports H.264/H.265. For AV1/VP9, use fMP4 segments
|
|
# which browsers can play natively via HLS.
|
|
if video_codec in ('h264', 'hevc'):
|
|
seg_type = 'mpegts'
|
|
seg_ext = 'ts'
|
|
else:
|
|
seg_type = 'fmp4'
|
|
seg_ext = 'mp4'
|
|
|
|
cmd = [
|
|
'ffmpeg',
|
|
'-fflags', '+genpts+igndts',
|
|
'-avoid_negative_ts', 'make_zero',
|
|
'-i', str(file_path),
|
|
'-c:v', 'copy',
|
|
'-c:a', 'aac', '-b:a', '128k',
|
|
'-async', '1',
|
|
'-hls_time', '4',
|
|
'-hls_list_size', '0',
|
|
'-hls_segment_type', seg_type,
|
|
'-hls_segment_filename', str(cache_path / f'segment_%03d.{seg_ext}'),
|
|
'-f', 'hls',
|
|
str(playlist_path),
|
|
'-y',
|
|
'-loglevel', 'error'
|
|
]
|
|
|
|
result = subprocess.run(cmd, capture_output=True, timeout=300)
|
|
if result.returncode != 0:
|
|
logger.error(f"FFmpeg HLS error: {result.stderr.decode()}")
|
|
return False
|
|
|
|
return playlist_path.exists()
|
|
except subprocess.TimeoutExpired:
|
|
logger.error("FFmpeg HLS generation timed out")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"HLS generation error: {e}")
|
|
return False
|
|
|
|
|
|
@router.get("/files/hls/{attachment_id}/playlist.m3u8")
|
|
@limiter.limit("100/minute")
|
|
@handle_exceptions
|
|
async def get_hls_playlist(
|
|
request: Request,
|
|
attachment_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get HLS playlist for video streaming. Generates on first request."""
|
|
db = _get_db_adapter()
|
|
|
|
# Get attachment
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError("Attachment not found")
|
|
|
|
if not attachment.get('local_path'):
|
|
raise NotFoundError("File not downloaded")
|
|
|
|
file_path = Path(attachment['local_path'])
|
|
if not file_path.exists():
|
|
raise NotFoundError("File not found on disk")
|
|
|
|
# Get cache path and generate HLS if needed
|
|
cache_path = _get_hls_cache_path(file_path)
|
|
|
|
# Run generation in thread pool to not block
|
|
import asyncio
|
|
loop = asyncio.get_event_loop()
|
|
success = await loop.run_in_executor(None, _generate_hls_playlist, file_path, cache_path)
|
|
|
|
if not success:
|
|
raise ValidationError("Failed to generate HLS stream")
|
|
|
|
playlist_path = cache_path / "playlist.m3u8"
|
|
|
|
# Read and modify playlist to use our segment endpoint
|
|
playlist_content = playlist_path.read_text()
|
|
|
|
# Replace segment filenames with our API endpoint (handles both .ts and .mp4)
|
|
import re
|
|
playlist_content = re.sub(
|
|
r'segment_(\d+)\.(ts|mp4)',
|
|
f'/api/paid-content/files/hls/{attachment_id}/segment_\\1.\\2',
|
|
playlist_content
|
|
)
|
|
# Also rewrite init segment for fMP4
|
|
playlist_content = playlist_content.replace(
|
|
'init.mp4',
|
|
f'/api/paid-content/files/hls/{attachment_id}/init.mp4'
|
|
)
|
|
|
|
from starlette.responses import Response
|
|
return Response(
|
|
content=playlist_content,
|
|
media_type='application/vnd.apple.mpegurl',
|
|
headers={
|
|
'Cache-Control': 'no-cache', # Don't cache playlist (segments can change)
|
|
}
|
|
)
|
|
|
|
|
|
@router.get("/files/hls/{attachment_id}/init.mp4")
|
|
@limiter.limit("1000/minute")
|
|
@handle_exceptions
|
|
async def get_hls_init_segment(
|
|
request: Request,
|
|
attachment_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get fMP4 init segment."""
|
|
db = _get_db_adapter()
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment or not attachment.get('local_path'):
|
|
raise NotFoundError("Attachment not found")
|
|
file_path = Path(attachment['local_path'])
|
|
if not file_path.exists():
|
|
raise NotFoundError("File not found on disk")
|
|
cache_path = _get_hls_cache_path(file_path)
|
|
init_path = cache_path / "init.mp4"
|
|
if not init_path.exists():
|
|
raise NotFoundError("Init segment not found")
|
|
return FileResponse(
|
|
path=str(init_path),
|
|
media_type='video/mp4',
|
|
headers={'Cache-Control': 'public, max-age=86400'},
|
|
)
|
|
|
|
|
|
@router.get("/files/hls/{attachment_id}/segment_{segment_num}.{ext}")
|
|
@limiter.limit("1000/minute")
|
|
@handle_exceptions
|
|
async def get_hls_segment(
|
|
request: Request,
|
|
attachment_id: int,
|
|
segment_num: str,
|
|
ext: str,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get HLS segment file (.ts or .mp4)."""
|
|
if ext not in ('ts', 'mp4'):
|
|
raise NotFoundError("Invalid segment type")
|
|
|
|
db = _get_db_adapter()
|
|
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError("Attachment not found")
|
|
|
|
if not attachment.get('local_path'):
|
|
raise NotFoundError("File not downloaded")
|
|
|
|
file_path = Path(attachment['local_path'])
|
|
if not file_path.exists():
|
|
raise NotFoundError("File not found on disk")
|
|
|
|
cache_path = _get_hls_cache_path(file_path)
|
|
segment_path = cache_path / f"segment_{segment_num}.{ext}"
|
|
|
|
if not segment_path.exists():
|
|
raise NotFoundError("Segment not found - playlist may need regeneration")
|
|
|
|
media_type = 'video/mp4' if ext == 'mp4' else 'video/mp2t'
|
|
return FileResponse(
|
|
path=str(segment_path),
|
|
media_type=media_type,
|
|
headers={
|
|
'Cache-Control': 'public, max-age=86400',
|
|
}
|
|
)
|
|
|
|
|
|
@router.get("/files/thumbnail/{attachment_id}")
|
|
@limiter.limit("1000/minute")
|
|
@handle_exceptions
|
|
async def get_attachment_thumbnail(
|
|
request: Request,
|
|
attachment_id: int,
|
|
size: str = "small",
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get cached thumbnail for an attachment, generating if needed.
|
|
|
|
Size options:
|
|
- small: 300x300 (default, cached in DB)
|
|
- medium: 600x600 (file cached)
|
|
- large: 800x800 (file cached)
|
|
- native: original video resolution (file cached, for video posters)
|
|
"""
|
|
# Size presets (native=None tells ffmpeg to skip scaling)
|
|
SIZE_PRESETS = {
|
|
"small": (300, 300),
|
|
"medium": (600, 600),
|
|
"large": (800, 800),
|
|
"native": None,
|
|
}
|
|
max_size = SIZE_PRESETS.get(size, SIZE_PRESETS["small"])
|
|
use_db_cache = (size == "small") # Only cache small thumbnails in DB
|
|
|
|
db = _get_db_adapter()
|
|
thumbnail_data = None
|
|
|
|
# File cache directory for larger thumbnails
|
|
cache_dir = Path("/opt/media-downloader/cache/thumbnails") / size
|
|
cache_file = cache_dir / f"{attachment_id}.jpg"
|
|
|
|
# Try to get cached thumbnail
|
|
if use_db_cache:
|
|
thumbnail_data = db.get_attachment_thumbnail(attachment_id)
|
|
elif cache_file.exists():
|
|
# Read from file cache for medium/large
|
|
thumbnail_data = cache_file.read_bytes()
|
|
|
|
# If no cached thumbnail, generate it
|
|
if not thumbnail_data:
|
|
attachment = db.get_attachment(attachment_id)
|
|
if not attachment:
|
|
raise NotFoundError("Attachment not found")
|
|
|
|
# For attachments without a local_path (duplicates, message copies), find the original by file_hash
|
|
if not attachment.get('local_path') and attachment.get('file_hash'):
|
|
original = db.find_attachment_by_hash(attachment['file_hash'], exclude_id=attachment_id)
|
|
if original and original.get('local_path'):
|
|
attachment = original
|
|
|
|
if not attachment.get('local_path'):
|
|
raise NotFoundError("Attachment not found")
|
|
|
|
file_path = Path(attachment['local_path'])
|
|
if not file_path.exists():
|
|
raise NotFoundError("File not found")
|
|
|
|
file_type = attachment.get('file_type', '')
|
|
if file_type not in ('image', 'video'):
|
|
raise NotFoundError("Thumbnail not available for this file type")
|
|
|
|
# For YouTube videos, download the official poster image instead of extracting a frame
|
|
# For portrait/Shorts, center-crop the 16:9 poster to 9:16
|
|
is_youtube = 'youtube' in (attachment.get('local_path') or '')
|
|
is_portrait = (attachment.get('height') or 0) > (attachment.get('width') or 1)
|
|
video_id = attachment.get('post_api_id') if is_youtube else None
|
|
if video_id and file_type == 'video':
|
|
thumbnail_data = await _download_youtube_poster(video_id, max_size, crop_portrait=is_portrait)
|
|
|
|
# Fall back to generating from the video/image file
|
|
if not thumbnail_data:
|
|
from modules.paid_content.scraper import PaidContentScraper
|
|
app_state = get_app_state()
|
|
scraper = PaidContentScraper(app_state.db)
|
|
thumbnail_data = scraper._generate_thumbnail(file_path, file_type, max_size=max_size)
|
|
|
|
# Cache the thumbnail
|
|
if thumbnail_data:
|
|
if use_db_cache:
|
|
db.update_attachment_status(attachment_id, attachment['status'],
|
|
thumbnail_data=thumbnail_data)
|
|
else:
|
|
# File cache for medium/large
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
cache_file.write_bytes(thumbnail_data)
|
|
|
|
if not thumbnail_data:
|
|
raise NotFoundError("Failed to generate thumbnail")
|
|
|
|
# Longer cache for larger sizes since they're more expensive to generate
|
|
cache_duration = 604800 if size in ("medium", "large") else 86400 # 7 days vs 1 day
|
|
|
|
# Generate ETag from cache file mtime for cache busting
|
|
etag = None
|
|
if cache_file.exists():
|
|
etag = f'"{attachment_id}-{int(cache_file.stat().st_mtime)}"'
|
|
|
|
headers = {
|
|
"Cache-Control": f"public, max-age={cache_duration}",
|
|
}
|
|
if etag:
|
|
headers["ETag"] = etag
|
|
|
|
return Response(
|
|
content=thumbnail_data,
|
|
media_type="image/jpeg",
|
|
headers=headers
|
|
)
|
|
|
|
|
|
@router.post("/files/thumbnails/backfill")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def backfill_thumbnails(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
batch_size: int = Query(default=50, le=200),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start thumbnail backfill for existing attachments"""
|
|
db = _get_db_adapter()
|
|
missing_count = db.count_attachments_missing_thumbnails()
|
|
|
|
if missing_count == 0:
|
|
return {"status": "complete", "message": "All attachments have thumbnails"}
|
|
|
|
# Start background task
|
|
background_tasks.add_task(_backfill_thumbnails_background, batch_size)
|
|
|
|
return {
|
|
"status": "started",
|
|
"missing_thumbnails": missing_count,
|
|
"batch_size": batch_size
|
|
}
|
|
|
|
|
|
@router.get("/thumbnail")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_thumbnail_by_path(
|
|
request: Request,
|
|
file_path: str = Query(..., description="Full path to the file"),
|
|
size: str = Query(default="small", regex="^(small|medium|large)$"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get thumbnail for a file by its path (for notifications page)"""
|
|
from pathlib import Path
|
|
|
|
path = Path(file_path)
|
|
if not path.exists():
|
|
raise HTTPException(status_code=404, detail="File not found")
|
|
|
|
# Determine file type from extension
|
|
ext = path.suffix.lower()
|
|
if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp']:
|
|
file_type = 'image'
|
|
elif ext in ['.mp4', '.mov', '.webm', '.avi', '.mkv', '.m4v']:
|
|
file_type = 'video'
|
|
else:
|
|
raise HTTPException(status_code=400, detail="Unsupported file type")
|
|
|
|
# Size mapping
|
|
size_map = {"small": (200, 200), "medium": (400, 400), "large": (800, 800)}
|
|
max_size = size_map.get(size, (200, 200))
|
|
|
|
# Generate thumbnail
|
|
scraper = _get_scraper()
|
|
thumbnail_data = scraper._generate_thumbnail(path, file_type, max_size=max_size)
|
|
await scraper.close()
|
|
|
|
if not thumbnail_data:
|
|
raise HTTPException(status_code=500, detail="Failed to generate thumbnail")
|
|
|
|
return Response(
|
|
content=thumbnail_data,
|
|
media_type="image/jpeg",
|
|
headers={"Cache-Control": "public, max-age=86400"}
|
|
)
|
|
|
|
|
|
@router.get("/preview")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_preview_by_path(
|
|
request: Request,
|
|
file_path: str = Query(..., description="Full path to the file"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Serve a file for preview (for notifications lightbox)"""
|
|
from pathlib import Path
|
|
|
|
path = Path(file_path)
|
|
if not path.exists():
|
|
raise HTTPException(status_code=404, detail="File not found")
|
|
|
|
# Determine media type from extension
|
|
ext = path.suffix.lower()
|
|
media_types = {
|
|
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
|
|
'.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
|
|
'.mp4': 'video/mp4', '.mov': 'video/quicktime', '.webm': 'video/webm',
|
|
'.avi': 'video/x-msvideo', '.mkv': 'video/x-matroska', '.m4v': 'video/mp4'
|
|
}
|
|
media_type = media_types.get(ext, 'application/octet-stream')
|
|
|
|
return FileResponse(
|
|
path=str(path),
|
|
media_type=media_type,
|
|
headers={"Cache-Control": "public, max-age=3600"}
|
|
)
|
|
|
|
|
|
@router.get("/files/thumbnails/status")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_thumbnail_backfill_status(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get thumbnail backfill status"""
|
|
db = _get_db_adapter()
|
|
missing_count = db.count_attachments_missing_thumbnails()
|
|
|
|
return {
|
|
"missing_thumbnails": missing_count,
|
|
"status": "complete" if missing_count == 0 else "pending"
|
|
}
|
|
|
|
|
|
@router.post("/files/thumbnails/backfill-large")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def backfill_large_thumbnails(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
batch_size: int = Query(default=50, le=200),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start large thumbnail backfill for feed view optimization"""
|
|
db = _get_db_adapter()
|
|
|
|
# Count attachments that don't have large thumbnails cached
|
|
cache_dir = Path("/opt/media-downloader/cache/thumbnails/large")
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
existing_cache = set(int(f.stem) for f in cache_dir.glob("*.jpg"))
|
|
|
|
# Get all completed image/video attachments
|
|
with db.db.get_connection() as conn:
|
|
cursor = conn.execute(
|
|
"SELECT id FROM paid_content_attachments WHERE status='completed' AND file_type IN ('image', 'video') AND local_path IS NOT NULL"
|
|
)
|
|
all_ids = set(row[0] for row in cursor.fetchall())
|
|
|
|
missing_ids = all_ids - existing_cache
|
|
missing_count = len(missing_ids)
|
|
|
|
if missing_count == 0:
|
|
return {"status": "complete", "message": "All large thumbnails cached", "total": len(all_ids)}
|
|
|
|
# Start background task
|
|
background_tasks.add_task(_backfill_large_thumbnails_background, list(missing_ids), batch_size)
|
|
|
|
return {
|
|
"status": "started",
|
|
"missing_thumbnails": missing_count,
|
|
"total_attachments": len(all_ids),
|
|
"batch_size": batch_size
|
|
}
|
|
|
|
|
|
async def _backfill_large_thumbnails_background(attachment_ids: list, batch_size: int = 50):
|
|
"""Background task to generate large thumbnails for feed view"""
|
|
from modules.paid_content.scraper import PaidContentScraper
|
|
|
|
app_state = get_app_state()
|
|
db = _get_db_adapter()
|
|
scraper = PaidContentScraper(app_state.db)
|
|
cache_dir = Path("/opt/media-downloader/cache/thumbnails/large")
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
processed = 0
|
|
for att_id in attachment_ids:
|
|
try:
|
|
cache_file = cache_dir / f"{att_id}.jpg"
|
|
if cache_file.exists():
|
|
continue
|
|
|
|
attachment = db.get_attachment(att_id)
|
|
if not attachment or not attachment.get('local_path'):
|
|
continue
|
|
|
|
file_path = Path(attachment['local_path'])
|
|
if not file_path.exists():
|
|
continue
|
|
|
|
file_type = attachment.get('file_type', '')
|
|
if file_type not in ('image', 'video'):
|
|
continue
|
|
|
|
thumbnail_data = scraper._generate_thumbnail(file_path, file_type, max_size=(800, 800))
|
|
if thumbnail_data:
|
|
cache_file.write_bytes(thumbnail_data)
|
|
processed += 1
|
|
except Exception as e:
|
|
logger.error(f"Error generating large thumbnail for {att_id}: {e}", module="Thumbnail")
|
|
|
|
logger.info(f"Large thumbnail backfill complete: generated {processed} thumbnails", module="Thumbnail")
|
|
|
|
|
|
# ============================================================================
|
|
# DIMENSION BACKFILL ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.post("/files/dimensions/backfill")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def backfill_dimensions(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
batch_size: int = Query(default=100, le=500),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start dimension backfill for existing attachments that have files but no resolution info"""
|
|
db = _get_db_adapter()
|
|
missing_count = db.count_attachments_missing_dimensions()
|
|
|
|
if missing_count == 0:
|
|
return {"status": "complete", "message": "All attachments have dimensions"}
|
|
|
|
# Start background task
|
|
background_tasks.add_task(_backfill_dimensions_background, batch_size)
|
|
|
|
return {
|
|
"status": "started",
|
|
"missing_dimensions": missing_count,
|
|
"batch_size": batch_size
|
|
}
|
|
|
|
|
|
@router.get("/files/dimensions/status")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_dimension_backfill_status(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get dimension backfill status"""
|
|
db = _get_db_adapter()
|
|
missing_count = db.count_attachments_missing_dimensions()
|
|
|
|
return {
|
|
"missing_dimensions": missing_count,
|
|
"status": "complete" if missing_count == 0 else "pending"
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# CONTENT BACKFILL ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.post("/posts/content/backfill")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def backfill_truncated_content(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
creator_id: Optional[int] = Query(default=None, description="Limit to specific creator"),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""
|
|
Re-fetch full content for posts with truncated descriptions from Coomer/Kemono.
|
|
The list endpoint returns a truncated 'substring' field. This backfill fetches
|
|
the full content from the individual post endpoint.
|
|
"""
|
|
db = _get_db_adapter()
|
|
|
|
# Count posts with missing/truncated content
|
|
with db.unified_db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
query = """
|
|
SELECT COUNT(*) FROM paid_content_posts p
|
|
JOIN paid_content_creators c ON p.creator_id = c.id
|
|
WHERE c.service_id IN ('coomer', 'kemono')
|
|
AND (
|
|
(p.content IS NULL OR p.content = '') AND p.title IS NOT NULL AND p.title != ''
|
|
OR p.content LIKE '%..'
|
|
OR p.title LIKE '%..'
|
|
)
|
|
"""
|
|
params = []
|
|
if creator_id:
|
|
query += " AND p.creator_id = ?"
|
|
params.append(creator_id)
|
|
cursor.execute(query, params)
|
|
truncated_count = cursor.fetchone()[0]
|
|
|
|
if truncated_count == 0:
|
|
return {"status": "complete", "message": "No truncated posts found"}
|
|
|
|
background_tasks.add_task(_backfill_content_background, creator_id)
|
|
|
|
return {
|
|
"status": "started",
|
|
"truncated_posts": truncated_count,
|
|
"creator_id": creator_id,
|
|
}
|
|
|
|
|
|
@router.get("/posts/content/backfill/status")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_content_backfill_status(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get count of posts with truncated content"""
|
|
db = _get_db_adapter()
|
|
|
|
with db.unified_db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM paid_content_posts p
|
|
JOIN paid_content_creators c ON p.creator_id = c.id
|
|
WHERE c.service_id IN ('coomer', 'kemono')
|
|
AND (
|
|
(p.content IS NULL OR p.content = '') AND p.title IS NOT NULL AND p.title != ''
|
|
OR p.content LIKE '%..'
|
|
OR p.title LIKE '%..'
|
|
)
|
|
""")
|
|
truncated_count = cursor.fetchone()[0]
|
|
|
|
return {
|
|
"truncated_posts": truncated_count,
|
|
"status": "complete" if truncated_count == 0 else "pending",
|
|
}
|
|
|
|
|
|
async def _backfill_content_background(creator_id: int = None):
|
|
"""Background task to re-fetch full content for truncated posts"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
result = await scraper.backfill_truncated_content(creator_id=creator_id)
|
|
logger.info(
|
|
f"Content backfill complete: {result['updated']} updated, "
|
|
f"{result['failed']} failed, {result['total_truncated']} total",
|
|
module="PaidContent"
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Content backfill failed: {e}", module="PaidContent")
|
|
finally:
|
|
await scraper.close()
|
|
|
|
|
|
# ============================================================================
|
|
# BACKGROUND TASKS
|
|
# ============================================================================
|
|
|
|
async def _backfill_thumbnails_background(batch_size: int = 50):
|
|
"""Background task to generate thumbnails for existing attachments"""
|
|
from modules.paid_content.scraper import PaidContentScraper
|
|
|
|
db = _get_db_adapter()
|
|
app_state = get_app_state()
|
|
scraper = PaidContentScraper(app_state.db)
|
|
|
|
processed = 0
|
|
while True:
|
|
attachments = db.get_attachments_missing_thumbnails(limit=batch_size)
|
|
if not attachments:
|
|
break
|
|
|
|
for att in attachments:
|
|
try:
|
|
file_path = Path(att['local_path'])
|
|
if not file_path.exists():
|
|
continue
|
|
|
|
thumbnail_data = scraper._generate_thumbnail(file_path, att['file_type'])
|
|
if thumbnail_data:
|
|
db.update_attachment_status(att['id'], 'completed',
|
|
thumbnail_data=thumbnail_data)
|
|
processed += 1
|
|
except Exception as e:
|
|
logger.error(f"Error generating thumbnail for {att['id']}: {e}", module="Thumbnail")
|
|
|
|
# Small delay between batches to not overload the system
|
|
await asyncio.sleep(0.5)
|
|
|
|
logger.info(f"Thumbnail backfill complete: generated {processed} thumbnails", module="Thumbnail")
|
|
|
|
|
|
async def _backfill_dimensions_background(batch_size: int = 100):
|
|
"""Background task to extract dimensions for existing attachments"""
|
|
from modules.paid_content.scraper import PaidContentScraper
|
|
|
|
db = _get_db_adapter()
|
|
app_state = get_app_state()
|
|
scraper = PaidContentScraper(app_state.db)
|
|
|
|
processed = 0
|
|
errors = 0
|
|
while True:
|
|
attachments = db.get_attachments_missing_dimensions(limit=batch_size)
|
|
if not attachments:
|
|
break
|
|
|
|
for att in attachments:
|
|
try:
|
|
local_path = att.get('local_path')
|
|
if not local_path:
|
|
continue
|
|
|
|
file_path = Path(local_path)
|
|
if not file_path.exists():
|
|
continue
|
|
|
|
file_type = att.get('file_type', '')
|
|
if file_type not in ('image', 'video'):
|
|
continue
|
|
|
|
width, height, duration = scraper._extract_dimensions(file_path, file_type)
|
|
if width and height:
|
|
db.update_attachment_status(att['id'], att['status'],
|
|
width=width,
|
|
height=height,
|
|
duration=duration)
|
|
processed += 1
|
|
if processed % 50 == 0:
|
|
logger.info(f"Dimension backfill progress: {processed} attachments processed", module="Dimensions")
|
|
except Exception as e:
|
|
logger.error(f"Error extracting dimensions for {att['id']}: {e}", module="Dimensions")
|
|
errors += 1
|
|
|
|
# Small delay between batches to not overload the system
|
|
await asyncio.sleep(0.2)
|
|
|
|
logger.info(f"Dimension backfill complete: processed {processed} attachments, {errors} errors", module="Dimensions")
|
|
|
|
|
|
async def _import_url_background(url: str, creator_id: Optional[int]):
|
|
"""Background task for importing from URL"""
|
|
try:
|
|
from modules.paid_content import FileHostDownloader
|
|
from pathlib import Path
|
|
|
|
db = _get_db_adapter()
|
|
config = db.get_config()
|
|
base_path = Path(config.get('base_download_path', '/paid-content'))
|
|
|
|
# Determine save directory
|
|
if creator_id:
|
|
creator = db.get_creator(creator_id)
|
|
if creator:
|
|
save_dir = base_path / creator['platform'] / creator['username'] / 'imports'
|
|
else:
|
|
save_dir = base_path / 'imports'
|
|
else:
|
|
save_dir = base_path / 'imports'
|
|
|
|
downloader = FileHostDownloader()
|
|
result = await downloader.download_url(url, save_dir)
|
|
|
|
# Create notification
|
|
if result['success']:
|
|
db.create_notification(
|
|
notification_type='download_complete',
|
|
title="Import Complete",
|
|
message=f"Downloaded {len(result['files'])} files from {url[:50]}...",
|
|
file_count=len(result['files'])
|
|
)
|
|
else:
|
|
db.create_notification(
|
|
notification_type='error',
|
|
title="Import Failed",
|
|
message=f"Failed to import from {url[:50]}...: {result.get('error')}"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Background import failed for {url}: {e}", module="PaidContent")
|
|
|
|
|
|
# Allowed domain suffixes for image proxy (security measure)
|
|
# Uses suffix matching so CDN subdomain variations are automatically covered
|
|
ALLOWED_IMAGE_DOMAIN_SUFFIXES = [
|
|
'.coomer.st',
|
|
'.coomer.party',
|
|
'.coomer.su',
|
|
'.kemono.st',
|
|
'.kemono.party',
|
|
'.kemono.su',
|
|
'.onlyfans.com',
|
|
'.fansly.com',
|
|
'.tiktokcdn-us.com',
|
|
'.tiktokcdn.com',
|
|
'.xhcdn.com',
|
|
'.imginn.com',
|
|
'.cdninstagram.com',
|
|
]
|
|
# Exact-match domains (for bare domains without subdomain)
|
|
ALLOWED_IMAGE_DOMAINS_EXACT = [
|
|
'public.onlyfans.com',
|
|
]
|
|
|
|
|
|
def _is_domain_allowed(domain: str) -> bool:
|
|
"""Check if a domain is allowed for image proxying."""
|
|
domain = domain.lower()
|
|
if domain in ALLOWED_IMAGE_DOMAINS_EXACT:
|
|
return True
|
|
return any(domain.endswith(suffix) for suffix in ALLOWED_IMAGE_DOMAIN_SUFFIXES)
|
|
|
|
|
|
@router.get("/proxy/image")
|
|
@handle_exceptions
|
|
async def proxy_image(url: str, current_user: Dict = Depends(get_current_user)):
|
|
"""
|
|
Proxy external images to bypass CORS restrictions.
|
|
Only allows images from approved domains for security.
|
|
"""
|
|
from urllib.parse import urlparse
|
|
import aiohttp
|
|
|
|
# Validate URL
|
|
try:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ('http', 'https'):
|
|
return Response(content="Invalid URL scheme", status_code=400)
|
|
|
|
# Check if domain is allowed
|
|
domain = parsed.netloc.lower()
|
|
if not _is_domain_allowed(domain):
|
|
return Response(content=f"Domain not allowed: {domain}", status_code=403)
|
|
except Exception:
|
|
return Response(content="Invalid URL", status_code=400)
|
|
|
|
# Fetch the image
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
}
|
|
async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
|
if resp.status != 200:
|
|
return Response(content=f"Failed to fetch image: {resp.status}", status_code=resp.status)
|
|
|
|
content_type = resp.headers.get('Content-Type', 'image/jpeg')
|
|
image_data = await resp.read()
|
|
|
|
return Response(
|
|
content=image_data,
|
|
media_type=content_type,
|
|
headers={
|
|
'Cache-Control': 'public, max-age=86400', # Cache for 24 hours
|
|
}
|
|
)
|
|
except asyncio.TimeoutError:
|
|
return Response(content="Image fetch timeout", status_code=504)
|
|
except Exception as e:
|
|
logger.error(f"Image proxy error for {url}: {e}", module="PaidContent")
|
|
return Response(content="Failed to fetch image", status_code=500)
|
|
|
|
|
|
@router.get("/cache/profile-image/{filename}")
|
|
@handle_exceptions
|
|
async def serve_cached_profile_image(filename: str, current_user: Dict = Depends(get_current_user)):
|
|
"""Serve a locally cached profile image (avatar or banner)."""
|
|
import re as _re
|
|
|
|
# Sanitize filename — only allow alphanumeric, underscore, hyphen, dot
|
|
if not _re.match(r'^[\w.-]+$', filename):
|
|
return Response(content="Invalid filename", status_code=400)
|
|
|
|
from ..core.config import Settings
|
|
cache_dir = Settings.DATA_DIR / 'cache' / 'profile_images'
|
|
filepath = cache_dir / filename
|
|
|
|
if not filepath.exists() or not filepath.is_file():
|
|
return Response(content="Not found", status_code=404)
|
|
|
|
# Determine content type from extension
|
|
ext = filepath.suffix.lower()
|
|
content_types = {
|
|
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
|
|
'.webp': 'image/webp', '.gif': 'image/gif',
|
|
}
|
|
content_type = content_types.get(ext, 'image/jpeg')
|
|
|
|
return Response(
|
|
content=filepath.read_bytes(),
|
|
media_type=content_type,
|
|
headers={'Cache-Control': 'public, max-age=604800'},
|
|
)
|
|
|
|
|
|
# ==================== MESSAGES ====================
|
|
|
|
|
|
@router.get("/messages/conversations")
|
|
@handle_exceptions
|
|
async def get_conversations(
|
|
request: Request,
|
|
creator_id: Optional[int] = Query(None),
|
|
service_id: Optional[str] = Query(None),
|
|
search: Optional[str] = Query(None),
|
|
limit: int = Query(50, ge=1, le=200),
|
|
offset: int = Query(0, ge=0),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get list of conversations with message stats"""
|
|
db = _get_db_adapter()
|
|
conversations = db.get_conversations(
|
|
creator_id=creator_id,
|
|
service_id=service_id,
|
|
search=search,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
return {"conversations": conversations}
|
|
|
|
|
|
@router.get("/messages/unread-count")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_unread_messages_count(request: Request, current_user: Dict = Depends(get_current_user)):
|
|
"""Get total count of unread messages."""
|
|
db = _get_db_adapter()
|
|
count = db.get_total_unread_messages_count()
|
|
return {"count": count}
|
|
|
|
|
|
@router.post("/messages/mark-all-read")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def mark_all_messages_read_endpoint(request: Request, current_user: Dict = Depends(get_current_user)):
|
|
"""Mark all messages as read."""
|
|
db = _get_db_adapter()
|
|
updated = db.mark_all_messages_read()
|
|
return {"updated": updated}
|
|
|
|
|
|
@router.get("/messages/{creator_id}")
|
|
@handle_exceptions
|
|
async def get_messages(
|
|
request: Request,
|
|
creator_id: int,
|
|
before: Optional[str] = Query(None),
|
|
limit: int = Query(50, ge=1, le=200),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get messages for a conversation"""
|
|
db = _get_db_adapter()
|
|
messages = db.get_messages(creator_id, before=before, limit=limit)
|
|
total = db.get_message_count(creator_id)
|
|
return {"messages": messages, "total": total}
|
|
|
|
|
|
@router.post("/messages/{creator_id}/sync")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def sync_messages(
|
|
request: Request,
|
|
creator_id: int,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Trigger manual message sync for a creator"""
|
|
db = _get_db_adapter()
|
|
creator = db.get_creator(creator_id)
|
|
if not creator:
|
|
raise NotFoundError("Creator not found")
|
|
|
|
background_tasks.add_task(_sync_messages_background, creator_id)
|
|
|
|
return {
|
|
"status": "queued",
|
|
"creator_id": creator_id,
|
|
"username": creator['username']
|
|
}
|
|
|
|
|
|
async def _sync_messages_background(creator_id: int):
|
|
"""Background task for message sync and attachment download"""
|
|
scraper = _get_scraper()
|
|
try:
|
|
creator = scraper.db.get_creator(creator_id)
|
|
if not creator:
|
|
return
|
|
|
|
service_id = creator.get('service_id', '')
|
|
|
|
if service_id == 'onlyfans_direct':
|
|
client = scraper._get_onlyfans_direct_client()
|
|
if client:
|
|
await scraper._sync_messages_for_creator(creator, client, 'onlyfans')
|
|
elif service_id == 'fansly_direct':
|
|
client = scraper._get_fansly_direct_client()
|
|
if client:
|
|
await scraper._sync_messages_for_creator(creator, client, 'fansly')
|
|
|
|
# Download any pending message attachments
|
|
pending = scraper.db.get_pending_message_attachments(creator_id)
|
|
if pending:
|
|
base_path = Path(scraper.config.get('base_download_path', '/paid-content'))
|
|
logger.info(f"Downloading {len(pending)} message attachments for {creator['username']}", module="PaidContent")
|
|
await scraper._download_message_attachments(pending, base_path, creator)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
|
|
@router.post("/messages/{creator_id}/mark-read")
|
|
@handle_exceptions
|
|
async def mark_messages_read(
|
|
request: Request,
|
|
creator_id: int,
|
|
body: Dict = None,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Mark messages as read. If message_ids provided, marks only those; otherwise marks all."""
|
|
db = _get_db_adapter()
|
|
message_ids = body.get('message_ids') if body else None
|
|
count = db.mark_messages_read(creator_id, message_ids=message_ids)
|
|
return {"status": "ok", "marked_read": count}
|
|
|
|
|
|
# ============================================================================
|
|
# AUTO-TAG RULES ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/auto-tag-rules")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_auto_tag_rules(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all auto-tag rules"""
|
|
db = _get_db_adapter()
|
|
rules = db.get_auto_tag_rules()
|
|
# Parse JSON fields
|
|
for rule in rules:
|
|
if isinstance(rule.get('conditions'), str):
|
|
rule['conditions'] = json.loads(rule['conditions'])
|
|
if isinstance(rule.get('tag_ids'), str):
|
|
rule['tag_ids'] = json.loads(rule['tag_ids'])
|
|
return {"rules": rules}
|
|
|
|
|
|
@router.get("/auto-tag-rules/{rule_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_auto_tag_rule(
|
|
request: Request,
|
|
rule_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get a single auto-tag rule"""
|
|
db = _get_db_adapter()
|
|
rule = db.get_auto_tag_rule(rule_id)
|
|
if not rule:
|
|
raise NotFoundError(f"Auto-tag rule {rule_id} not found")
|
|
if isinstance(rule.get('conditions'), str):
|
|
rule['conditions'] = json.loads(rule['conditions'])
|
|
if isinstance(rule.get('tag_ids'), str):
|
|
rule['tag_ids'] = json.loads(rule['tag_ids'])
|
|
return {"rule": rule}
|
|
|
|
|
|
@router.post("/auto-tag-rules")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def create_auto_tag_rule(
|
|
request: Request,
|
|
body: CreateAutoTagRuleRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Create a new auto-tag rule"""
|
|
db = _get_db_adapter()
|
|
rule_id = db.create_auto_tag_rule(
|
|
name=body.name,
|
|
conditions=body.conditions,
|
|
tag_ids=body.tag_ids,
|
|
priority=body.priority
|
|
)
|
|
rule = db.get_auto_tag_rule(rule_id)
|
|
if isinstance(rule.get('conditions'), str):
|
|
rule['conditions'] = json.loads(rule['conditions'])
|
|
if isinstance(rule.get('tag_ids'), str):
|
|
rule['tag_ids'] = json.loads(rule['tag_ids'])
|
|
return {"rule": rule}
|
|
|
|
|
|
@router.put("/auto-tag-rules/{rule_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def update_auto_tag_rule(
|
|
request: Request,
|
|
rule_id: int,
|
|
body: UpdateAutoTagRuleRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Update an auto-tag rule"""
|
|
db = _get_db_adapter()
|
|
existing = db.get_auto_tag_rule(rule_id)
|
|
if not existing:
|
|
raise NotFoundError(f"Auto-tag rule {rule_id} not found")
|
|
updates = {k: v for k, v in body.model_dump().items() if v is not None}
|
|
db.update_auto_tag_rule(rule_id, updates)
|
|
rule = db.get_auto_tag_rule(rule_id)
|
|
if isinstance(rule.get('conditions'), str):
|
|
rule['conditions'] = json.loads(rule['conditions'])
|
|
if isinstance(rule.get('tag_ids'), str):
|
|
rule['tag_ids'] = json.loads(rule['tag_ids'])
|
|
return {"rule": rule}
|
|
|
|
|
|
@router.delete("/auto-tag-rules/{rule_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def delete_auto_tag_rule(
|
|
request: Request,
|
|
rule_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Delete an auto-tag rule"""
|
|
db = _get_db_adapter()
|
|
if not db.delete_auto_tag_rule(rule_id):
|
|
raise NotFoundError(f"Auto-tag rule {rule_id} not found")
|
|
return message_response("Rule deleted")
|
|
|
|
|
|
_auto_tag_run_status = {}
|
|
|
|
@router.post("/auto-tag-rules/run")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def run_all_auto_tag_rules(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Run all enabled auto-tag rules on existing posts (background)"""
|
|
import uuid
|
|
job_id = str(uuid.uuid4())[:8]
|
|
_auto_tag_run_status[job_id] = {'status': 'running', 'started_at': now_iso8601()}
|
|
|
|
def _run():
|
|
try:
|
|
db = _get_db_adapter()
|
|
result = db.run_rules_on_existing_posts()
|
|
_auto_tag_run_status[job_id] = {**result, 'status': 'completed', 'completed_at': now_iso8601()}
|
|
except Exception as e:
|
|
_auto_tag_run_status[job_id] = {'status': 'error', 'error': str(e)}
|
|
|
|
background_tasks.add_task(_run)
|
|
return {"status": "started", "job_id": job_id}
|
|
|
|
|
|
@router.post("/auto-tag-rules/{rule_id}/run")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def run_single_auto_tag_rule(
|
|
request: Request,
|
|
rule_id: int,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Run a single auto-tag rule on existing posts (background)"""
|
|
db = _get_db_adapter()
|
|
rule = db.get_auto_tag_rule(rule_id)
|
|
if not rule:
|
|
raise NotFoundError(f"Auto-tag rule {rule_id} not found")
|
|
|
|
import uuid
|
|
job_id = str(uuid.uuid4())[:8]
|
|
_auto_tag_run_status[job_id] = {'status': 'running', 'started_at': now_iso8601()}
|
|
|
|
def _run():
|
|
try:
|
|
db2 = _get_db_adapter()
|
|
result = db2.run_rules_on_existing_posts(rule_id=rule_id)
|
|
_auto_tag_run_status[job_id] = {**result, 'status': 'completed', 'completed_at': now_iso8601()}
|
|
except Exception as e:
|
|
_auto_tag_run_status[job_id] = {'status': 'error', 'error': str(e)}
|
|
|
|
background_tasks.add_task(_run)
|
|
return {"status": "started", "job_id": job_id}
|
|
|
|
|
|
# ============================================================================
|
|
# ANALYTICS ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/analytics/storage-growth")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_storage_growth(
|
|
request: Request,
|
|
days: int = Query(default=30, ge=1, le=365),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get storage growth over time"""
|
|
db = _get_db_adapter()
|
|
data = db.get_storage_growth_over_time(days)
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/downloads-per-period")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_downloads_per_period(
|
|
request: Request,
|
|
period: str = Query(default='day'),
|
|
days: int = Query(default=30, ge=1, le=365),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get download counts per day or week"""
|
|
db = _get_db_adapter()
|
|
data = db.get_downloads_per_period(period, days)
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/storage-by-creator")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_storage_by_creator(
|
|
request: Request,
|
|
limit: int = Query(default=20, ge=1, le=100),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get top creators by storage"""
|
|
db = _get_db_adapter()
|
|
data = db.get_storage_by_creator(limit)
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/platform-distribution")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_platform_distribution(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get platform distribution"""
|
|
db = _get_db_adapter()
|
|
data = db.get_platform_distribution()
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/content-type-distribution")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_content_type_distribution(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get content type distribution"""
|
|
db = _get_db_adapter()
|
|
data = db.get_content_type_distribution()
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/creator-scorecards")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_creator_scorecards(
|
|
request: Request,
|
|
sort_by: str = Query(default='total_bytes'),
|
|
limit: int = Query(default=50, ge=1, le=200),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get creator stats table"""
|
|
db = _get_db_adapter()
|
|
data = db.get_creator_scorecards(sort_by, limit)
|
|
return {"data": data}
|
|
|
|
|
|
@router.get("/analytics/summary")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def get_analytics_summary(
|
|
request: Request,
|
|
days: int = Query(default=30, ge=1, le=365),
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all analytics data in one call"""
|
|
db = _get_db_adapter()
|
|
return {
|
|
"storage_growth": db.get_storage_growth_over_time(days),
|
|
"downloads_per_period": db.get_downloads_per_period('day', days),
|
|
"storage_by_creator": db.get_storage_by_creator(20),
|
|
"platform_distribution": db.get_platform_distribution(),
|
|
"content_type_distribution": db.get_content_type_distribution(),
|
|
"creator_scorecards": db.get_creator_scorecards('total_bytes', 50),
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# WATCH LATER ENDPOINTS
|
|
# ============================================================================
|
|
|
|
@router.get("/watch-later")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_watch_later(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all watch later items with metadata"""
|
|
db = _get_db_adapter()
|
|
items = db.get_watch_later()
|
|
return {"items": items}
|
|
|
|
|
|
@router.get("/watch-later/count")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def get_watch_later_count(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get watch later count"""
|
|
db = _get_db_adapter()
|
|
count = db.get_watch_later_count()
|
|
return {"count": count}
|
|
|
|
|
|
@router.get("/watch-later/attachment-ids")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_watch_later_attachment_ids(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get set of attachment IDs in watch later (for toggle state)"""
|
|
db = _get_db_adapter()
|
|
ids = db.get_watch_later_attachment_ids()
|
|
return {"attachment_ids": ids}
|
|
|
|
|
|
@router.post("/watch-later")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def add_to_watch_later(
|
|
request: Request,
|
|
body: WatchLaterAddRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add an attachment to watch later"""
|
|
db = _get_db_adapter()
|
|
wl_id = db.add_to_watch_later(body.attachment_id)
|
|
if wl_id is None:
|
|
return {"status": "already_exists"}
|
|
return {"status": "added", "id": wl_id}
|
|
|
|
|
|
@router.post("/watch-later/bulk")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def add_bulk_to_watch_later(
|
|
request: Request,
|
|
body: WatchLaterBulkAddRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add multiple attachments to watch later"""
|
|
db = _get_db_adapter()
|
|
added = 0
|
|
for aid in body.attachment_ids:
|
|
if db.add_to_watch_later(aid) is not None:
|
|
added += 1
|
|
return {"status": "ok", "added": added}
|
|
|
|
|
|
@router.delete("/watch-later/{attachment_id}")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def remove_from_watch_later(
|
|
request: Request,
|
|
attachment_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove an attachment from watch later"""
|
|
db = _get_db_adapter()
|
|
db.remove_from_watch_later(attachment_id)
|
|
return message_response("Removed from watch later")
|
|
|
|
|
|
@router.post("/watch-later/remove")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def remove_multiple_from_watch_later(
|
|
request: Request,
|
|
body: WatchLaterRemoveRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Remove multiple attachments from watch later"""
|
|
db = _get_db_adapter()
|
|
removed = 0
|
|
for aid in body.attachment_ids:
|
|
if db.remove_from_watch_later(aid):
|
|
removed += 1
|
|
return {"status": "ok", "removed": removed}
|
|
|
|
|
|
@router.post("/watch-later/reorder")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def reorder_watch_later(
|
|
request: Request,
|
|
body: WatchLaterReorderRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Reorder watch later items"""
|
|
db = _get_db_adapter()
|
|
db.reorder_watch_later(body.ordered_ids)
|
|
return message_response("Reordered")
|
|
|
|
|
|
@router.delete("/watch-later")
|
|
@limiter.limit("10/minute")
|
|
@handle_exceptions
|
|
async def clear_watch_later(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Clear all watch later items"""
|
|
db = _get_db_adapter()
|
|
count = db.clear_watch_later()
|
|
return {"status": "ok", "removed": count}
|
|
|
|
|
|
# ============================================================================
|
|
# FACE RECOGNITION FOR BULK DELETE
|
|
# ============================================================================
|
|
|
|
def _ensure_face_table():
|
|
"""Create paid_content_face_references table if it doesn't exist."""
|
|
app_state = get_app_state()
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS paid_content_face_references (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
person_name TEXT NOT NULL,
|
|
encoding_data TEXT NOT NULL,
|
|
thumbnail_data TEXT,
|
|
is_active INTEGER DEFAULT 1,
|
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
conn.commit()
|
|
|
|
|
|
def _face_scan_background(creator_id: int, person_name: str, tolerance: float):
|
|
"""Background task: scan all posts for a creator and tag by face match."""
|
|
import pickle
|
|
import base64
|
|
import gc
|
|
from modules.face_recognition_module import FaceRecognitionModule
|
|
|
|
app_state = get_app_state()
|
|
db = _get_db_adapter()
|
|
|
|
with _face_scan_lock:
|
|
_face_scan_status['current'] = {
|
|
'status': 'initializing',
|
|
'processed': 0,
|
|
'total': 0,
|
|
'matched': 0,
|
|
'unmatched': 0,
|
|
'errors': 0,
|
|
'current_post': None,
|
|
}
|
|
|
|
try:
|
|
# 1. Create FaceRecognitionModule, load ONLY paid_content references
|
|
fr = FaceRecognitionModule(unified_db=app_state.db)
|
|
fr.reference_encodings = {} # Clear default references
|
|
fr.reference_encodings_fr = {}
|
|
|
|
_ensure_face_table()
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT person_name, encoding_data
|
|
FROM paid_content_face_references
|
|
WHERE is_active = 1
|
|
""")
|
|
rows = cursor.fetchall()
|
|
|
|
for pname, enc_data in rows:
|
|
enc = pickle.loads(base64.b64decode(enc_data))
|
|
if pname not in fr.reference_encodings:
|
|
fr.reference_encodings[pname] = []
|
|
fr.reference_encodings[pname].append(enc)
|
|
|
|
if not fr.reference_encodings:
|
|
with _face_scan_lock:
|
|
_face_scan_status['current'] = {
|
|
'status': 'error',
|
|
'processed': 0, 'total': 0, 'matched': 0, 'unmatched': 0, 'errors': 0,
|
|
'error_message': 'No face references loaded. Add reference photos first.',
|
|
}
|
|
return
|
|
|
|
logger.info(f"Loaded {sum(len(v) for v in fr.reference_encodings.values())} paid-content face references", module="FaceScan")
|
|
|
|
# 2. (Tags no longer created - matches add tagged user 'lovefromreyn' instead)
|
|
|
|
# 3. Get all non-deleted, non-pinned posts for creator with completed image attachments
|
|
# Skip posts that already have tagged users (they were tagged via Instagram backfill)
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT DISTINCT p.id
|
|
FROM paid_content_posts p
|
|
JOIN paid_content_attachments a ON a.post_id = p.id
|
|
WHERE p.creator_id = ?
|
|
AND p.deleted_at IS NULL
|
|
AND p.is_pinned = 0
|
|
AND a.status = 'completed'
|
|
AND a.local_path IS NOT NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM paid_content_post_tagged_users tu
|
|
WHERE tu.post_id = p.id
|
|
)
|
|
ORDER BY p.id
|
|
""", (creator_id,))
|
|
post_ids = [row[0] for row in cursor.fetchall()]
|
|
|
|
total = len(post_ids)
|
|
with _face_scan_lock:
|
|
_face_scan_status['current']['total'] = total
|
|
_face_scan_status['current']['status'] = 'scanning'
|
|
|
|
logger.info(f"Face scan: {total} posts to scan for creator {creator_id}", module="FaceScan")
|
|
|
|
matched = 0
|
|
unmatched = 0
|
|
errors = 0
|
|
match_confidences = [] # track confidence of each match
|
|
|
|
for idx, post_id in enumerate(post_ids):
|
|
# Get attachments for this post
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT local_path, file_type
|
|
FROM paid_content_attachments
|
|
WHERE post_id = ?
|
|
AND status = 'completed'
|
|
AND local_path IS NOT NULL
|
|
""", (post_id,))
|
|
attachments = cursor.fetchall()
|
|
|
|
post_has_match = False
|
|
best_confidence = 0.0
|
|
|
|
for local_path, file_type in attachments:
|
|
if not Path(local_path).exists():
|
|
continue
|
|
|
|
try:
|
|
is_video = (file_type == 'video')
|
|
# Directly detect + match, bypassing Immich shortcut
|
|
# which would use the wrong reference table
|
|
video_tolerance = tolerance + 0.10 # looser for video frames
|
|
if is_video:
|
|
# Sample 3 frames for better coverage
|
|
temp_frames = fr._extract_video_frames_at_positions(
|
|
local_path, [0.15, 0.5, 0.85]
|
|
)
|
|
for temp_frame in temp_frames:
|
|
try:
|
|
face_encodings = fr.detect_faces_insightface(temp_frame)
|
|
for enc in face_encodings:
|
|
matched_person, confidence, _ = fr.match_face_insightface(enc, video_tolerance)
|
|
if confidence > best_confidence:
|
|
best_confidence = confidence
|
|
if matched_person:
|
|
post_has_match = True
|
|
break
|
|
finally:
|
|
import os as _os
|
|
try:
|
|
_os.unlink(temp_frame)
|
|
except Exception:
|
|
pass
|
|
if post_has_match:
|
|
break
|
|
else:
|
|
face_encodings = fr.detect_faces_insightface(local_path)
|
|
for enc in face_encodings:
|
|
matched_person, confidence, _ = fr.match_face_insightface(enc, tolerance)
|
|
if confidence > best_confidence:
|
|
best_confidence = confidence
|
|
if matched_person:
|
|
post_has_match = True
|
|
break
|
|
if post_has_match:
|
|
break # One match is enough for the post
|
|
except Exception as e:
|
|
logger.debug(f"Face scan error on {local_path}: {e}", module="FaceScan")
|
|
errors += 1
|
|
|
|
# Apply result: match adds tagged user 'lovefromreyn', no match does nothing
|
|
if post_has_match:
|
|
db.add_tagged_user(post_id, 'lovefromreyn')
|
|
matched += 1
|
|
match_confidences.append(round(float(best_confidence) * 100, 1))
|
|
else:
|
|
unmatched += 1
|
|
|
|
avg_conf = float(round(sum(match_confidences) / len(match_confidences), 1)) if match_confidences else 0.0
|
|
min_conf = float(min(match_confidences)) if match_confidences else 0.0
|
|
max_conf = float(max(match_confidences)) if match_confidences else 0.0
|
|
|
|
with _face_scan_lock:
|
|
_face_scan_status['current'].update({
|
|
'processed': idx + 1,
|
|
'matched': matched,
|
|
'unmatched': unmatched,
|
|
'avg_confidence': avg_conf,
|
|
'min_confidence': min_conf,
|
|
'max_confidence': max_conf,
|
|
'errors': errors,
|
|
})
|
|
|
|
# Periodic GC every 50 posts
|
|
if (idx + 1) % 50 == 0:
|
|
gc.collect()
|
|
|
|
# Done
|
|
fr.release_model()
|
|
gc.collect()
|
|
|
|
with _face_scan_lock:
|
|
_face_scan_status['current'].update({
|
|
'status': 'completed',
|
|
'processed': total,
|
|
'matched': matched,
|
|
'unmatched': unmatched,
|
|
'errors': errors,
|
|
})
|
|
|
|
logger.info(f"Face scan complete: {matched} matched, {unmatched} unmatched, {errors} errors", module="FaceScan")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Face scan failed: {e}", module="FaceScan")
|
|
with _face_scan_lock:
|
|
_face_scan_status['current'] = {
|
|
'status': 'error',
|
|
'processed': 0, 'total': 0, 'matched': 0, 'unmatched': 0, 'errors': 0,
|
|
'error_message': str(e),
|
|
}
|
|
|
|
|
|
@router.post("/face/add-reference")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def face_add_reference(
|
|
request: Request,
|
|
body: FaceAddReferenceRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Add a face reference from an existing attachment's file path."""
|
|
import pickle
|
|
import base64
|
|
from modules.face_recognition_module import FaceRecognitionModule
|
|
|
|
file_path = body.file_path
|
|
if not Path(file_path).exists():
|
|
raise ValidationError(f"File not found: {file_path}")
|
|
|
|
app_state = get_app_state()
|
|
_ensure_face_table()
|
|
|
|
# Detect face and extract encoding
|
|
fr = FaceRecognitionModule(unified_db=app_state.db)
|
|
fr.reference_encodings = {}
|
|
fr.reference_encodings_fr = {}
|
|
|
|
encodings = fr.detect_faces_insightface(file_path)
|
|
if not encodings:
|
|
fr.release_model()
|
|
raise ValidationError("No face detected in image")
|
|
|
|
# Use the first (largest/most prominent) face
|
|
encoding = encodings[0]
|
|
encoding_b64 = base64.b64encode(pickle.dumps(encoding)).decode('utf-8')
|
|
|
|
# Generate thumbnail
|
|
thumbnail_b64 = fr._generate_thumbnail(file_path)
|
|
fr.release_model()
|
|
|
|
# Store in paid_content_face_references
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
INSERT INTO paid_content_face_references (person_name, encoding_data, thumbnail_data, is_active, created_at)
|
|
VALUES (?, ?, ?, 1, ?)
|
|
""", (body.person_name, encoding_b64, thumbnail_b64, datetime.now().isoformat()))
|
|
conn.commit()
|
|
ref_id = cursor.lastrowid
|
|
|
|
return {"id": ref_id, "person_name": body.person_name, "thumbnail": thumbnail_b64}
|
|
|
|
|
|
@router.get("/face/references")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def face_get_references(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get all active face references from paid_content_face_references."""
|
|
app_state = get_app_state()
|
|
_ensure_face_table()
|
|
|
|
with app_state.db.get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
SELECT id, person_name, thumbnail_data, created_at
|
|
FROM paid_content_face_references
|
|
WHERE is_active = 1
|
|
ORDER BY created_at DESC
|
|
""")
|
|
refs = [
|
|
{"id": row[0], "person_name": row[1], "thumbnail": row[2], "created_at": row[3]}
|
|
for row in cursor.fetchall()
|
|
]
|
|
|
|
return {"references": refs}
|
|
|
|
|
|
@router.delete("/face/references/{ref_id}")
|
|
@limiter.limit("30/minute")
|
|
@handle_exceptions
|
|
async def face_delete_reference(
|
|
request: Request,
|
|
ref_id: int,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Soft-delete a face reference."""
|
|
app_state = get_app_state()
|
|
_ensure_face_table()
|
|
|
|
with app_state.db.get_connection(for_write=True) as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("UPDATE paid_content_face_references SET is_active = 0 WHERE id = ?", (ref_id,))
|
|
conn.commit()
|
|
if cursor.rowcount == 0:
|
|
raise NotFoundError(f"Reference {ref_id} not found")
|
|
|
|
return message_response("Reference deleted")
|
|
|
|
|
|
@router.post("/face/scan-creator")
|
|
@limiter.limit("5/minute")
|
|
@handle_exceptions
|
|
async def face_scan_creator(
|
|
request: Request,
|
|
background_tasks: BackgroundTasks,
|
|
body: FaceScanCreatorRequest,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Start a background face scan for a creator's posts."""
|
|
# Clear any stale status (e.g. from a previous run killed by restart)
|
|
with _face_scan_lock:
|
|
_face_scan_status.pop('current', None)
|
|
|
|
background_tasks.add_task(
|
|
_face_scan_background,
|
|
body.creator_id,
|
|
body.person_name,
|
|
body.tolerance,
|
|
)
|
|
|
|
return {"status": "started", "creator_id": body.creator_id, "person_name": body.person_name}
|
|
|
|
|
|
@router.get("/face/scan-status")
|
|
@limiter.limit("120/minute")
|
|
@handle_exceptions
|
|
async def face_scan_status(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get current face scan status."""
|
|
with _face_scan_lock:
|
|
status = _face_scan_status.get('current', {'status': 'idle'})
|
|
return status
|
|
|
|
|
|
# ============================================================================
|
|
# GALLERY (flat media timeline)
|
|
# ============================================================================
|
|
|
|
@router.get("/gallery/groups")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_gallery_groups(
|
|
request: Request,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get creator groups enriched with media counts for gallery landing."""
|
|
db = _get_db_adapter()
|
|
groups = db.get_gallery_group_stats()
|
|
return {"groups": groups}
|
|
|
|
|
|
@router.get("/gallery/media")
|
|
@limiter.limit("300/minute")
|
|
@handle_exceptions
|
|
async def get_gallery_media(
|
|
request: Request,
|
|
creator_group_id: Optional[int] = None,
|
|
creator_id: Optional[int] = None,
|
|
content_type: Optional[str] = None,
|
|
min_resolution: Optional[str] = None,
|
|
date_from: Optional[str] = None,
|
|
date_to: Optional[str] = None,
|
|
search: Optional[str] = None,
|
|
shuffle: bool = False,
|
|
shuffle_seed: Optional[int] = None,
|
|
limit: int = Query(default=200, le=500),
|
|
offset: int = 0,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get flat media items for gallery timeline."""
|
|
db = _get_db_adapter()
|
|
items = db.get_gallery_media(
|
|
creator_group_id=creator_group_id,
|
|
creator_id=creator_id,
|
|
content_type=content_type,
|
|
min_resolution=min_resolution,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
search=search,
|
|
shuffle=shuffle,
|
|
shuffle_seed=shuffle_seed,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
# Only run COUNT on first page — subsequent pages don't need it
|
|
total = None
|
|
if offset == 0:
|
|
total = db.get_gallery_media_count(
|
|
creator_group_id=creator_group_id,
|
|
creator_id=creator_id,
|
|
content_type=content_type,
|
|
min_resolution=min_resolution,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
search=search
|
|
)
|
|
return {"items": items, "total": total, "limit": limit, "offset": offset}
|
|
|
|
|
|
@router.get("/gallery/date-range")
|
|
@limiter.limit("60/minute")
|
|
@handle_exceptions
|
|
async def get_gallery_date_range(
|
|
request: Request,
|
|
creator_group_id: Optional[int] = None,
|
|
creator_id: Optional[int] = None,
|
|
content_type: Optional[str] = None,
|
|
current_user: Dict = Depends(get_current_user)
|
|
):
|
|
"""Get year/month distribution for timeline scrubber."""
|
|
db = _get_db_adapter()
|
|
ranges = db.get_gallery_date_range(
|
|
creator_group_id=creator_group_id,
|
|
creator_id=creator_id,
|
|
content_type=content_type
|
|
)
|
|
return {"ranges": ranges}
|