1159 lines
48 KiB
Python
1159 lines
48 KiB
Python
"""
|
|
Fansly Direct API Client
|
|
|
|
Downloads content directly from the Fansly API (not via Coomer).
|
|
"""
|
|
|
|
import aiohttp
|
|
import asyncio
|
|
from datetime import datetime
|
|
from typing import List, Optional, Dict, Any, Callable
|
|
|
|
from modules.base_module import LoggingMixin, RateLimitMixin
|
|
from .models import Post, Attachment, Message
|
|
|
|
|
|
class FanslyDirectClient(LoggingMixin, RateLimitMixin):
|
|
"""
|
|
API client for downloading content directly from Fansly
|
|
|
|
API Endpoints:
|
|
- Base URL: https://apiv3.fansly.com/api/v1
|
|
- Auth: Authorization header with token
|
|
- GET /account?usernames={username} - Get account info
|
|
- GET /timelinenew/{account_id}?before={cursor} - Get posts (paginated)
|
|
"""
|
|
|
|
BASE_URL = "https://apiv3.fansly.com/api/v1"
|
|
SERVICE_ID = "fansly_direct"
|
|
PLATFORM = "fansly"
|
|
|
|
def __init__(self, auth_token: str, log_callback: Optional[Callable] = None):
|
|
self._init_logger('PaidContent', log_callback, default_module='FanslyDirect')
|
|
# Conservative rate limiting for real Fansly API
|
|
self._init_rate_limiter(min_delay=1.0, max_delay=3.0, batch_delay_min=2, batch_delay_max=5)
|
|
|
|
self.auth_token = auth_token
|
|
self._session: Optional[aiohttp.ClientSession] = None
|
|
|
|
async def _get_session(self) -> aiohttp.ClientSession:
|
|
"""Get or create aiohttp session with Fansly headers"""
|
|
if self._session is None or self._session.closed:
|
|
headers = {
|
|
'Authorization': self.auth_token,
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'application/json, text/plain, */*',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Origin': 'https://fansly.com',
|
|
'Referer': 'https://fansly.com/',
|
|
}
|
|
timeout = aiohttp.ClientTimeout(total=60)
|
|
self._session = aiohttp.ClientSession(headers=headers, timeout=timeout)
|
|
return self._session
|
|
|
|
async def close(self):
|
|
"""Close the aiohttp session"""
|
|
if self._session and not self._session.closed:
|
|
await self._session.close()
|
|
self._session = None
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
await self.close()
|
|
|
|
async def check_auth(self) -> Dict[str, Any]:
|
|
"""
|
|
Verify the auth token is valid.
|
|
|
|
Returns:
|
|
Dict with 'valid' bool and optionally 'username' and 'account_id'
|
|
"""
|
|
self._delay_between_items()
|
|
try:
|
|
session = await self._get_session()
|
|
|
|
# Get own account info to verify auth
|
|
async with session.get(f"{self.BASE_URL}/account/me") as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
account = data.get('response', {}).get('account', {})
|
|
return {
|
|
'valid': True,
|
|
'account_id': account.get('id'),
|
|
'username': account.get('username'),
|
|
'display_name': account.get('displayName'),
|
|
}
|
|
elif resp.status == 401:
|
|
return {'valid': False, 'error': 'Invalid or expired auth token'}
|
|
else:
|
|
return {'valid': False, 'error': f'HTTP {resp.status}'}
|
|
|
|
except Exception as e:
|
|
self.log(f"Error checking auth: {e}", 'error')
|
|
return {'valid': False, 'error': str(e)}
|
|
|
|
async def get_account_info(self, username: str) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Get account info for a username.
|
|
|
|
Args:
|
|
username: The Fansly username
|
|
|
|
Returns:
|
|
Account info dict or None if not found
|
|
"""
|
|
self._delay_between_items()
|
|
try:
|
|
session = await self._get_session()
|
|
|
|
async with session.get(f"{self.BASE_URL}/account", params={'usernames': username}) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
accounts = data.get('response', [])
|
|
if accounts and len(accounts) > 0:
|
|
account = accounts[0]
|
|
# Extract bio from various possible field names
|
|
bio = None
|
|
for field in ['about', 'bio', 'description', 'rawAbout', 'profileDescription']:
|
|
if account.get(field):
|
|
bio = account.get(field)
|
|
self.log(f"Found bio in field '{field}': {bio[:50] if len(bio) > 50 else bio}...", 'debug')
|
|
break
|
|
|
|
# Extract location
|
|
location = account.get('location') or None
|
|
|
|
# Extract social links (profileSocials)
|
|
external_links = None
|
|
profile_socials = account.get('profileSocials', [])
|
|
if profile_socials:
|
|
import json
|
|
external_links = json.dumps(profile_socials)
|
|
|
|
# Log available fields for debugging
|
|
self.log(f"Account fields: {list(account.keys())}", 'debug')
|
|
|
|
return {
|
|
'account_id': account.get('id'),
|
|
'username': account.get('username'),
|
|
'display_name': account.get('displayName'),
|
|
'avatar_url': self._get_avatar_url(account),
|
|
'banner_url': self._get_banner_url(account),
|
|
'bio': bio,
|
|
'location': location,
|
|
'external_links': external_links,
|
|
}
|
|
self.log(f"Account not found: {username}", 'warning')
|
|
return None
|
|
elif resp.status == 404:
|
|
self.log(f"Account not found: {username}", 'warning')
|
|
return None
|
|
else:
|
|
self.log(f"Error getting account info: HTTP {resp.status}", 'warning')
|
|
return None
|
|
|
|
except Exception as e:
|
|
self.log(f"Error getting account info for {username}: {e}", 'error')
|
|
return None
|
|
|
|
def _get_avatar_url(self, account: Dict) -> Optional[str]:
|
|
"""Extract avatar URL from account data"""
|
|
avatar = account.get('avatar')
|
|
if avatar and avatar.get('locations'):
|
|
locations = avatar.get('locations', [])
|
|
if locations:
|
|
return locations[0].get('location')
|
|
return None
|
|
|
|
def _get_banner_url(self, account: Dict) -> Optional[str]:
|
|
"""Extract banner URL from account data"""
|
|
banner = account.get('banner')
|
|
if banner and banner.get('locations'):
|
|
locations = banner.get('locations', [])
|
|
if locations:
|
|
return locations[0].get('location')
|
|
return None
|
|
|
|
async def get_single_post(self, post_id: str, account_id: str) -> Optional['Post']:
|
|
"""
|
|
Fetch a single post by its Fansly post ID.
|
|
Uses the /post endpoint to get one specific post with its media.
|
|
|
|
Args:
|
|
post_id: The Fansly post ID
|
|
account_id: The account ID of the creator (for parsing)
|
|
|
|
Returns:
|
|
Post object or None
|
|
"""
|
|
self._delay_between_items()
|
|
try:
|
|
session = await self._get_session()
|
|
|
|
params = {'ngsw-bypass': 'true', 'ids': post_id}
|
|
url = f"{self.BASE_URL}/post"
|
|
async with session.get(url, params=params) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
response = data.get('response', {})
|
|
|
|
posts = response.get('posts', [])
|
|
if not posts:
|
|
self.log(f"Post {post_id} not found in API response", 'warning')
|
|
return None
|
|
|
|
# Build lookup dicts same as timeline
|
|
media_dict = {}
|
|
for m in response.get('media', []):
|
|
media_dict[m.get('id')] = m
|
|
|
|
account_media_dict = {}
|
|
for am in response.get('accountMedia', []):
|
|
account_media_dict[am.get('id')] = am
|
|
|
|
bundle_dict = {}
|
|
for bundle in response.get('accountMediaBundles', []):
|
|
bundle_dict[bundle.get('id')] = bundle
|
|
|
|
polls_dict = {}
|
|
for poll in response.get('polls', []):
|
|
polls_dict[poll.get('id')] = poll
|
|
|
|
# Build minimal account dict for parsing
|
|
account = {'account_id': account_id}
|
|
# Try to get account info from response
|
|
accounts = response.get('accounts', [])
|
|
if accounts:
|
|
acc = accounts[0]
|
|
account['account_id'] = str(acc.get('id', account_id))
|
|
|
|
post = self._parse_post(posts[0], account, media_dict, account_media_dict, bundle_dict, polls_dict)
|
|
return post
|
|
else:
|
|
self.log(f"Error fetching post {post_id}: HTTP {resp.status}", 'warning')
|
|
return None
|
|
|
|
except Exception as e:
|
|
self.log(f"Error fetching single post {post_id}: {e}", 'error')
|
|
return None
|
|
|
|
async def get_posts(
|
|
self,
|
|
username: str,
|
|
since_date: Optional[str] = None,
|
|
until_date: Optional[str] = None,
|
|
days_back: Optional[int] = None,
|
|
max_posts: Optional[int] = None,
|
|
progress_callback: Optional[Callable[[int, int], None]] = None
|
|
) -> List[Post]:
|
|
"""
|
|
Fetch posts from a creator's timeline.
|
|
|
|
Args:
|
|
username: The Fansly username
|
|
since_date: Only fetch posts after this date (ISO format)
|
|
until_date: Only fetch posts before this date (ISO format)
|
|
days_back: Fetch posts from the last N days
|
|
max_posts: Maximum number of posts to fetch
|
|
progress_callback: Called with (page, total_posts) during fetching
|
|
|
|
Returns:
|
|
List of Post objects
|
|
"""
|
|
# First get account info to get the account ID
|
|
account = await self.get_account_info(username)
|
|
if not account:
|
|
self.log(f"Could not find account for {username}", 'error')
|
|
return []
|
|
|
|
account_id = account['account_id']
|
|
self.log(f"Fetching posts for {username} (account_id: {account_id})", 'info')
|
|
|
|
# Calculate date filters
|
|
since_timestamp = None
|
|
until_timestamp = None
|
|
|
|
if days_back:
|
|
from datetime import timedelta
|
|
since_date = (datetime.now() - timedelta(days=days_back)).isoformat()
|
|
|
|
if since_date:
|
|
try:
|
|
since_dt = datetime.fromisoformat(since_date.replace('Z', '+00:00'))
|
|
since_timestamp = int(since_dt.timestamp()) # Fansly uses seconds
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
if until_date:
|
|
try:
|
|
until_dt = datetime.fromisoformat(until_date.replace('Z', '+00:00'))
|
|
until_timestamp = int(until_dt.timestamp()) # Fansly uses seconds
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
# Fetch posts with pagination
|
|
all_posts: List[Post] = []
|
|
cursor: Optional[str] = None
|
|
page = 0
|
|
all_pinned_posts: Dict[str, Dict] = {}
|
|
|
|
while True:
|
|
posts_batch, new_cursor, media_dict, account_media_dict, bundle_dict, pinned_posts_dict, polls_dict = await self._fetch_timeline_page(
|
|
account_id, cursor
|
|
)
|
|
|
|
# Collect pinned posts from all pages (they're in every response)
|
|
all_pinned_posts.update(pinned_posts_dict)
|
|
|
|
if not posts_batch:
|
|
break
|
|
|
|
for post_data in posts_batch:
|
|
post = self._parse_post(post_data, account, media_dict, account_media_dict, bundle_dict, polls_dict)
|
|
if not post:
|
|
continue
|
|
|
|
# Check if this post is pinned
|
|
post_id_str = str(post_data.get('id', ''))
|
|
if post_id_str in all_pinned_posts:
|
|
post.is_pinned = True
|
|
post.pinned_at = all_pinned_posts[post_id_str].get('pinned_at')
|
|
self.log(f"Post {post_id_str} is pinned (pinned at {post.pinned_at})", 'debug')
|
|
|
|
# Check date filters
|
|
post_timestamp = post_data.get('createdAt', 0)
|
|
|
|
# Stop if we've gone past the since_date
|
|
if since_timestamp and post_timestamp < since_timestamp:
|
|
self.log(f"Reached posts older than since_date, stopping", 'debug')
|
|
return all_posts
|
|
|
|
# Skip posts newer than until_date
|
|
if until_timestamp and post_timestamp > until_timestamp:
|
|
continue
|
|
|
|
all_posts.append(post)
|
|
|
|
if max_posts and len(all_posts) >= max_posts:
|
|
self.log(f"Reached max_posts limit: {max_posts}", 'debug')
|
|
return all_posts
|
|
|
|
page += 1
|
|
if progress_callback:
|
|
progress_callback(page, len(all_posts))
|
|
|
|
# Check if we have more pages
|
|
if not new_cursor:
|
|
break
|
|
|
|
cursor = new_cursor
|
|
self._delay_between_batches()
|
|
|
|
# Store pinned posts info for the caller to update DB for posts outside date range
|
|
self._last_pinned_posts = all_pinned_posts
|
|
|
|
self.log(f"Fetched {len(all_posts)} posts for {username}", 'info')
|
|
return all_posts
|
|
|
|
async def _fetch_timeline_page(
|
|
self,
|
|
account_id: str,
|
|
cursor: Optional[str] = None
|
|
) -> tuple[List[Dict], Optional[str], Dict, Dict, Dict, Dict, Dict]:
|
|
"""
|
|
Fetch a single page of timeline posts.
|
|
|
|
Returns:
|
|
Tuple of (posts, next_cursor, media_dict, account_media_dict, bundle_dict, pinned_posts_dict, polls_dict)
|
|
"""
|
|
self._delay_between_items()
|
|
try:
|
|
session = await self._get_session()
|
|
|
|
params = {'ngsw-bypass': 'true'}
|
|
if cursor:
|
|
params['before'] = cursor
|
|
|
|
url = f"{self.BASE_URL}/timelinenew/{account_id}"
|
|
async with session.get(url, params=params) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
response = data.get('response', {})
|
|
|
|
posts = response.get('posts', [])
|
|
# Build media lookup dicts
|
|
media_dict = {}
|
|
for m in response.get('media', []):
|
|
media_dict[m.get('id')] = m
|
|
|
|
account_media_dict = {}
|
|
for am in response.get('accountMedia', []):
|
|
account_media_dict[am.get('id')] = am
|
|
|
|
# Build bundle lookup dict (for contentType=2 attachments)
|
|
bundle_dict = {}
|
|
for bundle in response.get('accountMediaBundles', []):
|
|
bundle_dict[bundle.get('id')] = bundle
|
|
|
|
# Build polls lookup dict (for contentType=42001 attachments)
|
|
polls_dict = {}
|
|
for poll in response.get('polls', []):
|
|
polls_dict[poll.get('id')] = poll
|
|
|
|
# Extract pinned posts from accounts
|
|
pinned_posts_dict = {}
|
|
for acc in response.get('accounts', []):
|
|
for pinned in acc.get('pinnedPosts', []):
|
|
post_id = pinned.get('postId')
|
|
if post_id:
|
|
# createdAt is in milliseconds
|
|
pinned_at = pinned.get('createdAt', 0)
|
|
if pinned_at > 1000000000000:
|
|
pinned_at = pinned_at / 1000
|
|
pinned_posts_dict[str(post_id)] = {
|
|
'pinned_at': datetime.fromtimestamp(pinned_at).isoformat() if pinned_at else None,
|
|
'pos': pinned.get('pos', 0)
|
|
}
|
|
|
|
# Get cursor for next page (id of last post)
|
|
next_cursor = None
|
|
if posts:
|
|
last_post = posts[-1]
|
|
next_cursor = str(last_post.get('id'))
|
|
|
|
return posts, next_cursor, media_dict, account_media_dict, bundle_dict, pinned_posts_dict, polls_dict
|
|
|
|
elif resp.status == 401:
|
|
self.log("Auth token expired or invalid", 'error')
|
|
return [], None, {}, {}, {}, {}, {}
|
|
else:
|
|
self.log(f"Error fetching timeline: HTTP {resp.status}", 'warning')
|
|
return [], None, {}, {}, {}, {}, {}
|
|
|
|
except Exception as e:
|
|
self.log(f"Error fetching timeline page: {e}", 'error')
|
|
return [], None, {}, {}, {}, {}, {}
|
|
|
|
def _parse_post(
|
|
self,
|
|
post_data: Dict,
|
|
account: Dict,
|
|
media_dict: Dict,
|
|
account_media_dict: Dict,
|
|
bundle_dict: Dict,
|
|
polls_dict: Optional[Dict] = None
|
|
) -> Optional[Post]:
|
|
"""
|
|
Parse Fansly post data into a Post model.
|
|
|
|
Args:
|
|
post_data: Raw post data from API
|
|
account: Account info dict
|
|
media_dict: Media lookup dict
|
|
account_media_dict: Account media lookup dict
|
|
bundle_dict: Media bundle lookup dict (for contentType=2)
|
|
polls_dict: Poll lookup dict (for contentType=42001)
|
|
|
|
Returns:
|
|
Post object or None if parsing fails
|
|
"""
|
|
try:
|
|
post_id = str(post_data.get('id', ''))
|
|
if not post_id:
|
|
return None
|
|
|
|
# Parse timestamp
|
|
created_at = post_data.get('createdAt', 0)
|
|
published_at = None
|
|
if created_at:
|
|
try:
|
|
# Fansly uses seconds
|
|
published_at = datetime.fromtimestamp(created_at).isoformat()
|
|
except (ValueError, TypeError, OSError):
|
|
pass
|
|
|
|
# Get content/description
|
|
content = post_data.get('content', '')
|
|
|
|
# Parse attachments
|
|
# Attachments are objects with contentId that maps to accountMedia or bundles
|
|
# contentType=1: single media in accountMedia
|
|
# contentType=2: media bundle in accountMediaBundles
|
|
# contentType=42001: poll (looked up in polls_dict)
|
|
attachments = []
|
|
poll_texts = []
|
|
attachment_list = post_data.get('attachments', [])
|
|
|
|
for att_obj in attachment_list:
|
|
# att_obj is a dict like: {"postId": "...", "pos": 0, "contentType": 1, "contentId": "..."}
|
|
if not isinstance(att_obj, dict):
|
|
continue
|
|
content_id = att_obj.get('contentId')
|
|
content_type = att_obj.get('contentType', 1)
|
|
if not content_id:
|
|
continue
|
|
|
|
# Handle polls (contentType 42001)
|
|
if content_type == 42001:
|
|
if polls_dict:
|
|
poll = polls_dict.get(content_id)
|
|
if poll:
|
|
poll_text = self._format_poll(poll)
|
|
if poll_text:
|
|
poll_texts.append(poll_text)
|
|
continue
|
|
|
|
# Only process known media content types
|
|
# contentType 1 = single media
|
|
# contentType 2 = media bundle
|
|
if content_type not in (1, 2):
|
|
continue
|
|
|
|
if content_type == 2:
|
|
# Bundle: look up in bundle_dict, then expand to multiple media items
|
|
bundle = bundle_dict.get(content_id, {})
|
|
account_media_ids = bundle.get('accountMediaIds', [])
|
|
for am_id in account_media_ids:
|
|
account_media = account_media_dict.get(am_id, {})
|
|
media = account_media.get('media', {})
|
|
attachment = self._parse_attachment(account_media, media)
|
|
if attachment:
|
|
attachments.append(attachment)
|
|
else:
|
|
# Single media: look up directly in accountMedia
|
|
account_media = account_media_dict.get(content_id, {})
|
|
media = account_media.get('media', {})
|
|
attachment = self._parse_attachment(account_media, media)
|
|
if attachment:
|
|
attachments.append(attachment)
|
|
|
|
# Append poll text to content if any polls were found
|
|
if poll_texts:
|
|
poll_section = '\n'.join(poll_texts)
|
|
if content:
|
|
content = content + '\n\n' + poll_section
|
|
else:
|
|
content = poll_section
|
|
|
|
return Post(
|
|
post_id=post_id,
|
|
service_id=self.SERVICE_ID,
|
|
platform=self.PLATFORM,
|
|
creator_id=account.get('account_id', ''),
|
|
title=None, # Fansly posts don't have titles
|
|
content=content,
|
|
published_at=published_at,
|
|
added_at=datetime.now().isoformat(),
|
|
attachments=attachments,
|
|
)
|
|
|
|
except Exception as e:
|
|
self.log(f"Error parsing post: {e}", 'error')
|
|
return None
|
|
|
|
def _format_poll(self, poll: Dict) -> Optional[str]:
|
|
"""Format a Fansly poll into readable text for post content."""
|
|
try:
|
|
# Fansly polls use 'title' for the question, options use 'title' and 'voteCount'
|
|
question = (poll.get('title') or poll.get('question') or '').strip()
|
|
options = poll.get('options', [])
|
|
if not options:
|
|
return None
|
|
|
|
lines = []
|
|
if question:
|
|
lines.append(f"[Poll] {question}")
|
|
else:
|
|
lines.append("[Poll]")
|
|
|
|
total_votes = sum(opt.get('voteCount', 0) or opt.get('votes', 0) for opt in options)
|
|
for opt in options:
|
|
text = (opt.get('title') or opt.get('text') or '').strip()
|
|
votes = opt.get('voteCount', 0) or opt.get('votes', 0)
|
|
if total_votes > 0:
|
|
pct = round(votes / total_votes * 100)
|
|
lines.append(f" - {text} ({pct}%, {votes} votes)")
|
|
else:
|
|
lines.append(f" - {text}")
|
|
|
|
if total_votes > 0:
|
|
lines.append(f" Total: {total_votes} votes")
|
|
|
|
return '\n'.join(lines)
|
|
except Exception:
|
|
return None
|
|
|
|
def _parse_attachment(self, account_media: Dict, media: Dict) -> Optional[Attachment]:
|
|
"""
|
|
Parse Fansly media data into an Attachment model.
|
|
|
|
Args:
|
|
account_media: Account media data (links post to media)
|
|
media: Media details (embedded in account_media)
|
|
|
|
Returns:
|
|
Attachment object or None if parsing fails
|
|
"""
|
|
try:
|
|
if not media:
|
|
return None
|
|
|
|
media_id = str(media.get('id', ''))
|
|
mimetype = media.get('mimetype', '')
|
|
|
|
# Determine file type
|
|
file_type = 'unknown'
|
|
if mimetype.startswith('image/'):
|
|
file_type = 'image'
|
|
elif mimetype.startswith('video/'):
|
|
file_type = 'video'
|
|
elif mimetype.startswith('audio/'):
|
|
file_type = 'audio'
|
|
|
|
# Get dimensions and metadata from main media object
|
|
width = media.get('width')
|
|
height = media.get('height')
|
|
file_size = None
|
|
duration = None
|
|
|
|
# Parse metadata JSON for duration and original dimensions
|
|
metadata_str = media.get('metadata', '{}')
|
|
try:
|
|
import json
|
|
metadata = json.loads(metadata_str) if metadata_str else {}
|
|
duration = metadata.get('duration')
|
|
# Use original dimensions if available (higher quality)
|
|
if metadata.get('originalWidth'):
|
|
width = metadata.get('originalWidth')
|
|
if metadata.get('originalHeight'):
|
|
height = metadata.get('originalHeight')
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
|
|
# Get signed download URL
|
|
# Priority: 1) Direct video URL (MP4/MOV) from main locations
|
|
# 2) Direct video URL from variants
|
|
# 3) Streaming manifest (m3u8/mpd) as fallback
|
|
download_url = None
|
|
|
|
# Extensions for different types
|
|
streaming_exts = ('.m3u8', '.mpd')
|
|
image_exts = ('.jpeg', '.jpg', '.png', '.gif', '.webp')
|
|
video_exts = ('.mp4', '.mov', '.webm', '.mkv', '.avi')
|
|
|
|
def url_ext(url: str) -> str:
|
|
"""Get extension from URL, ignoring query string"""
|
|
path = url.split('?')[0] if url else ''
|
|
return '.' + path.split('.')[-1].lower() if '.' in path else ''
|
|
|
|
# Check for quality options
|
|
main_locations = media.get('locations', [])
|
|
variants = media.get('variants', [])
|
|
self.log(f"Media {media_id}: {len(main_locations)} main URLs, {len(variants)} variants", 'debug')
|
|
|
|
if variants:
|
|
# Debug: log available variants at debug level
|
|
for v in variants:
|
|
v_w, v_h = v.get('width', 0), v.get('height', 0)
|
|
v_locs = v.get('locations', [])
|
|
for loc in v_locs:
|
|
loc_url = loc.get('location', '')
|
|
ext = url_ext(loc_url)
|
|
self.log(f" Variant {v_w}x{v_h}: ext={ext}", 'debug')
|
|
|
|
# Sort by resolution (highest first)
|
|
sorted_variants = sorted(
|
|
variants,
|
|
key=lambda v: (v.get('width', 0) or 0) * (v.get('height', 0) or 0),
|
|
reverse=True
|
|
)
|
|
|
|
# For videos: PRIORITIZE highest resolution, even if streaming
|
|
# 1) First look for highest res direct video in variants
|
|
# 2) Then highest res streaming (m3u8/mpd) - this is often 4K
|
|
# 3) Fall back to main locations (often lower res direct)
|
|
if file_type == 'video':
|
|
# Check for direct video at highest res
|
|
for variant in sorted_variants:
|
|
var_locations = variant.get('locations', [])
|
|
for loc in var_locations:
|
|
url = loc.get('location', '')
|
|
ext = url_ext(url)
|
|
if url and ext in video_exts:
|
|
download_url = url
|
|
var_width = variant.get('width')
|
|
var_height = variant.get('height')
|
|
if var_width and var_height:
|
|
self.log(f"Selected: {var_width}x{var_height} direct video from variants", 'info')
|
|
width = var_width
|
|
height = var_height
|
|
break
|
|
if download_url:
|
|
break
|
|
|
|
# If no direct video, use streaming at highest res (4K usually)
|
|
if not download_url:
|
|
for variant in sorted_variants:
|
|
var_locations = variant.get('locations', [])
|
|
for loc in var_locations:
|
|
url = loc.get('location', '')
|
|
ext = url_ext(url)
|
|
if url and ext in streaming_exts:
|
|
# Get CloudFront signed URL params from metadata
|
|
loc_metadata = loc.get('metadata', {})
|
|
if loc_metadata:
|
|
# Append signed URL params: Key-Pair-Id, Signature, Policy
|
|
params = []
|
|
for key in ['Key-Pair-Id', 'Signature', 'Policy']:
|
|
if key in loc_metadata:
|
|
params.append(f"{key}={loc_metadata[key]}")
|
|
if params:
|
|
separator = '&' if '?' in url else '?'
|
|
url = url + separator + '&'.join(params)
|
|
self.log(f"Constructed signed streaming URL with {len(params)} params", 'info')
|
|
|
|
download_url = url
|
|
var_width = variant.get('width')
|
|
var_height = variant.get('height')
|
|
stream_type = 'HLS' if ext == '.m3u8' else 'DASH'
|
|
if var_width and var_height:
|
|
self.log(f"Selected: {var_width}x{var_height} {stream_type} stream (highest quality)", 'info')
|
|
width = var_width
|
|
height = var_height
|
|
break
|
|
if download_url:
|
|
break
|
|
|
|
# Fall back to main locations (often 720p direct)
|
|
if not download_url:
|
|
for loc in main_locations:
|
|
url = loc.get('location', '')
|
|
ext = url_ext(url)
|
|
if url and ext in video_exts:
|
|
download_url = url
|
|
self.log(f"Fallback: direct video from main locations (lower res)", 'info')
|
|
break
|
|
|
|
# For images: prefer main locations (original full resolution)
|
|
# Variants are scaled-down thumbnails (1080p max)
|
|
if file_type == 'image' and not download_url:
|
|
# First check main locations for original resolution
|
|
for loc in main_locations:
|
|
url = loc.get('location', '')
|
|
ext = url_ext(url)
|
|
if url and ext in image_exts:
|
|
download_url = url
|
|
# Use dimensions from media object (original size)
|
|
orig_width = media.get('width')
|
|
orig_height = media.get('height')
|
|
if orig_width and orig_height:
|
|
self.log(f"Selected: {orig_width}x{orig_height} original image", 'info')
|
|
width = orig_width
|
|
height = orig_height
|
|
break
|
|
|
|
# Fallback to variants if main locations don't have image
|
|
if not download_url:
|
|
for variant in sorted_variants:
|
|
var_locations = variant.get('locations', [])
|
|
for loc in var_locations:
|
|
url = loc.get('location', '')
|
|
ext = url_ext(url)
|
|
if url and ext in image_exts:
|
|
download_url = url
|
|
var_width = variant.get('width')
|
|
var_height = variant.get('height')
|
|
if var_width and var_height:
|
|
self.log(f"Selected: {var_width}x{var_height} image (variant)", 'info')
|
|
width = var_width
|
|
height = var_height
|
|
break
|
|
if download_url:
|
|
break
|
|
|
|
# Final fallback: any URL from main locations
|
|
if not download_url and main_locations:
|
|
for loc in main_locations:
|
|
url = loc.get('location', '')
|
|
if url:
|
|
download_url = url
|
|
self.log(f"Final fallback: {url[:80]}...", 'info')
|
|
break
|
|
|
|
# Fallback: relative path (won't work for download but useful for reference)
|
|
location_path = media.get('location', '')
|
|
|
|
# Extract filename and extension first (needed for placeholder too)
|
|
ext = ''
|
|
if '/' in mimetype:
|
|
ext = mimetype.split('/')[-1]
|
|
if ext == 'jpeg':
|
|
ext = 'jpg'
|
|
elif ext == 'quicktime':
|
|
ext = 'mov'
|
|
|
|
filename = f"{media_id}.{ext}" if ext else media_id
|
|
|
|
# Log if this is PPV content (no download URL available)
|
|
if not download_url:
|
|
self.log(f"PPV/locked content detected: {filename} - creating placeholder for manual import", 'info')
|
|
|
|
# Flag videos below 4K for quality recheck (Fansly may still be processing)
|
|
needs_recheck = False
|
|
if file_type == 'video' and download_url and width and height:
|
|
is_4k = (width >= 3840 and height >= 2160) or (width >= 2160 and height >= 3840)
|
|
if not is_4k:
|
|
needs_recheck = True
|
|
|
|
return Attachment(
|
|
name=filename,
|
|
server_path=location_path, # Relative path for reference
|
|
file_type=file_type,
|
|
extension=ext if ext else None,
|
|
download_url=download_url, # None for PPV content - will need manual import
|
|
file_size=file_size,
|
|
width=width,
|
|
height=height,
|
|
duration=duration,
|
|
needs_quality_recheck=needs_recheck,
|
|
)
|
|
|
|
except Exception as e:
|
|
self.log(f"Error parsing attachment: {e}", 'error')
|
|
return None
|
|
|
|
# ==================== MESSAGES ====================
|
|
|
|
async def get_chat_list(self) -> List[Dict]:
|
|
"""
|
|
Get list of messaging conversations.
|
|
|
|
Uses GET /messaging/groups to list all chat groups.
|
|
|
|
Returns:
|
|
List of dicts with 'group_id' and 'partner_account_id' keys
|
|
"""
|
|
self._delay_between_items()
|
|
try:
|
|
session = await self._get_session()
|
|
url = f"{self.BASE_URL}/messaging/groups"
|
|
params = {'ngsw-bypass': 'true'}
|
|
|
|
async with session.get(url, params=params) as resp:
|
|
if resp.status == 200:
|
|
data = await resp.json()
|
|
response = data.get('response', {})
|
|
groups = response.get('groups', [])
|
|
|
|
results = []
|
|
for group in groups:
|
|
group_id = group.get('id')
|
|
# partnerAccountId is the other user in the conversation
|
|
partner_ids = group.get('users', [])
|
|
partner_id = None
|
|
for uid in partner_ids:
|
|
# The partner is the one that's not us
|
|
if uid != group.get('createdBy'):
|
|
partner_id = uid
|
|
break
|
|
if not partner_id and partner_ids:
|
|
partner_id = partner_ids[0]
|
|
|
|
if group_id:
|
|
results.append({
|
|
'group_id': group_id,
|
|
'partner_account_id': partner_id,
|
|
})
|
|
|
|
self.log(f"Found {len(results)} chat groups", 'info')
|
|
return results
|
|
|
|
elif resp.status == 401:
|
|
self.log("Auth token expired or invalid", 'error')
|
|
return []
|
|
else:
|
|
self.log(f"Error fetching chat list: HTTP {resp.status}", 'warning')
|
|
return []
|
|
|
|
except Exception as e:
|
|
self.log(f"Error fetching chat list: {e}", 'error')
|
|
return []
|
|
|
|
async def get_messages(self, group_id: str, creator_account_id: str,
|
|
max_messages: int = 500) -> List[Message]:
|
|
"""
|
|
Fetch messages from a conversation.
|
|
|
|
Uses GET /message?groupId={id}&limit=50 with cursor-based pagination.
|
|
|
|
Args:
|
|
group_id: Fansly messaging group ID
|
|
creator_account_id: Fansly account ID of the creator (to determine direction)
|
|
max_messages: Maximum number of messages to fetch
|
|
|
|
Returns:
|
|
List of Message objects
|
|
"""
|
|
messages = []
|
|
cursor = None
|
|
page = 0
|
|
|
|
while len(messages) < max_messages:
|
|
page += 1
|
|
self._delay_between_items()
|
|
|
|
try:
|
|
session = await self._get_session()
|
|
params = {
|
|
'groupId': group_id,
|
|
'limit': 50,
|
|
'ngsw-bypass': 'true',
|
|
}
|
|
if cursor:
|
|
params['before'] = cursor
|
|
|
|
url = f"{self.BASE_URL}/message"
|
|
async with session.get(url, params=params) as resp:
|
|
if resp.status != 200:
|
|
self.log(f"Error fetching messages: HTTP {resp.status}", 'warning')
|
|
break
|
|
|
|
data = await resp.json()
|
|
response = data.get('response', {})
|
|
|
|
msg_list = response.get('messages', [])
|
|
if not msg_list:
|
|
break
|
|
|
|
# Build media lookup dicts (same structure as timeline)
|
|
account_media_dict = {}
|
|
for am in response.get('accountMedia', []):
|
|
account_media_dict[am.get('id')] = am
|
|
|
|
bundle_dict = {}
|
|
for bundle in response.get('accountMediaBundles', []):
|
|
bundle_dict[bundle.get('id')] = bundle
|
|
|
|
for msg_data in msg_list:
|
|
msg = self._parse_fansly_message(
|
|
msg_data, creator_account_id,
|
|
account_media_dict, bundle_dict
|
|
)
|
|
if msg:
|
|
messages.append(msg)
|
|
|
|
self.log(f"Fetched page {page}: {len(msg_list)} messages (total: {len(messages)})", 'debug')
|
|
|
|
if len(msg_list) < 50:
|
|
break # Last page
|
|
|
|
# Cursor for next page
|
|
last_msg = msg_list[-1]
|
|
next_cursor = str(last_msg.get('id', ''))
|
|
if next_cursor and next_cursor != str(cursor):
|
|
cursor = next_cursor
|
|
else:
|
|
break
|
|
|
|
except Exception as e:
|
|
self.log(f"Error fetching messages page {page}: {e}", 'error')
|
|
break
|
|
|
|
self.log(f"Fetched {len(messages)} messages for group {group_id}", 'info')
|
|
return messages
|
|
|
|
def _parse_fansly_message(self, msg_data: Dict, creator_account_id: str,
|
|
account_media_dict: Dict, bundle_dict: Dict) -> Optional[Message]:
|
|
"""
|
|
Parse a Fansly message into a Message model.
|
|
|
|
Args:
|
|
msg_data: Raw message dict from API
|
|
creator_account_id: Fansly account ID of the creator
|
|
account_media_dict: Media lookup dict from response
|
|
bundle_dict: Bundle lookup dict from response
|
|
|
|
Returns:
|
|
Message object or None
|
|
"""
|
|
try:
|
|
msg_id = str(msg_data.get('id', ''))
|
|
if not msg_id:
|
|
return None
|
|
|
|
sender_id = str(msg_data.get('senderId', ''))
|
|
is_from_creator = (sender_id == str(creator_account_id))
|
|
|
|
text = msg_data.get('content', '')
|
|
|
|
# Parse timestamp (Fansly uses seconds or milliseconds)
|
|
created_at = msg_data.get('createdAt', 0)
|
|
sent_at = None
|
|
if created_at:
|
|
try:
|
|
ts = created_at
|
|
if ts > 1000000000000:
|
|
ts = ts / 1000
|
|
sent_at = datetime.fromtimestamp(ts).isoformat()
|
|
except (ValueError, TypeError, OSError):
|
|
pass
|
|
|
|
# PPV/price info
|
|
price = msg_data.get('price')
|
|
is_tip = msg_data.get('isTip', False)
|
|
tip_amount = msg_data.get('tipAmount')
|
|
|
|
# Parse attachments - same pattern as posts
|
|
attachments = []
|
|
attachment_list = msg_data.get('attachments', []) or []
|
|
for att_obj in attachment_list:
|
|
if not isinstance(att_obj, dict):
|
|
continue
|
|
content_id = att_obj.get('contentId')
|
|
content_type = att_obj.get('contentType', 1)
|
|
if not content_id:
|
|
continue
|
|
if content_type not in (1, 2):
|
|
continue
|
|
|
|
if content_type == 2:
|
|
bundle = bundle_dict.get(content_id, {})
|
|
for am_id in bundle.get('accountMediaIds', []):
|
|
account_media = account_media_dict.get(am_id, {})
|
|
media = account_media.get('media', {})
|
|
att = self._parse_attachment(account_media, media)
|
|
if att:
|
|
attachments.append(att)
|
|
else:
|
|
account_media = account_media_dict.get(content_id, {})
|
|
media = account_media.get('media', {})
|
|
att = self._parse_attachment(account_media, media)
|
|
if att:
|
|
attachments.append(att)
|
|
|
|
return Message(
|
|
message_id=msg_id,
|
|
platform=self.PLATFORM,
|
|
service_id=self.SERVICE_ID,
|
|
creator_id=str(creator_account_id),
|
|
text=text if text else None,
|
|
sent_at=sent_at,
|
|
is_from_creator=is_from_creator,
|
|
is_tip=bool(is_tip),
|
|
tip_amount=float(tip_amount) if tip_amount else None,
|
|
price=float(price) if price else None,
|
|
is_free=price is None or price == 0,
|
|
is_purchased=False,
|
|
attachments=attachments,
|
|
)
|
|
|
|
except Exception as e:
|
|
self.log(f"Error parsing fansly message: {e}", 'error')
|
|
return None
|
|
|
|
async def recheck_attachment_quality(self, attachment_id: int, db) -> Dict:
|
|
"""
|
|
Recheck a single attachment for higher quality variants via the Fansly API.
|
|
Returns: {'upgraded': bool, 'new_width': int, 'new_height': int, 'old_width': int, 'old_height': int, 'creator_id': int}
|
|
"""
|
|
att = db.get_attachment(attachment_id)
|
|
if not att:
|
|
return {'upgraded': False, 'error': 'Attachment not found'}
|
|
|
|
post = db.get_post(att['post_id'])
|
|
if not post:
|
|
return {'upgraded': False, 'error': 'Post not found'}
|
|
|
|
creator = db.get_creator(post['creator_id'])
|
|
if not creator:
|
|
return {'upgraded': False, 'error': 'Creator not found'}
|
|
|
|
account_id = creator.get('creator_id')
|
|
post_id = post.get('post_id')
|
|
old_width = att.get('width') or 0
|
|
old_height = att.get('height') or 0
|
|
media_id = att.get('name', '').replace('.mp4', '').replace('.mov', '').replace('.jpg', '').replace('.png', '')
|
|
|
|
try:
|
|
# Fetch timeline page containing this post
|
|
posts, _, media_dict, account_media_dict, bundle_dict, _, _ = await self._fetch_timeline_page(
|
|
account_id=account_id,
|
|
cursor=str(int(post_id) + 1)
|
|
)
|
|
|
|
best_width = old_width
|
|
best_height = old_height
|
|
best_url = None
|
|
|
|
# Search account_media_dict for matching media
|
|
for am_id, am_data in account_media_dict.items():
|
|
media = am_data.get('media', {})
|
|
if str(media.get('id')) == media_id:
|
|
variants = media.get('variants', [])
|
|
for v in variants:
|
|
v_w = v.get('width', 0) or 0
|
|
v_h = v.get('height', 0) or 0
|
|
if v_w * v_h > best_width * best_height:
|
|
for loc in v.get('locations', []):
|
|
loc_url = loc.get('location', '')
|
|
if '.m3u8' in loc_url or '.mp4' in loc_url or '.mov' in loc_url:
|
|
best_width = v_w
|
|
best_height = v_h
|
|
metadata = loc.get('metadata', {})
|
|
if metadata:
|
|
params = []
|
|
for key in ['Key-Pair-Id', 'Signature', 'Policy']:
|
|
if key in metadata:
|
|
params.append(f"{key}={metadata[key]}")
|
|
best_url = loc_url + ('?' + '&'.join(params) if params else '')
|
|
else:
|
|
best_url = loc_url
|
|
break
|
|
break
|
|
|
|
# Update quality check tracking
|
|
now = datetime.now().isoformat()
|
|
recheck_count = (att.get('quality_recheck_count') or 0) + 1
|
|
|
|
if best_url and (best_width > old_width or best_height > old_height):
|
|
# Upgrade found — update attachment
|
|
with db.get_connection(for_write=True) as conn:
|
|
conn.execute("""
|
|
UPDATE paid_content_attachments
|
|
SET download_url = ?, width = ?, height = ?,
|
|
status = 'pending', download_attempts = 0, error_message = NULL,
|
|
local_path = NULL, local_filename = NULL, file_hash = NULL,
|
|
needs_quality_recheck = 0, last_quality_check = ?, quality_recheck_count = ?
|
|
WHERE id = ?
|
|
""", (best_url, best_width, best_height, now, recheck_count, attachment_id))
|
|
conn.commit()
|
|
|
|
self.log(f"Quality upgrade: {att.get('name')} {old_width}x{old_height} -> {best_width}x{best_height}", 'info')
|
|
return {
|
|
'upgraded': True,
|
|
'old_width': old_width, 'old_height': old_height,
|
|
'new_width': best_width, 'new_height': best_height,
|
|
'creator_id': post['creator_id']
|
|
}
|
|
else:
|
|
# No upgrade found
|
|
with db.get_connection(for_write=True) as conn:
|
|
conn.execute("""
|
|
UPDATE paid_content_attachments
|
|
SET last_quality_check = ?, quality_recheck_count = ?
|
|
WHERE id = ?
|
|
""", (now, recheck_count, attachment_id))
|
|
conn.commit()
|
|
|
|
return {
|
|
'upgraded': False,
|
|
'old_width': old_width, 'old_height': old_height,
|
|
'creator_id': post['creator_id']
|
|
}
|
|
|
|
except Exception as e:
|
|
self.log(f"Quality recheck error for attachment {attachment_id}: {e}", 'error')
|
|
return {'upgraded': False, 'error': str(e)}
|