503 lines
17 KiB
Python
503 lines
17 KiB
Python
"""
|
|
Easynews Client Module
|
|
|
|
Provides a client for interacting with the Easynews API to search for and download files.
|
|
All connections use HTTPS with HTTP Basic Auth.
|
|
"""
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
from urllib.parse import quote, urljoin
|
|
|
|
import requests
|
|
from requests.auth import HTTPBasicAuth
|
|
|
|
from modules.universal_logger import get_logger
|
|
|
|
logger = get_logger('EasynewsClient')
|
|
|
|
|
|
@dataclass
|
|
class EasynewsResult:
|
|
"""Represents a single search result from Easynews."""
|
|
filename: str
|
|
download_url: str
|
|
size_bytes: int
|
|
post_date: Optional[str]
|
|
subject: Optional[str]
|
|
poster: Optional[str]
|
|
newsgroup: Optional[str]
|
|
extension: Optional[str]
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
'filename': self.filename,
|
|
'download_url': self.download_url,
|
|
'size_bytes': self.size_bytes,
|
|
'post_date': self.post_date,
|
|
'subject': self.subject,
|
|
'poster': self.poster,
|
|
'newsgroup': self.newsgroup,
|
|
'extension': self.extension,
|
|
}
|
|
|
|
|
|
class EasynewsClient:
|
|
"""
|
|
Client for interacting with Easynews search and download APIs.
|
|
|
|
All connections use HTTPS with HTTP Basic Auth.
|
|
Supports HTTP, HTTPS, SOCKS4, and SOCKS5 proxies.
|
|
"""
|
|
|
|
BASE_URL = "https://members.easynews.com"
|
|
SEARCH_URL = "https://members.easynews.com/2.0/search/solr-search/advanced"
|
|
|
|
# Quality patterns for parsing
|
|
QUALITY_PATTERNS = [
|
|
(r'2160p|4k|uhd', '2160p'),
|
|
(r'1080p|fhd', '1080p'),
|
|
(r'720p|hd', '720p'),
|
|
(r'480p|sd', '480p'),
|
|
(r'360p', '360p'),
|
|
]
|
|
|
|
# Audio codec patterns (order matters - check combinations first)
|
|
AUDIO_PATTERNS = [
|
|
(r'truehd.*atmos|atmos.*truehd', 'Atmos'),
|
|
(r'atmos', 'Atmos'),
|
|
(r'truehd', 'TrueHD'),
|
|
(r'dts[\.\-]?hd[\.\-]?ma', 'DTS-HD'),
|
|
(r'dts[\.\-]?hd', 'DTS-HD'),
|
|
(r'dts[\.\-]?x', 'DTS:X'),
|
|
(r'dts', 'DTS'),
|
|
(r'7[\.\-]?1', '7.1'),
|
|
(r'ddp[\.\-\s]?5[\.\-]?1|eac3|e[\.\-]?ac[\.\-]?3|dd[\.\-]?5[\.\-]?1|ac3|5[\.\-]?1', '5.1'),
|
|
(r'ddp|dd\+', '5.1'),
|
|
(r'aac[\.\-]?5[\.\-]?1', '5.1'),
|
|
(r'aac', 'AAC'),
|
|
(r'flac', 'FLAC'),
|
|
(r'mp3', 'MP3'),
|
|
]
|
|
|
|
# Source/release type patterns
|
|
SOURCE_PATTERNS = [
|
|
(r'remux', 'Remux'),
|
|
(r'blu[\.\-]?ray|bdrip|brrip', 'BluRay'),
|
|
(r'web[\.\-]?dl', 'WEB-DL'),
|
|
(r'webrip', 'WEBRip'),
|
|
(r'web', 'WEB'),
|
|
(r'hdtv', 'HDTV'),
|
|
(r'dvdrip', 'DVDRip'),
|
|
(r'dvd', 'DVD'),
|
|
(r'hdcam|cam', 'CAM'),
|
|
]
|
|
|
|
def __init__(
|
|
self,
|
|
username: str,
|
|
password: str,
|
|
proxy_enabled: bool = False,
|
|
proxy_type: str = 'http',
|
|
proxy_host: Optional[str] = None,
|
|
proxy_port: Optional[int] = None,
|
|
proxy_username: Optional[str] = None,
|
|
proxy_password: Optional[str] = None,
|
|
):
|
|
"""
|
|
Initialize the Easynews client.
|
|
|
|
Args:
|
|
username: Easynews username
|
|
password: Easynews password
|
|
proxy_enabled: Whether to use a proxy
|
|
proxy_type: Proxy type (http, https, socks4, socks5)
|
|
proxy_host: Proxy hostname/IP
|
|
proxy_port: Proxy port
|
|
proxy_username: Proxy auth username (optional)
|
|
proxy_password: Proxy auth password (optional)
|
|
"""
|
|
self.username = username
|
|
self.password = password
|
|
self.auth = HTTPBasicAuth(username, password)
|
|
|
|
# Set up session with retry logic
|
|
self.session = requests.Session()
|
|
self.session.auth = self.auth
|
|
|
|
# Configure proxy if enabled
|
|
self.proxies = {}
|
|
if proxy_enabled and proxy_host and proxy_port:
|
|
proxy_url = self._build_proxy_url(
|
|
proxy_type, proxy_host, proxy_port,
|
|
proxy_username, proxy_password
|
|
)
|
|
self.proxies = {
|
|
'http': proxy_url,
|
|
'https': proxy_url,
|
|
}
|
|
self.session.proxies.update(self.proxies)
|
|
logger.info(f"Easynews client configured with {proxy_type} proxy: {proxy_host}:{proxy_port}")
|
|
|
|
def _build_proxy_url(
|
|
self,
|
|
proxy_type: str,
|
|
host: str,
|
|
port: int,
|
|
username: Optional[str] = None,
|
|
password: Optional[str] = None,
|
|
) -> str:
|
|
"""Build a proxy URL with optional authentication."""
|
|
scheme = proxy_type.lower()
|
|
if scheme not in ('http', 'https', 'socks4', 'socks5'):
|
|
scheme = 'http'
|
|
|
|
if username and password:
|
|
return f"{scheme}://{quote(username)}:{quote(password)}@{host}:{port}"
|
|
return f"{scheme}://{host}:{port}"
|
|
|
|
def test_connection(self) -> Dict[str, Any]:
|
|
"""
|
|
Test the connection to Easynews with current credentials.
|
|
|
|
Returns:
|
|
Dict with 'success' bool and 'message' string
|
|
"""
|
|
try:
|
|
# Try to access the members area
|
|
response = self.session.get(
|
|
f"{self.BASE_URL}/",
|
|
timeout=30,
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
# Check if we're actually authenticated (not redirected to login)
|
|
if 'login' in response.url.lower() or 'sign in' in response.text.lower():
|
|
return {
|
|
'success': False,
|
|
'message': 'Invalid credentials - authentication failed'
|
|
}
|
|
return {
|
|
'success': True,
|
|
'message': 'Successfully connected to Easynews'
|
|
}
|
|
elif response.status_code == 401:
|
|
return {
|
|
'success': False,
|
|
'message': 'Invalid credentials - authentication failed'
|
|
}
|
|
else:
|
|
return {
|
|
'success': False,
|
|
'message': f'Unexpected response: HTTP {response.status_code}'
|
|
}
|
|
except requests.exceptions.ProxyError as e:
|
|
return {
|
|
'success': False,
|
|
'message': f'Proxy connection failed: {str(e)}'
|
|
}
|
|
except requests.exceptions.ConnectionError as e:
|
|
return {
|
|
'success': False,
|
|
'message': f'Connection failed: {str(e)}'
|
|
}
|
|
except requests.exceptions.Timeout:
|
|
return {
|
|
'success': False,
|
|
'message': 'Connection timed out'
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Easynews connection test failed: {e}")
|
|
return {
|
|
'success': False,
|
|
'message': f'Connection test failed: {str(e)}'
|
|
}
|
|
|
|
def search(
|
|
self,
|
|
query: str,
|
|
page: int = 1,
|
|
results_per_page: int = 50,
|
|
file_types: Optional[List[str]] = None,
|
|
) -> List[EasynewsResult]:
|
|
"""
|
|
Search Easynews for files matching the query.
|
|
|
|
Args:
|
|
query: Search query string
|
|
page: Page number (1-indexed)
|
|
results_per_page: Number of results per page (max 250)
|
|
file_types: Optional list of file extensions to filter (e.g., ['mkv', 'mp4'])
|
|
|
|
Returns:
|
|
List of EasynewsResult objects
|
|
"""
|
|
try:
|
|
# Build search parameters
|
|
params = {
|
|
'gps': query,
|
|
'pby': min(results_per_page, 250),
|
|
'pno': page,
|
|
'sS': 1, # Safe search off
|
|
'saession': '', # Session
|
|
'sb': 1, # Sort by date
|
|
'sbj': 1, # Subject search
|
|
'fly': 2, # File type filter mode
|
|
'fex': 'mkv,mp4', # Only mkv and mp4 files
|
|
}
|
|
|
|
# Add file type filter if specified
|
|
if file_types:
|
|
params['fty[]'] = file_types
|
|
else:
|
|
# Default to video file types
|
|
params['fty[]'] = ['VIDEO']
|
|
|
|
response = self.session.get(
|
|
self.SEARCH_URL,
|
|
params=params,
|
|
timeout=60,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
# Check for empty response
|
|
if not response.content or not response.content.strip():
|
|
logger.warning(f"Easynews search for '{query}' returned empty response (HTTP {response.status_code})")
|
|
return []
|
|
|
|
try:
|
|
data = response.json()
|
|
except (ValueError, Exception) as json_err:
|
|
logger.warning(f"Easynews search for '{query}' returned invalid JSON (HTTP {response.status_code}, body: {response.text[:200]}): {json_err}")
|
|
return []
|
|
results = []
|
|
|
|
# Parse the response
|
|
if 'data' in data and isinstance(data['data'], list):
|
|
for item in data['data']:
|
|
result = self._parse_search_result(item)
|
|
if result:
|
|
results.append(result)
|
|
|
|
logger.info(f"Easynews search for '{query}' returned {len(results)} results")
|
|
return results
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"Easynews search failed: {e}")
|
|
return []
|
|
except Exception as e:
|
|
logger.error(f"Error parsing Easynews search results: {e}")
|
|
return []
|
|
|
|
def _parse_search_result(self, item: Dict[str, Any]) -> Optional[EasynewsResult]:
|
|
"""Parse a single search result from the API response."""
|
|
try:
|
|
# Extract the filename
|
|
filename = item.get('fn', '') or item.get('0', '')
|
|
if not filename:
|
|
return None
|
|
|
|
# Build download URL
|
|
# Format: https://username:password@members.easynews.com/dl/{hash}/{filename}
|
|
file_hash = item.get('hash', '') or item.get('0', '')
|
|
sig = item.get('sig', '')
|
|
|
|
if file_hash and sig:
|
|
# Use the authenticated download URL format
|
|
download_path = f"/dl/{file_hash}/{quote(filename)}?sig={sig}"
|
|
download_url = f"https://{quote(self.username)}:{quote(self.password)}@members.easynews.com{download_path}"
|
|
else:
|
|
# Fallback to basic URL
|
|
download_url = item.get('url', '') or item.get('rawURL', '')
|
|
if download_url and not download_url.startswith('http'):
|
|
download_url = urljoin(self.BASE_URL, download_url)
|
|
|
|
if not download_url:
|
|
return None
|
|
|
|
# Parse size
|
|
size_bytes = 0
|
|
size_str = item.get('rawSize', '') or item.get('size', '')
|
|
if isinstance(size_str, (int, float)):
|
|
size_bytes = int(size_str)
|
|
elif isinstance(size_str, str):
|
|
size_bytes = self._parse_size(size_str)
|
|
|
|
# Parse date
|
|
post_date = item.get('date', '') or item.get('d', '')
|
|
if post_date:
|
|
try:
|
|
# Try to parse and standardize the date format
|
|
if isinstance(post_date, str):
|
|
post_date = post_date.strip()
|
|
except Exception:
|
|
pass
|
|
|
|
# Get extension from API field (more reliable than parsing filename)
|
|
extension = item.get('extension', '') or item.get('11', '') or item.get('2', '')
|
|
if extension and not extension.startswith('.'):
|
|
extension = '.' + extension
|
|
|
|
return EasynewsResult(
|
|
filename=filename,
|
|
download_url=download_url,
|
|
size_bytes=size_bytes,
|
|
post_date=post_date if post_date else None,
|
|
subject=item.get('subject', '') or item.get('s', ''),
|
|
poster=item.get('poster', '') or item.get('p', ''),
|
|
newsgroup=item.get('newsgroup', '') or item.get('ng', ''),
|
|
extension=extension if extension else self._get_extension(filename),
|
|
)
|
|
except Exception as e:
|
|
logger.debug(f"Failed to parse search result: {e}")
|
|
return None
|
|
|
|
def _parse_size(self, size_str: str) -> int:
|
|
"""Parse a size string like '1.5 GB' to bytes."""
|
|
try:
|
|
size_str = size_str.strip().upper()
|
|
multipliers = {
|
|
'B': 1,
|
|
'KB': 1024,
|
|
'MB': 1024 ** 2,
|
|
'GB': 1024 ** 3,
|
|
'TB': 1024 ** 4,
|
|
}
|
|
|
|
for suffix, multiplier in multipliers.items():
|
|
if size_str.endswith(suffix):
|
|
value = float(size_str[:-len(suffix)].strip())
|
|
return int(value * multiplier)
|
|
|
|
# Try to parse as plain number
|
|
return int(float(size_str))
|
|
except Exception:
|
|
return 0
|
|
|
|
def _get_extension(self, filename: str) -> Optional[str]:
|
|
"""Extract file extension from filename."""
|
|
if '.' in filename:
|
|
return filename.rsplit('.', 1)[-1].lower()
|
|
return None
|
|
|
|
@staticmethod
|
|
def detect_quality(filename: str) -> Optional[str]:
|
|
"""Detect video quality from filename."""
|
|
filename_lower = filename.lower()
|
|
for pattern, quality in EasynewsClient.QUALITY_PATTERNS:
|
|
if re.search(pattern, filename_lower):
|
|
return quality
|
|
return None
|
|
|
|
@staticmethod
|
|
def detect_audio(filename: str) -> Optional[str]:
|
|
"""Detect audio codec from filename."""
|
|
filename_lower = filename.lower()
|
|
for pattern, audio in EasynewsClient.AUDIO_PATTERNS:
|
|
if re.search(pattern, filename_lower):
|
|
return audio
|
|
return None
|
|
|
|
@staticmethod
|
|
def detect_source(filename: str) -> Optional[str]:
|
|
"""Detect source/release type from filename."""
|
|
filename_lower = filename.lower()
|
|
for pattern, source in EasynewsClient.SOURCE_PATTERNS:
|
|
if re.search(pattern, filename_lower):
|
|
return source
|
|
return None
|
|
|
|
def download_file(
|
|
self,
|
|
url: str,
|
|
dest_path: str,
|
|
progress_callback: Optional[Callable[[int, int], None]] = None,
|
|
chunk_size: int = 8192,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Download a file from Easynews.
|
|
|
|
Args:
|
|
url: Download URL (with authentication embedded or using session)
|
|
dest_path: Destination file path
|
|
progress_callback: Optional callback(downloaded_bytes, total_bytes)
|
|
chunk_size: Download chunk size in bytes
|
|
|
|
Returns:
|
|
Dict with 'success' bool and 'message' or 'path'
|
|
"""
|
|
try:
|
|
# Start the download with streaming
|
|
response = self.session.get(
|
|
url,
|
|
stream=True,
|
|
timeout=30,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
total_size = int(response.headers.get('content-length', 0))
|
|
downloaded = 0
|
|
|
|
with open(dest_path, 'wb') as f:
|
|
for chunk in response.iter_content(chunk_size=chunk_size):
|
|
if chunk:
|
|
f.write(chunk)
|
|
downloaded += len(chunk)
|
|
if progress_callback:
|
|
progress_callback(downloaded, total_size)
|
|
|
|
logger.info(f"Downloaded file to {dest_path} ({downloaded} bytes)")
|
|
return {
|
|
'success': True,
|
|
'path': dest_path,
|
|
'size': downloaded,
|
|
}
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"Download failed: {e}")
|
|
return {
|
|
'success': False,
|
|
'message': f'Download failed: {str(e)}'
|
|
}
|
|
except IOError as e:
|
|
logger.error(f"Failed to write file: {e}")
|
|
return {
|
|
'success': False,
|
|
'message': f'Failed to write file: {str(e)}'
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error during download: {e}")
|
|
return {
|
|
'success': False,
|
|
'message': f'Download error: {str(e)}'
|
|
}
|
|
|
|
def get_file_info(self, url: str) -> Dict[str, Any]:
|
|
"""
|
|
Get information about a file without downloading it.
|
|
|
|
Args:
|
|
url: File URL
|
|
|
|
Returns:
|
|
Dict with file information (size, content-type, etc.)
|
|
"""
|
|
try:
|
|
response = self.session.head(url, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
return {
|
|
'success': True,
|
|
'size': int(response.headers.get('content-length', 0)),
|
|
'content_type': response.headers.get('content-type', ''),
|
|
'last_modified': response.headers.get('last-modified', ''),
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Failed to get file info: {e}")
|
|
return {
|
|
'success': False,
|
|
'message': str(e)
|
|
}
|