Todd
502
modules/easynews_client.py
Normal file
502
modules/easynews_client.py
Normal file
@@ -0,0 +1,502 @@
|
||||
"""
|
||||
Easynews Client Module
|
||||
|
||||
Provides a client for interacting with the Easynews API to search for and download files.
|
||||
All connections use HTTPS with HTTP Basic Auth.
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
import requests
|
||||
from requests.auth import HTTPBasicAuth
|
||||
|
||||
from modules.universal_logger import get_logger
|
||||
|
||||
logger = get_logger('EasynewsClient')
|
||||
|
||||
|
||||
@dataclass
|
||||
class EasynewsResult:
|
||||
"""Represents a single search result from Easynews."""
|
||||
filename: str
|
||||
download_url: str
|
||||
size_bytes: int
|
||||
post_date: Optional[str]
|
||||
subject: Optional[str]
|
||||
poster: Optional[str]
|
||||
newsgroup: Optional[str]
|
||||
extension: Optional[str]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'filename': self.filename,
|
||||
'download_url': self.download_url,
|
||||
'size_bytes': self.size_bytes,
|
||||
'post_date': self.post_date,
|
||||
'subject': self.subject,
|
||||
'poster': self.poster,
|
||||
'newsgroup': self.newsgroup,
|
||||
'extension': self.extension,
|
||||
}
|
||||
|
||||
|
||||
class EasynewsClient:
|
||||
"""
|
||||
Client for interacting with Easynews search and download APIs.
|
||||
|
||||
All connections use HTTPS with HTTP Basic Auth.
|
||||
Supports HTTP, HTTPS, SOCKS4, and SOCKS5 proxies.
|
||||
"""
|
||||
|
||||
BASE_URL = "https://members.easynews.com"
|
||||
SEARCH_URL = "https://members.easynews.com/2.0/search/solr-search/advanced"
|
||||
|
||||
# Quality patterns for parsing
|
||||
QUALITY_PATTERNS = [
|
||||
(r'2160p|4k|uhd', '2160p'),
|
||||
(r'1080p|fhd', '1080p'),
|
||||
(r'720p|hd', '720p'),
|
||||
(r'480p|sd', '480p'),
|
||||
(r'360p', '360p'),
|
||||
]
|
||||
|
||||
# Audio codec patterns (order matters - check combinations first)
|
||||
AUDIO_PATTERNS = [
|
||||
(r'truehd.*atmos|atmos.*truehd', 'Atmos'),
|
||||
(r'atmos', 'Atmos'),
|
||||
(r'truehd', 'TrueHD'),
|
||||
(r'dts[\.\-]?hd[\.\-]?ma', 'DTS-HD'),
|
||||
(r'dts[\.\-]?hd', 'DTS-HD'),
|
||||
(r'dts[\.\-]?x', 'DTS:X'),
|
||||
(r'dts', 'DTS'),
|
||||
(r'7[\.\-]?1', '7.1'),
|
||||
(r'ddp[\.\-\s]?5[\.\-]?1|eac3|e[\.\-]?ac[\.\-]?3|dd[\.\-]?5[\.\-]?1|ac3|5[\.\-]?1', '5.1'),
|
||||
(r'ddp|dd\+', '5.1'),
|
||||
(r'aac[\.\-]?5[\.\-]?1', '5.1'),
|
||||
(r'aac', 'AAC'),
|
||||
(r'flac', 'FLAC'),
|
||||
(r'mp3', 'MP3'),
|
||||
]
|
||||
|
||||
# Source/release type patterns
|
||||
SOURCE_PATTERNS = [
|
||||
(r'remux', 'Remux'),
|
||||
(r'blu[\.\-]?ray|bdrip|brrip', 'BluRay'),
|
||||
(r'web[\.\-]?dl', 'WEB-DL'),
|
||||
(r'webrip', 'WEBRip'),
|
||||
(r'web', 'WEB'),
|
||||
(r'hdtv', 'HDTV'),
|
||||
(r'dvdrip', 'DVDRip'),
|
||||
(r'dvd', 'DVD'),
|
||||
(r'hdcam|cam', 'CAM'),
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
proxy_enabled: bool = False,
|
||||
proxy_type: str = 'http',
|
||||
proxy_host: Optional[str] = None,
|
||||
proxy_port: Optional[int] = None,
|
||||
proxy_username: Optional[str] = None,
|
||||
proxy_password: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the Easynews client.
|
||||
|
||||
Args:
|
||||
username: Easynews username
|
||||
password: Easynews password
|
||||
proxy_enabled: Whether to use a proxy
|
||||
proxy_type: Proxy type (http, https, socks4, socks5)
|
||||
proxy_host: Proxy hostname/IP
|
||||
proxy_port: Proxy port
|
||||
proxy_username: Proxy auth username (optional)
|
||||
proxy_password: Proxy auth password (optional)
|
||||
"""
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.auth = HTTPBasicAuth(username, password)
|
||||
|
||||
# Set up session with retry logic
|
||||
self.session = requests.Session()
|
||||
self.session.auth = self.auth
|
||||
|
||||
# Configure proxy if enabled
|
||||
self.proxies = {}
|
||||
if proxy_enabled and proxy_host and proxy_port:
|
||||
proxy_url = self._build_proxy_url(
|
||||
proxy_type, proxy_host, proxy_port,
|
||||
proxy_username, proxy_password
|
||||
)
|
||||
self.proxies = {
|
||||
'http': proxy_url,
|
||||
'https': proxy_url,
|
||||
}
|
||||
self.session.proxies.update(self.proxies)
|
||||
logger.info(f"Easynews client configured with {proxy_type} proxy: {proxy_host}:{proxy_port}")
|
||||
|
||||
def _build_proxy_url(
|
||||
self,
|
||||
proxy_type: str,
|
||||
host: str,
|
||||
port: int,
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build a proxy URL with optional authentication."""
|
||||
scheme = proxy_type.lower()
|
||||
if scheme not in ('http', 'https', 'socks4', 'socks5'):
|
||||
scheme = 'http'
|
||||
|
||||
if username and password:
|
||||
return f"{scheme}://{quote(username)}:{quote(password)}@{host}:{port}"
|
||||
return f"{scheme}://{host}:{port}"
|
||||
|
||||
def test_connection(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Test the connection to Easynews with current credentials.
|
||||
|
||||
Returns:
|
||||
Dict with 'success' bool and 'message' string
|
||||
"""
|
||||
try:
|
||||
# Try to access the members area
|
||||
response = self.session.get(
|
||||
f"{self.BASE_URL}/",
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
# Check if we're actually authenticated (not redirected to login)
|
||||
if 'login' in response.url.lower() or 'sign in' in response.text.lower():
|
||||
return {
|
||||
'success': False,
|
||||
'message': 'Invalid credentials - authentication failed'
|
||||
}
|
||||
return {
|
||||
'success': True,
|
||||
'message': 'Successfully connected to Easynews'
|
||||
}
|
||||
elif response.status_code == 401:
|
||||
return {
|
||||
'success': False,
|
||||
'message': 'Invalid credentials - authentication failed'
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Unexpected response: HTTP {response.status_code}'
|
||||
}
|
||||
except requests.exceptions.ProxyError as e:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Proxy connection failed: {str(e)}'
|
||||
}
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Connection failed: {str(e)}'
|
||||
}
|
||||
except requests.exceptions.Timeout:
|
||||
return {
|
||||
'success': False,
|
||||
'message': 'Connection timed out'
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Easynews connection test failed: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Connection test failed: {str(e)}'
|
||||
}
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
page: int = 1,
|
||||
results_per_page: int = 50,
|
||||
file_types: Optional[List[str]] = None,
|
||||
) -> List[EasynewsResult]:
|
||||
"""
|
||||
Search Easynews for files matching the query.
|
||||
|
||||
Args:
|
||||
query: Search query string
|
||||
page: Page number (1-indexed)
|
||||
results_per_page: Number of results per page (max 250)
|
||||
file_types: Optional list of file extensions to filter (e.g., ['mkv', 'mp4'])
|
||||
|
||||
Returns:
|
||||
List of EasynewsResult objects
|
||||
"""
|
||||
try:
|
||||
# Build search parameters
|
||||
params = {
|
||||
'gps': query,
|
||||
'pby': min(results_per_page, 250),
|
||||
'pno': page,
|
||||
'sS': 1, # Safe search off
|
||||
'saession': '', # Session
|
||||
'sb': 1, # Sort by date
|
||||
'sbj': 1, # Subject search
|
||||
'fly': 2, # File type filter mode
|
||||
'fex': 'mkv,mp4', # Only mkv and mp4 files
|
||||
}
|
||||
|
||||
# Add file type filter if specified
|
||||
if file_types:
|
||||
params['fty[]'] = file_types
|
||||
else:
|
||||
# Default to video file types
|
||||
params['fty[]'] = ['VIDEO']
|
||||
|
||||
response = self.session.get(
|
||||
self.SEARCH_URL,
|
||||
params=params,
|
||||
timeout=60,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Check for empty response
|
||||
if not response.content or not response.content.strip():
|
||||
logger.warning(f"Easynews search for '{query}' returned empty response (HTTP {response.status_code})")
|
||||
return []
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except (ValueError, Exception) as json_err:
|
||||
logger.warning(f"Easynews search for '{query}' returned invalid JSON (HTTP {response.status_code}, body: {response.text[:200]}): {json_err}")
|
||||
return []
|
||||
results = []
|
||||
|
||||
# Parse the response
|
||||
if 'data' in data and isinstance(data['data'], list):
|
||||
for item in data['data']:
|
||||
result = self._parse_search_result(item)
|
||||
if result:
|
||||
results.append(result)
|
||||
|
||||
logger.info(f"Easynews search for '{query}' returned {len(results)} results")
|
||||
return results
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Easynews search failed: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing Easynews search results: {e}")
|
||||
return []
|
||||
|
||||
def _parse_search_result(self, item: Dict[str, Any]) -> Optional[EasynewsResult]:
|
||||
"""Parse a single search result from the API response."""
|
||||
try:
|
||||
# Extract the filename
|
||||
filename = item.get('fn', '') or item.get('0', '')
|
||||
if not filename:
|
||||
return None
|
||||
|
||||
# Build download URL
|
||||
# Format: https://username:password@members.easynews.com/dl/{hash}/{filename}
|
||||
file_hash = item.get('hash', '') or item.get('0', '')
|
||||
sig = item.get('sig', '')
|
||||
|
||||
if file_hash and sig:
|
||||
# Use the authenticated download URL format
|
||||
download_path = f"/dl/{file_hash}/{quote(filename)}?sig={sig}"
|
||||
download_url = f"https://{quote(self.username)}:{quote(self.password)}@members.easynews.com{download_path}"
|
||||
else:
|
||||
# Fallback to basic URL
|
||||
download_url = item.get('url', '') or item.get('rawURL', '')
|
||||
if download_url and not download_url.startswith('http'):
|
||||
download_url = urljoin(self.BASE_URL, download_url)
|
||||
|
||||
if not download_url:
|
||||
return None
|
||||
|
||||
# Parse size
|
||||
size_bytes = 0
|
||||
size_str = item.get('rawSize', '') or item.get('size', '')
|
||||
if isinstance(size_str, (int, float)):
|
||||
size_bytes = int(size_str)
|
||||
elif isinstance(size_str, str):
|
||||
size_bytes = self._parse_size(size_str)
|
||||
|
||||
# Parse date
|
||||
post_date = item.get('date', '') or item.get('d', '')
|
||||
if post_date:
|
||||
try:
|
||||
# Try to parse and standardize the date format
|
||||
if isinstance(post_date, str):
|
||||
post_date = post_date.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get extension from API field (more reliable than parsing filename)
|
||||
extension = item.get('extension', '') or item.get('11', '') or item.get('2', '')
|
||||
if extension and not extension.startswith('.'):
|
||||
extension = '.' + extension
|
||||
|
||||
return EasynewsResult(
|
||||
filename=filename,
|
||||
download_url=download_url,
|
||||
size_bytes=size_bytes,
|
||||
post_date=post_date if post_date else None,
|
||||
subject=item.get('subject', '') or item.get('s', ''),
|
||||
poster=item.get('poster', '') or item.get('p', ''),
|
||||
newsgroup=item.get('newsgroup', '') or item.get('ng', ''),
|
||||
extension=extension if extension else self._get_extension(filename),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse search result: {e}")
|
||||
return None
|
||||
|
||||
def _parse_size(self, size_str: str) -> int:
|
||||
"""Parse a size string like '1.5 GB' to bytes."""
|
||||
try:
|
||||
size_str = size_str.strip().upper()
|
||||
multipliers = {
|
||||
'B': 1,
|
||||
'KB': 1024,
|
||||
'MB': 1024 ** 2,
|
||||
'GB': 1024 ** 3,
|
||||
'TB': 1024 ** 4,
|
||||
}
|
||||
|
||||
for suffix, multiplier in multipliers.items():
|
||||
if size_str.endswith(suffix):
|
||||
value = float(size_str[:-len(suffix)].strip())
|
||||
return int(value * multiplier)
|
||||
|
||||
# Try to parse as plain number
|
||||
return int(float(size_str))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def _get_extension(self, filename: str) -> Optional[str]:
|
||||
"""Extract file extension from filename."""
|
||||
if '.' in filename:
|
||||
return filename.rsplit('.', 1)[-1].lower()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def detect_quality(filename: str) -> Optional[str]:
|
||||
"""Detect video quality from filename."""
|
||||
filename_lower = filename.lower()
|
||||
for pattern, quality in EasynewsClient.QUALITY_PATTERNS:
|
||||
if re.search(pattern, filename_lower):
|
||||
return quality
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def detect_audio(filename: str) -> Optional[str]:
|
||||
"""Detect audio codec from filename."""
|
||||
filename_lower = filename.lower()
|
||||
for pattern, audio in EasynewsClient.AUDIO_PATTERNS:
|
||||
if re.search(pattern, filename_lower):
|
||||
return audio
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def detect_source(filename: str) -> Optional[str]:
|
||||
"""Detect source/release type from filename."""
|
||||
filename_lower = filename.lower()
|
||||
for pattern, source in EasynewsClient.SOURCE_PATTERNS:
|
||||
if re.search(pattern, filename_lower):
|
||||
return source
|
||||
return None
|
||||
|
||||
def download_file(
|
||||
self,
|
||||
url: str,
|
||||
dest_path: str,
|
||||
progress_callback: Optional[Callable[[int, int], None]] = None,
|
||||
chunk_size: int = 8192,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Download a file from Easynews.
|
||||
|
||||
Args:
|
||||
url: Download URL (with authentication embedded or using session)
|
||||
dest_path: Destination file path
|
||||
progress_callback: Optional callback(downloaded_bytes, total_bytes)
|
||||
chunk_size: Download chunk size in bytes
|
||||
|
||||
Returns:
|
||||
Dict with 'success' bool and 'message' or 'path'
|
||||
"""
|
||||
try:
|
||||
# Start the download with streaming
|
||||
response = self.session.get(
|
||||
url,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
downloaded = 0
|
||||
|
||||
with open(dest_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback:
|
||||
progress_callback(downloaded, total_size)
|
||||
|
||||
logger.info(f"Downloaded file to {dest_path} ({downloaded} bytes)")
|
||||
return {
|
||||
'success': True,
|
||||
'path': dest_path,
|
||||
'size': downloaded,
|
||||
}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Download failed: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Download failed: {str(e)}'
|
||||
}
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to write file: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Failed to write file: {str(e)}'
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error during download: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Download error: {str(e)}'
|
||||
}
|
||||
|
||||
def get_file_info(self, url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get information about a file without downloading it.
|
||||
|
||||
Args:
|
||||
url: File URL
|
||||
|
||||
Returns:
|
||||
Dict with file information (size, content-type, etc.)
|
||||
"""
|
||||
try:
|
||||
response = self.session.head(url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'size': int(response.headers.get('content-length', 0)),
|
||||
'content_type': response.headers.get('content-type', ''),
|
||||
'last_modified': response.headers.get('last-modified', ''),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get file info: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'message': str(e)
|
||||
}
|
||||
Reference in New Issue
Block a user