"""
Easynews Monitor Module
Monitors Easynews for new media files matching saved search terms.
Uses TMDB for metadata enrichment and organizes downloads properly.
"""
import json
import os
import sqlite3
from datetime import datetime
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from modules.universal_logger import get_logger
from modules.easynews_client import EasynewsClient, EasynewsResult
from modules.media_identifier import MediaIdentifier
from modules.activity_status import get_activity_manager
logger = get_logger('EasynewsMonitor')
# Common words to remove from search queries
STOPWORDS = {
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
'no', 'not', 'very', 'just', 'only', 'also', 'too', 'so', 'than',
}
def clean_search_query(title: str, max_words: int = 5) -> str:
"""
Clean a title for Easynews search by removing stopwords and limiting length.
Keeps the most significant words for better search results.
"""
import re
# Remove punctuation and extra spaces
cleaned = re.sub(r'[^\w\s]', ' ', title)
words = cleaned.split()
# Filter out stopwords and short words
significant_words = [w for w in words if w.lower() not in STOPWORDS and len(w) > 2]
# If we filtered too much, use original words (but still limit)
if len(significant_words) < 2:
significant_words = [w for w in words if len(w) > 2]
# Take first N significant words
return ' '.join(significant_words[:max_words])
# Show name overrides for ambiguous titles
# Maps (show_name, tmdb_id) or just show_name to a more specific search term
SHOW_NAME_OVERRIDES = {
# Grand Hotel (Eva Longoria's ABC show) - search for "Grand Hotel US" to avoid
# matching French, Norwegian, or Spanish versions
('Grand Hotel', 82092): 'Grand Hotel US',
}
class EasynewsMonitor:
"""
Monitor for Easynews searches.
Checks configured search terms and stores discovered results.
"""
def __init__(self, db_path: str, activity_manager=None):
"""
Initialize the Easynews Monitor.
Args:
db_path: Path to the SQLite database
activity_manager: Optional activity manager for status updates
"""
self.db_path = db_path
self.activity_manager = activity_manager
self.default_output_path = '/opt/immich/md/easynews/'
self._ensure_tables()
def _get_connection(self) -> sqlite3.Connection:
"""Get a database connection with row factory."""
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def _ensure_tables(self):
"""Ensure all required tables exist."""
conn = sqlite3.connect(self.db_path)
try:
cursor = conn.cursor()
# Easynews configuration (singleton, id=1)
cursor.execute('''
CREATE TABLE IF NOT EXISTS easynews_config (
id INTEGER PRIMARY KEY CHECK (id = 1),
username TEXT,
password TEXT,
enabled INTEGER DEFAULT 0,
check_interval_hours INTEGER DEFAULT 12,
last_check TEXT,
auto_download INTEGER DEFAULT 0,
min_quality TEXT DEFAULT '720p',
proxy_enabled INTEGER DEFAULT 0,
proxy_type TEXT DEFAULT 'http',
proxy_host TEXT,
proxy_port INTEGER,
proxy_username TEXT,
proxy_password TEXT,
notifications_enabled INTEGER DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
)
''')
# Add notifications_enabled column if it doesn't exist (migration)
cursor.execute("PRAGMA table_info(easynews_config)")
columns = [col[1] for col in cursor.fetchall()]
if 'notifications_enabled' not in columns:
cursor.execute('ALTER TABLE easynews_config ADD COLUMN notifications_enabled INTEGER DEFAULT 1')
# Insert default config row if not exists
cursor.execute('''
INSERT OR IGNORE INTO easynews_config (id, enabled, check_interval_hours, auto_download, min_quality)
VALUES (1, 0, 12, 0, '720p')
''')
# Easynews results table (linked to celebrity_profiles)
cursor.execute('''
CREATE TABLE IF NOT EXISTS easynews_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
celebrity_id INTEGER,
celebrity_name TEXT,
filename TEXT NOT NULL,
download_url TEXT NOT NULL,
size_bytes INTEGER,
post_date TEXT,
discovered_at TEXT DEFAULT CURRENT_TIMESTAMP,
parsed_title TEXT,
parsed_season INTEGER,
parsed_episode INTEGER,
parsed_year INTEGER,
tmdb_id INTEGER,
tmdb_title TEXT,
poster_url TEXT,
quality TEXT,
status TEXT DEFAULT 'new',
download_path TEXT,
FOREIGN KEY (celebrity_id) REFERENCES celebrity_profiles(id)
)
''')
conn.commit()
logger.debug("Easynews tables ensured")
except Exception as e:
logger.error(f"Failed to ensure Easynews tables: {e}")
finally:
conn.close()
# =========================================================================
# CONFIGURATION METHODS
# =========================================================================
def get_config(self) -> Dict[str, Any]:
"""Get the Easynews configuration."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
SELECT username, password, enabled, check_interval_hours, last_check,
auto_download, min_quality, proxy_enabled, proxy_type,
proxy_host, proxy_port, proxy_username, proxy_password,
notifications_enabled, created_at, updated_at
FROM easynews_config
WHERE id = 1
''')
row = cursor.fetchone()
if row:
return {
'username': row['username'],
'password': row['password'],
'enabled': bool(row['enabled']),
'check_interval_hours': row['check_interval_hours'],
'last_check': row['last_check'],
'auto_download': bool(row['auto_download']),
'min_quality': row['min_quality'],
'proxy_enabled': bool(row['proxy_enabled']),
'proxy_type': row['proxy_type'],
'proxy_host': row['proxy_host'],
'proxy_port': row['proxy_port'],
'proxy_username': row['proxy_username'],
'proxy_password': row['proxy_password'],
'notifications_enabled': bool(row['notifications_enabled']) if row['notifications_enabled'] is not None else True,
'has_credentials': bool(row['username'] and row['password']),
}
return {
'username': None,
'password': None,
'enabled': False,
'check_interval_hours': 12,
'last_check': None,
'auto_download': False,
'min_quality': '720p',
'proxy_enabled': False,
'proxy_type': 'http',
'proxy_host': None,
'proxy_port': None,
'proxy_username': None,
'proxy_password': None,
'notifications_enabled': True,
'has_credentials': False,
}
finally:
conn.close()
def update_config(
self,
username: str = None,
password: str = None,
enabled: bool = None,
check_interval_hours: int = None,
auto_download: bool = None,
min_quality: str = None,
proxy_enabled: bool = None,
proxy_type: str = None,
proxy_host: str = None,
proxy_port: int = None,
proxy_username: str = None,
proxy_password: str = None,
notifications_enabled: bool = None,
) -> bool:
"""Update Easynews configuration."""
conn = self._get_connection()
try:
cursor = conn.cursor()
updates = []
values = []
if username is not None:
updates.append('username = ?')
values.append(username)
if password is not None:
updates.append('password = ?')
values.append(password)
if enabled is not None:
updates.append('enabled = ?')
values.append(1 if enabled else 0)
if check_interval_hours is not None:
updates.append('check_interval_hours = ?')
values.append(check_interval_hours)
if auto_download is not None:
updates.append('auto_download = ?')
values.append(1 if auto_download else 0)
if min_quality is not None:
updates.append('min_quality = ?')
values.append(min_quality)
if proxy_enabled is not None:
updates.append('proxy_enabled = ?')
values.append(1 if proxy_enabled else 0)
if proxy_type is not None:
updates.append('proxy_type = ?')
values.append(proxy_type)
if proxy_host is not None:
updates.append('proxy_host = ?')
values.append(proxy_host)
if proxy_port is not None:
updates.append('proxy_port = ?')
values.append(proxy_port)
if proxy_username is not None:
updates.append('proxy_username = ?')
values.append(proxy_username)
if proxy_password is not None:
updates.append('proxy_password = ?')
values.append(proxy_password)
if notifications_enabled is not None:
updates.append('notifications_enabled = ?')
values.append(1 if notifications_enabled else 0)
if not updates:
return False
updates.append('updated_at = ?')
values.append(datetime.now().isoformat())
cursor.execute(f'''
UPDATE easynews_config
SET {', '.join(updates)}
WHERE id = 1
''', values)
conn.commit()
logger.info("Updated Easynews configuration")
return cursor.rowcount > 0
finally:
conn.close()
def _update_last_checked(self):
"""Update the last_check timestamp."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
UPDATE easynews_config
SET last_check = ?
WHERE id = 1
''', (datetime.now().isoformat(),))
conn.commit()
finally:
conn.close()
def _get_client(self) -> Optional[EasynewsClient]:
"""Create an Easynews client from current configuration."""
config = self.get_config()
if not config['username'] or not config['password']:
return None
return EasynewsClient(
username=config['username'],
password=config['password'],
proxy_enabled=config['proxy_enabled'],
proxy_type=config['proxy_type'],
proxy_host=config['proxy_host'],
proxy_port=config['proxy_port'],
proxy_username=config['proxy_username'],
proxy_password=config['proxy_password'],
)
def test_connection(self) -> Dict[str, Any]:
"""Test the Easynews connection with current credentials."""
client = self._get_client()
if not client:
return {
'success': False,
'message': 'No credentials configured'
}
return client.test_connection()
# =========================================================================
# SEARCH MANAGEMENT METHODS
# =========================================================================
def _ensure_searches_table(self):
"""Ensure the easynews_searches table exists."""
conn = sqlite3.connect(self.db_path)
try:
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS easynews_searches (
id INTEGER PRIMARY KEY AUTOINCREMENT,
search_term TEXT NOT NULL,
media_type TEXT DEFAULT 'any',
tmdb_id INTEGER,
tmdb_title TEXT,
poster_url TEXT,
enabled INTEGER DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_ens_enabled ON easynews_searches(enabled)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_ens_term ON easynews_searches(search_term)')
conn.commit()
except Exception as e:
logger.error(f"Failed to ensure easynews_searches table: {e}")
finally:
conn.close()
def get_all_searches(self) -> List[Dict[str, Any]]:
"""Get all saved search terms."""
self._ensure_searches_table()
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
SELECT id, search_term, media_type, tmdb_id, tmdb_title,
poster_url, enabled, created_at, updated_at
FROM easynews_searches
ORDER BY created_at DESC
''')
return [dict(row) for row in cursor.fetchall()]
except Exception as e:
logger.error(f"Failed to get searches: {e}")
return []
finally:
conn.close()
def get_search(self, search_id: int) -> Optional[Dict[str, Any]]:
"""Get a specific search by ID."""
self._ensure_searches_table()
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
SELECT id, search_term, media_type, tmdb_id, tmdb_title,
poster_url, enabled, created_at, updated_at
FROM easynews_searches
WHERE id = ?
''', (search_id,))
row = cursor.fetchone()
return dict(row) if row else None
except Exception as e:
logger.error(f"Failed to get search {search_id}: {e}")
return None
finally:
conn.close()
def add_search(
self,
search_term: str,
media_type: str = 'any',
tmdb_id: Optional[int] = None,
tmdb_title: Optional[str] = None,
poster_url: Optional[str] = None
) -> Optional[int]:
"""Add a new search term."""
self._ensure_searches_table()
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
INSERT INTO easynews_searches
(search_term, media_type, tmdb_id, tmdb_title, poster_url)
VALUES (?, ?, ?, ?, ?)
''', (search_term, media_type, tmdb_id, tmdb_title, poster_url))
conn.commit()
logger.info(f"Added Easynews search: {search_term}")
return cursor.lastrowid
except Exception as e:
logger.error(f"Failed to add search: {e}")
return None
finally:
conn.close()
def update_search(
self,
search_id: int,
search_term: Optional[str] = None,
media_type: Optional[str] = None,
enabled: Optional[bool] = None,
tmdb_id: Optional[int] = None,
tmdb_title: Optional[str] = None,
poster_url: Optional[str] = None
) -> bool:
"""Update an existing search term."""
conn = self._get_connection()
try:
cursor = conn.cursor()
updates = []
values = []
if search_term is not None:
updates.append('search_term = ?')
values.append(search_term)
if media_type is not None:
updates.append('media_type = ?')
values.append(media_type)
if enabled is not None:
updates.append('enabled = ?')
values.append(1 if enabled else 0)
if tmdb_id is not None:
updates.append('tmdb_id = ?')
values.append(tmdb_id)
if tmdb_title is not None:
updates.append('tmdb_title = ?')
values.append(tmdb_title)
if poster_url is not None:
updates.append('poster_url = ?')
values.append(poster_url)
if not updates:
return False
updates.append('updated_at = ?')
values.append(datetime.now().isoformat())
values.append(search_id)
cursor.execute(f'''
UPDATE easynews_searches
SET {', '.join(updates)}
WHERE id = ?
''', values)
conn.commit()
logger.info(f"Updated Easynews search {search_id}")
return cursor.rowcount > 0
except Exception as e:
logger.error(f"Failed to update search {search_id}: {e}")
return False
finally:
conn.close()
def delete_search(self, search_id: int) -> bool:
"""Delete a search term."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('DELETE FROM easynews_searches WHERE id = ?', (search_id,))
conn.commit()
logger.info(f"Deleted Easynews search {search_id}")
return cursor.rowcount > 0
except Exception as e:
logger.error(f"Failed to delete search {search_id}: {e}")
return False
finally:
conn.close()
def check_single_search(self, search_id: int) -> Dict[str, Any]:
"""Check Easynews for a specific search term."""
search = self.get_search(search_id)
if not search:
return {'success': False, 'message': 'Search not found', 'results_found': 0}
config = self.get_config()
if not config['has_credentials']:
return {'success': False, 'message': 'No credentials configured', 'results_found': 0}
client = self._get_client()
if not client:
return {'success': False, 'message': 'Failed to create client', 'results_found': 0}
try:
results = client.search(search['search_term'], results_per_page=250)
# Store results - simplified version
results_found = len(results) if results else 0
logger.info(f"Easynews search '{search['search_term']}' found {results_found} results")
return {
'success': True,
'message': f"Found {results_found} results",
'results_found': results_found
}
except Exception as e:
logger.error(f"Error searching Easynews for '{search['search_term']}': {e}")
return {'success': False, 'message': str(e), 'results_found': 0}
# =========================================================================
# CELEBRITY METHODS (uses tracked celebrities from Appearances)
# =========================================================================
def get_celebrities(self) -> List[Dict[str, Any]]:
"""Get all enabled celebrity profiles for Easynews searching."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
SELECT id, name, slug, image_url, enabled
FROM celebrity_profiles
WHERE enabled = 1
ORDER BY name ASC
''')
return [dict(row) for row in cursor.fetchall()]
finally:
conn.close()
def get_celebrity_count(self) -> int:
"""Get count of enabled celebrities."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('SELECT COUNT(*) FROM celebrity_profiles WHERE enabled = 1')
return cursor.fetchone()[0]
except Exception:
return 0
finally:
conn.close()
def get_celebrity_appearances(self, celebrity_id: int) -> List[Dict[str, Any]]:
"""Get appearances (TV shows, movies) for a celebrity with TMDB data."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('''
SELECT DISTINCT
show_name, tmdb_show_id, appearance_type,
season_number, episode_number, episode_title,
celebrity_name
FROM celebrity_appearances
WHERE celebrity_id = ?
AND tmdb_show_id IS NOT NULL
ORDER BY show_name, season_number, episode_number
''', (celebrity_id,))
return [dict(row) for row in cursor.fetchall()]
finally:
conn.close()
def get_tmdb_poster(self, tmdb_id: int, media_type: str) -> Optional[str]:
"""Get TMDB backdrop URL (widescreen) for a show/movie."""
conn = self._get_connection()
try:
cursor = conn.cursor()
# Try to get from appearance_config (has TMDB API key)
cursor.execute('SELECT tmdb_api_key FROM appearance_config WHERE id = 1')
row = cursor.fetchone()
if not row or not row[0]:
return None
api_key = row[0]
endpoint = 'tv' if media_type == 'TV' else 'movie'
import requests
response = requests.get(
f"https://api.themoviedb.org/3/{endpoint}/{tmdb_id}",
params={'api_key': api_key},
timeout=10
)
if response.status_code == 200:
data = response.json()
# Prefer backdrop (widescreen) over poster (portrait)
backdrop_path = data.get('backdrop_path')
if backdrop_path:
return f"https://image.tmdb.org/t/p/w780{backdrop_path}"
# Fall back to poster if no backdrop
poster_path = data.get('poster_path')
if poster_path:
return f"https://image.tmdb.org/t/p/w500{poster_path}"
return None
except Exception as e:
logger.error(f"Failed to get TMDB backdrop: {e}")
return None
finally:
conn.close()
# =========================================================================
# RESULTS MANAGEMENT METHODS
# =========================================================================
def get_results(
self,
status: str = None,
celebrity_id: int = None,
limit: int = 100,
offset: int = 0,
) -> List[Dict[str, Any]]:
"""
Get discovered results.
Args:
status: Filter by status ('new', 'downloaded', 'ignored', 'failed')
celebrity_id: Filter by celebrity ID
limit: Maximum results to return
offset: Offset for pagination
Returns:
List of result dictionaries
"""
conn = self._get_connection()
try:
cursor = conn.cursor()
query = '''
SELECT r.*, c.name as celebrity_display_name, c.image_url as celebrity_image
FROM easynews_results r
LEFT JOIN celebrity_profiles c ON r.celebrity_id = c.id
'''
conditions = []
values = []
if status:
conditions.append('r.status = ?')
values.append(status)
if celebrity_id:
conditions.append('r.celebrity_id = ?')
values.append(celebrity_id)
if conditions:
query += ' WHERE ' + ' AND '.join(conditions)
query += ' ORDER BY r.discovered_at DESC LIMIT ? OFFSET ?'
values.extend([limit, offset])
cursor.execute(query, values)
return [dict(row) for row in cursor.fetchall()]
finally:
conn.close()
def get_result_count(self, status: str = None) -> int:
"""Get count of results, optionally filtered by status."""
conn = self._get_connection()
try:
cursor = conn.cursor()
if status:
cursor.execute('SELECT COUNT(*) FROM easynews_results WHERE status = ?', (status,))
else:
cursor.execute('SELECT COUNT(*) FROM easynews_results')
return cursor.fetchone()[0]
finally:
conn.close()
def update_result_status(self, result_id: int, status: str, download_path: str = None) -> bool:
"""Update a result's status."""
conn = self._get_connection()
try:
cursor = conn.cursor()
if download_path:
cursor.execute('''
UPDATE easynews_results
SET status = ?, download_path = ?
WHERE id = ?
''', (status, download_path, result_id))
else:
cursor.execute('''
UPDATE easynews_results
SET status = ?
WHERE id = ?
''', (status, result_id))
conn.commit()
return cursor.rowcount > 0
finally:
conn.close()
def _get_or_create_easynews_preset(self, celebrity_id: int, celebrity_name: str) -> int:
"""Get or create an Easynews preset for a celebrity."""
conn = self._get_connection()
try:
cursor = conn.cursor()
# Check if preset exists
cursor.execute('''
SELECT id FROM celebrity_search_presets
WHERE celebrity_id = ? AND source_type = 'easynews'
''', (celebrity_id,))
row = cursor.fetchone()
if row:
return row[0]
# Create new preset
cursor.execute('''
INSERT INTO celebrity_search_presets (
celebrity_id, name, source_type, source_value, content_type, enabled
) VALUES (?, ?, 'easynews', ?, 'other', 1)
''', (celebrity_id, f'Easynews - {celebrity_name}', celebrity_name))
conn.commit()
return cursor.lastrowid
finally:
conn.close()
def _save_result(
self,
celebrity_id: int,
celebrity_name: str,
easynews_result: EasynewsResult,
parsed_info: Dict[str, Any],
tmdb_match: Optional[Dict[str, Any]],
) -> Optional[int]:
"""Save a discovered result to celebrity_discovered_videos (same as YouTube videos)."""
import hashlib
# Get or create the Easynews preset for this celebrity
preset_id = self._get_or_create_easynews_preset(celebrity_id, celebrity_name)
conn = self._get_connection()
try:
cursor = conn.cursor()
# Create a unique video_id from the filename and URL
video_id = hashlib.md5(f"{easynews_result.filename}:{easynews_result.download_url}".encode()).hexdigest()[:16]
# Check for duplicate in celebrity_discovered_videos
cursor.execute('''
SELECT id FROM celebrity_discovered_videos
WHERE platform = 'easynews' AND video_id = ?
''', (video_id,))
if cursor.fetchone():
return None # Already exists
# Prepare values
parsed_title = parsed_info.get('title') if parsed_info else None
parsed_season = parsed_info.get('season') if parsed_info else None
parsed_episode = parsed_info.get('episode') if parsed_info else None
parsed_year = parsed_info.get('year') if parsed_info else None
quality = parsed_info.get('quality') if parsed_info else EasynewsClient.detect_quality(easynews_result.filename)
tmdb_title = tmdb_match.get('title') if tmdb_match else None
poster_url = tmdb_match.get('poster_path') if tmdb_match else None
# Build display title
if tmdb_title:
if parsed_season and parsed_episode:
display_title = f"{tmdb_title} S{parsed_season:02d}E{parsed_episode:02d}"
elif parsed_year:
display_title = f"{tmdb_title} ({parsed_year})"
else:
display_title = tmdb_title
elif parsed_title:
display_title = parsed_title
else:
display_title = easynews_result.filename
# Build description with file info
size_mb = easynews_result.size_bytes / (1024 * 1024) if easynews_result.size_bytes else 0
description = f"Quality: {quality or 'Unknown'} | Size: {size_mb:.1f} MB | File: {easynews_result.filename}"
# Insert into celebrity_discovered_videos (same table as YouTube)
cursor.execute('''
INSERT INTO celebrity_discovered_videos (
preset_id, celebrity_id, video_id, platform, url, title,
channel_name, thumbnail, upload_date, description,
content_type, status
) VALUES (?, ?, ?, 'easynews', ?, ?, ?, ?, ?, ?, 'other', 'new')
''', (
preset_id,
celebrity_id,
video_id,
easynews_result.download_url,
display_title,
f"Easynews - {celebrity_name}", # channel_name
poster_url, # thumbnail (TMDB poster)
easynews_result.post_date,
description,
))
conn.commit()
logger.info(f"Saved Easynews result for {celebrity_name}: {display_title}")
return cursor.lastrowid
except Exception as e:
logger.error(f"Failed to save result: {e}")
return None
finally:
conn.close()
# =========================================================================
# CHECK METHODS
# =========================================================================
def get_status(self) -> Dict[str, Any]:
"""Get current check status from database (works across processes)."""
activity_manager = get_activity_manager()
task_status = activity_manager.get_background_task("easynews_check")
if task_status and task_status.get('active'):
extra = task_status.get('extra_data', {}) or {}
return {
"is_running": True,
"current_search": extra.get('current_search'),
"searches_processed": task_status.get('progress', {}).get('current', 0),
"total_searches": task_status.get('progress', {}).get('total', 0),
"results_found": extra.get('results_found', 0),
"last_check": extra.get('last_check'),
"error": None,
}
# Not running - return default state
return {
"is_running": False,
"current_search": None,
"searches_processed": 0,
"total_searches": 0,
"results_found": 0,
"last_check": None,
"error": None,
}
def check_all_celebrities(
self,
progress_callback: Callable[[str, int, int], None] = None,
from_scheduler: bool = False,
) -> Dict[str, Any]:
"""
Check Easynews for all tracked celebrities by searching for their appearances.
Searches for TV shows and movies the celebrity appears in (from TMDB data),
not the celebrity name directly.
Args:
progress_callback: Optional callback(search_term, processed, total)
from_scheduler: If True, send push notifications (scheduler runs only)
Returns:
Dict with check results
"""
# Get configuration
config = self.get_config()
if not config['has_credentials']:
return {
'success': False,
'message': 'No Easynews credentials configured',
'results_found': 0,
}
# Get enabled celebrities
celebrities = self.get_celebrities()
if not celebrities:
return {
'success': True,
'message': 'No celebrities tracked',
'results_found': 0,
}
# Build list of all shows/movies to search (deduplicated)
searches_to_perform = [] # List of (celebrity_id, celebrity_name, show_name, tmdb_id, media_type, season, episode)
seen_searches = set()
poster_cache = {} # Cache TMDB posters
# First pass: identify TV shows that have episode-specific entries
# so we can skip generic entries for those shows
shows_with_episodes = set()
for celebrity in celebrities:
appearances = self.get_celebrity_appearances(celebrity['id'])
for app in appearances:
show_name = app.get('show_name')
media_type = app.get('appearance_type', 'TV')
season = app.get('season_number')
episode = app.get('episode_number')
if media_type != 'Movie' and season and episode:
shows_with_episodes.add(show_name)
for celebrity in celebrities:
appearances = self.get_celebrity_appearances(celebrity['id'])
for app in appearances:
show_name = app.get('show_name')
tmdb_id = app.get('tmdb_show_id')
media_type = app.get('appearance_type', 'TV')
season = app.get('season_number')
episode = app.get('episode_number')
if not show_name:
continue
# Build search term (clean to remove stopwords for better results)
# Check for show name overrides (for ambiguous titles)
override_key = (show_name, tmdb_id)
if override_key in SHOW_NAME_OVERRIDES:
clean_name = SHOW_NAME_OVERRIDES[override_key]
else:
clean_name = clean_search_query(show_name)
if media_type == 'Movie':
search_term = clean_name
search_key = f"movie:{show_name}"
elif season and episode:
search_term = f"{clean_name} S{season:02d}E{episode:02d}"
search_key = f"tv:{show_name}:S{season}E{episode}"
elif season:
search_term = f"{clean_name} S{season:02d}"
search_key = f"tv:{show_name}:S{season}"
else:
# Skip generic TV entries if we have episode-specific entries for this show
if show_name in shows_with_episodes:
logger.debug(f"Skipping generic search for '{show_name}' - has episode-specific entries")
continue
search_term = clean_name
search_key = f"tv:{show_name}"
if search_key not in seen_searches:
seen_searches.add(search_key)
searches_to_perform.append({
'celebrity_id': celebrity['id'],
'celebrity_name': celebrity['name'],
'show_name': show_name,
'search_term': search_term,
'tmdb_id': tmdb_id,
'media_type': media_type,
'season': season,
'episode': episode,
})
if not searches_to_perform:
return {
'success': True,
'message': 'No appearances to search for',
'results_found': 0,
}
# Initialize status tracking in database (works across processes)
activity_manager = get_activity_manager()
easynews_extra_data = {
"current_search": None,
"results_found": 0,
"last_check": None,
}
activity_manager.start_background_task(
task_id="easynews_check",
task_type="easynews_check",
display_name="Easynews",
status="Starting",
extra_data=easynews_extra_data
)
# Create client
client = self._get_client()
if not client:
activity_manager.stop_background_task("easynews_check")
return {
'success': False,
'message': 'Failed to create Easynews client',
'results_found': 0,
}
total_new = 0
# Crash recovery checkpoint
from modules.task_checkpoint import TaskCheckpoint
checkpoint = TaskCheckpoint('easynews_monitor', 'background')
checkpoint.start(total_items=len(searches_to_perform))
if checkpoint.is_recovering():
logger.info("Easynews monitor: recovering — skipping already-completed searches")
try:
for i, search in enumerate(searches_to_perform):
search_term = search['search_term']
celebrity_id = search['celebrity_id']
celebrity_name = search['celebrity_name']
show_name = search['show_name']
tmdb_id = search['tmdb_id']
media_type = search['media_type']
season = search['season']
episode = search['episode']
if checkpoint.is_completed(search_term):
continue
checkpoint.set_current(search_term)
easynews_extra_data["current_search"] = search_term
activity_manager.update_background_task(
"easynews_check", f"Searching: {search_term}",
progress_current=i, progress_total=len(searches_to_perform),
extra_data=easynews_extra_data
)
if progress_callback:
progress_callback(search_term, i, len(searches_to_perform))
logger.info(f"Searching Easynews: {search_term}")
try:
# Get TMDB poster (cached)
if tmdb_id and tmdb_id not in poster_cache:
poster_cache[tmdb_id] = self.get_tmdb_poster(tmdb_id, media_type)
poster_url = poster_cache.get(tmdb_id)
# Search Easynews (use max 250 results to find more episodes)
results = client.search(search_term, results_per_page=250)
for result in results:
# Skip sample files
if 'sample' in result.filename.lower():
continue
# Validate result matches the show we're looking for
if not self._result_matches_show(result.filename, show_name):
continue
# Check quality filter
quality = EasynewsClient.detect_quality(result.filename)
if config['min_quality'] and not self._meets_quality(quality, config['min_quality']):
continue
# Save result with proper metadata
new_id = self._save_appearance_result(
celebrity_id=celebrity_id,
celebrity_name=celebrity_name,
show_name=show_name,
tmdb_id=tmdb_id,
media_type=media_type,
season=season,
episode=episode,
poster_url=poster_url,
easynews_result=result,
quality=quality,
)
if new_id:
total_new += 1
easynews_extra_data["results_found"] = total_new
activity_manager.update_background_task(
"easynews_check", f"Found {total_new} new results",
progress_current=i, progress_total=len(searches_to_perform),
extra_data=easynews_extra_data
)
except Exception as e:
logger.error(f"Error searching Easynews for '{search_term}': {e}")
continue
checkpoint.mark_completed(search_term)
# Checkpoint complete
checkpoint.finish()
# Mark as complete
easynews_extra_data["last_check"] = datetime.now().isoformat()
activity_manager.stop_background_task("easynews_check")
self._update_last_checked()
logger.info(f"Easynews check complete: {total_new} new results found")
# Send notification if from scheduler and new results found
if from_scheduler and total_new > 0:
config = self.get_config()
if config.get('notifications_enabled', True):
self._send_notification(total_new)
return {
'success': True,
'message': f'Check complete: {total_new} new results found',
'results_found': total_new,
'searches_performed': len(searches_to_perform),
}
except Exception as e:
logger.error(f"Easynews check failed: {e}")
activity_manager.stop_background_task("easynews_check")
return {
'success': False,
'message': str(e),
'results_found': total_new,
}
def _send_notification(self, results_found: int):
"""
Send a Pushover notification about new Easynews results found.
Args:
results_found: Number of new results found
"""
# Don't send notification if no new results
if results_found <= 0:
logger.debug("Skipping Easynews notification - no new results")
return
try:
import random
from modules.pushover_notifier import PushoverNotifier
from modules.settings_manager import SettingsManager
from modules.unified_database import UnifiedDatabase
# Get pushover config from settings
settings_manager = SettingsManager(self.db_path)
pushover_config = settings_manager.get('pushover', {})
if not pushover_config.get('enabled'):
logger.debug("Pushover notifications disabled globally")
return
# Create unified_db for recording notification to database
unified_db = UnifiedDatabase(self.db_path)
# Create notifier with unified_db so notification is recorded
notifier = PushoverNotifier(
api_token=pushover_config.get('api_token'),
user_key=pushover_config.get('user_key'),
unified_db=unified_db
)
# Get show summary from recently discovered videos
image_path = None
thumbnail_url = None
shows = {}
show_list = []
conn = self._get_connection()
try:
cursor = conn.cursor()
# Get the most recent Easynews results
cursor.execute('''
SELECT thumbnail, title, channel_name FROM celebrity_discovered_videos
WHERE platform = 'easynews' AND thumbnail IS NOT NULL AND thumbnail != ''
ORDER BY discovered_at DESC
LIMIT ?
''', (results_found,))
rows = cursor.fetchall()
if rows:
# Pick a random thumbnail from this batch
selected = random.choice(rows)
thumbnail_url = selected['thumbnail']
# Build show summary from unique shows in THIS batch only
shows = {}
for row in rows:
# Extract show name from channel_name (e.g., "Easynews - tv:Show Name:S1E1")
channel = row['channel_name'] or ''
if channel.startswith('Easynews - '):
parts = channel.replace('Easynews - ', '').split(':')
if len(parts) >= 2:
show_name = parts[1]
else:
show_name = row['title'] or 'Unknown'
else:
show_name = row['title'] or 'Unknown'
shows[show_name] = shows.get(show_name, 0) + 1
if show_name not in show_list:
show_list.append(show_name)
# Download thumbnail to temp file
if thumbnail_url:
import urllib.request
import tempfile
try:
temp_dir = tempfile.gettempdir()
temp_path = f"{temp_dir}/easynews_thumb_{random.randint(1000, 9999)}.jpg"
urllib.request.urlretrieve(thumbnail_url, temp_path)
image_path = temp_path
logger.debug(f"Downloaded thumbnail for notification: {temp_path}")
except Exception as e:
logger.debug(f"Could not download thumbnail: {e}")
finally:
conn.close()
# Build title matching standard notification format
title = f"📰 {results_found} New Result{'s' if results_found > 1 else ''} on Easynews"
# Build HTML message matching standard notification format
from datetime import datetime
message_parts = []
message_parts.append("📱 Platform: Easynews")
message_parts.append(f"🎥 Results: {results_found} new result{'s' if results_found > 1 else ''}")
# Add show breakdown
if shows:
show_parts = [f"{name} ({count})" if count > 1 else name
for name, count in sorted(shows.items(), key=lambda x: -x[1])[:5]]
message_parts.append(f"📺 Shows: {', '.join(show_parts)}")
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
message_parts.append(f"⏰ Discovered: {now}")
message = "\n".join(message_parts)
# Set notification context for database recording
# Store thumbnail_url (not temp file path) so notification page can display it
notifier._current_notification_context = {
'platform': 'easynews',
'source': 'easynews_monitor',
'content_type': 'video',
'download_count': results_found,
'metadata': {'shows': show_list, 'thumbnail_url': thumbnail_url}
}
# Send notification
success = notifier.send_notification(
title=title,
message=message,
priority=0, # Normal priority
image_path=image_path,
html=True
)
# Clean up temp file
if image_path:
try:
import os
os.unlink(image_path)
except OSError:
pass # Best effort cleanup of temp file
if success:
logger.info(f"Sent notification: {results_found} Easynews results found")
else:
logger.debug("Notification not sent (disabled or failed)")
except Exception as e:
logger.warning(f"Could not send notification: {e}")
def _get_quality_rank(self, quality: Optional[str], source: Optional[str] = None, audio: Optional[str] = None) -> int:
"""Get numeric rank for quality comparison (higher is better).
Combines resolution, source, and audio into a single score:
- Resolution: 360p=100, 480p=200, 720p=300, 1080p=400, 2160p=500
- Source: Remux=50, BluRay=40, WEB-DL=30, WEB=20, HDTV=10
- Audio: Atmos=9, TrueHD=8, DTS-HD=7, 5.1=4, etc.
"""
score = 0
# Resolution score (most important)
if quality:
resolution_scores = {
'360p': 100,
'480p': 200,
'720p': 300,
'1080p': 400,
'2160p': 500,
}
score += resolution_scores.get(quality.lower(), 0)
# Source score
if source:
source_scores = {
'Remux': 50,
'BluRay': 40,
'WEB-DL': 30,
'WEBRip': 25,
'WEB': 20,
'HDTV': 10,
'DVDRip': 5,
'DVD': 3,
'CAM': 1,
}
score += source_scores.get(source, 0)
# Audio score
if audio:
audio_scores = {
'Atmos': 9,
'TrueHD': 8,
'DTS-HD': 7,
'DTS:X': 7,
'DTS': 6,
'7.1': 5,
'5.1': 4,
'FLAC': 3,
'AAC': 2,
'MP3': 1,
}
score += audio_scores.get(audio, 0)
return score
def _save_appearance_result(
self,
celebrity_id: int,
celebrity_name: str,
show_name: str,
tmdb_id: int,
media_type: str,
season: Optional[int],
episode: Optional[int],
poster_url: Optional[str],
easynews_result: EasynewsResult,
quality: Optional[str],
) -> Optional[int]:
"""Save an Easynews result - only keeps best quality per episode."""
import hashlib
# Get or create the Easynews preset for this celebrity
preset_id = self._get_or_create_easynews_preset(celebrity_id, celebrity_name)
conn = self._get_connection()
try:
cursor = conn.cursor()
# Build a unique key for this show/episode (not per file)
if media_type == 'Movie':
content_key = f"movie:{show_name}"
elif season and episode:
content_key = f"tv:{show_name}:S{season}E{episode}"
elif season:
content_key = f"tv:{show_name}:S{season}"
else:
content_key = f"tv:{show_name}"
# For TV shows without episode info, skip if we already have episode-specific entries
cursor.execute('''
SELECT COUNT(*) FROM celebrity_discovered_videos
WHERE platform = 'easynews'
AND celebrity_id = ?
AND channel_name LIKE ?
''', (celebrity_id, f"Easynews - tv:{show_name}:S%"))
if cursor.fetchone()[0] > 0:
# Already have episode-specific entries, skip this generic one
return None
# Detect audio codec and source FIRST (needed for comparison)
audio_codec = EasynewsClient.detect_audio(easynews_result.filename)
source = EasynewsClient.detect_source(easynews_result.filename)
# Calculate new result's score
new_score = self._get_quality_rank(quality, source, audio_codec)
# Check if we already have this episode - look by channel_name which contains show info
cursor.execute('''
SELECT id, metadata FROM celebrity_discovered_videos
WHERE platform = 'easynews'
AND celebrity_id = ?
AND channel_name = ?
''', (celebrity_id, f"Easynews - {content_key}"))
existing = cursor.fetchone()
if existing:
# Parse existing quality, source, audio from metadata
existing_meta = {}
if existing[1]:
try:
existing_meta = json.loads(existing[1])
except (json.JSONDecodeError, TypeError):
pass # Invalid JSON metadata, use empty dict
existing_quality = existing_meta.get('quality')
existing_source = existing_meta.get('source')
existing_audio = existing_meta.get('audio_codec')
existing_score = self._get_quality_rank(existing_quality, existing_source, existing_audio)
# Only replace if new score is better
if new_score <= existing_score:
return None # Existing is same or better quality
# Delete existing to replace with better quality (upgrade, not truly new)
logger.info(f"Upgrading {show_name}: score {existing_score} -> {new_score}")
cursor.execute('DELETE FROM celebrity_discovered_videos WHERE id = ?', (existing[0],))
is_upgrade = True
else:
is_upgrade = False
# Create a unique video_id from the filename and URL
video_id = hashlib.md5(f"{easynews_result.filename}:{easynews_result.download_url}".encode()).hexdigest()[:16]
# Build display title with proper formatting
if media_type == 'Movie':
display_title = show_name
elif season and episode:
display_title = f"{show_name} - S{season:02d}E{episode:02d}"
elif season:
display_title = f"{show_name} - Season {season}"
else:
display_title = show_name
# Build description with file info
size_mb = easynews_result.size_bytes / (1024 * 1024) if easynews_result.size_bytes else 0
description = f"Quality: {quality or 'Unknown'} | Source: {source or 'Unknown'} | Audio: {audio_codec or 'Unknown'} | Size: {size_mb:.1f} MB | File: {easynews_result.filename}"
# Extract numeric resolution (e.g., "720p" -> 720)
resolution_int = None
if quality:
try:
resolution_int = int(quality.lower().replace('p', ''))
except ValueError:
pass
# Build metadata JSON
metadata = json.dumps({
'audio_codec': audio_codec,
'source': source,
'quality': quality,
'size_bytes': easynews_result.size_bytes,
'original_filename': easynews_result.filename,
})
# Insert into celebrity_discovered_videos
cursor.execute('''
INSERT INTO celebrity_discovered_videos (
preset_id, celebrity_id, video_id, platform, url, title,
channel_name, thumbnail, upload_date, description,
content_type, status, max_resolution, metadata
) VALUES (?, ?, ?, 'easynews', ?, ?, ?, ?, ?, ?, 'other', 'new', ?, ?)
''', (
preset_id,
celebrity_id,
video_id,
easynews_result.download_url,
display_title,
f"Easynews - {content_key}", # channel_name stores content key for dedup
poster_url, # thumbnail (TMDB poster)
easynews_result.post_date,
description,
resolution_int,
metadata,
))
conn.commit()
# Pre-cache thumbnail for faster page loading
if poster_url:
self._cache_thumbnail(video_id, poster_url, cursor, conn)
# Return None for upgrades (don't count as "new" for notifications)
# Only return ID for truly new content
if is_upgrade:
logger.info(f"Upgraded Easynews result: {display_title} ({quality})")
return None
else:
logger.info(f"Saved NEW Easynews result: {display_title} ({quality})")
return cursor.lastrowid
except Exception as e:
logger.error(f"Failed to save appearance result: {e}")
return None
finally:
conn.close()
def _cache_thumbnail(self, video_id: str, thumbnail_url: str, cursor, conn) -> None:
"""
Pre-cache thumbnail by fetching from URL and storing in database.
This speeds up Internet Discovery page loading.
"""
try:
import requests
response = requests.get(thumbnail_url, timeout=10, headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
if response.status_code == 200 and response.content:
cursor.execute('''
UPDATE celebrity_discovered_videos
SET thumbnail_data = ?
WHERE video_id = ?
''', (response.content, video_id))
conn.commit()
logger.debug(f"Cached thumbnail for {video_id}", module="Easynews")
except Exception as e:
logger.debug(f"Failed to cache thumbnail for {video_id}: {e}", module="Easynews")
def _meets_quality(self, quality: Optional[str], min_quality: str) -> bool:
"""Check if quality meets minimum requirement."""
if not min_quality:
return True # No minimum set, allow all
if not quality:
return False # Can't detect quality but minimum is set, reject
quality_order = ['360p', '480p', '720p', '1080p', '2160p']
try:
quality_idx = quality_order.index(quality.lower())
min_idx = quality_order.index(min_quality.lower())
return quality_idx >= min_idx
except ValueError:
return False # Unknown quality format, reject when minimum is set
def _result_matches_show(self, filename: str, show_name: str) -> bool:
"""Check if a result filename matches the show we're looking for.
Validates that key words from the show name appear in the filename
to avoid false matches (e.g., 'Very Bad Day' vs 'Very Bad Road Trip').
"""
import re
# Normalize both strings
filename_lower = filename.lower()
filename_clean = re.sub(r'[^\w\s]', ' ', filename_lower)
filename_words = filename_clean.split()
# Get significant words from show name (skip stopwords)
show_words = re.sub(r'[^\w\s]', ' ', show_name.lower()).split()
significant_words = [w for w in show_words if w.lower() not in STOPWORDS and len(w) > 2]
if not significant_words:
return True # Can't validate, allow
# For short titles (2-3 words), require words to appear adjacent in filename
# This prevents matching scattered words like "christmas...karma" in unrelated content
if len(significant_words) <= 3:
# Build a pattern that matches the words in order with optional separators
pattern = r'[\.\s\-_]*'.join(re.escape(w) for w in significant_words)
if re.search(pattern, filename_clean):
return True
return False
# For longer titles, count matches and require 80%
matches = sum(1 for word in significant_words if word in filename_clean)
match_ratio = matches / len(significant_words)
if len(significant_words) <= 5:
return match_ratio >= 1.0
else:
return match_ratio >= 0.8
def _get_tmdb_api_key(self) -> Optional[str]:
"""Get TMDB API key from appearance config."""
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('SELECT tmdb_api_key FROM appearance_config WHERE id = 1')
row = cursor.fetchone()
return row[0] if row else None
except Exception:
return None
finally:
conn.close()
# =========================================================================
# DOWNLOAD METHODS
# =========================================================================
def download_result(
self,
result_id: int,
output_dir: str = None,
progress_callback: Callable[[int, int], None] = None,
) -> Dict[str, Any]:
"""
Download a result to the appropriate organized path.
Args:
result_id: ID of the result to download
output_dir: Override output directory (uses default if not specified)
progress_callback: Optional callback(downloaded_bytes, total_bytes)
Returns:
Dict with download result
"""
# Get result
conn = self._get_connection()
try:
cursor = conn.cursor()
cursor.execute('SELECT * FROM easynews_results WHERE id = ?', (result_id,))
row = cursor.fetchone()
if not row:
return {'success': False, 'message': 'Result not found'}
result = dict(row)
finally:
conn.close()
# Get client
client = self._get_client()
if not client:
return {'success': False, 'message': 'No Easynews credentials configured'}
# Determine output path
base_path = output_dir or self.default_output_path
# Use TMDB info to create organized path
if result.get('tmdb_id') and result.get('tmdb_title'):
tmdb_api_key = self._get_tmdb_api_key()
if tmdb_api_key:
identifier = MediaIdentifier(tmdb_api_key)
# Create TMDBMatch-like object
from modules.media_identifier import TMDBMatch
match = TMDBMatch(
tmdb_id=result['tmdb_id'],
title=result['tmdb_title'],
original_title=None,
media_type='tv' if result.get('parsed_season') else 'movie',
year=result.get('parsed_year'),
poster_path=result.get('poster_url'),
season_number=result.get('parsed_season'),
episode_number=result.get('parsed_episode'),
)
dest_path = identifier.get_organized_path(match, base_path, result['filename'])
else:
dest_path = os.path.join(base_path, result['filename'])
else:
dest_path = os.path.join(base_path, result['filename'])
# Create directory
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
# Mark as downloading
self.update_result_status(result_id, 'downloading')
try:
# Download
download_result = client.download_file(
result['download_url'],
dest_path,
progress_callback=progress_callback,
)
if download_result['success']:
self.update_result_status(result_id, 'downloaded', dest_path)
logger.info(f"Downloaded: {result['filename']} -> {dest_path}")
return {
'success': True,
'path': dest_path,
'size': download_result.get('size', 0),
}
else:
self.update_result_status(result_id, 'failed')
return {
'success': False,
'message': download_result.get('message', 'Download failed'),
}
except Exception as e:
logger.error(f"Download failed for result {result_id}: {e}")
self.update_result_status(result_id, 'failed')
return {
'success': False,
'message': str(e),
}