#!/usr/bin/env python3
"""
Instagram Client Module - Direct API downloader using curl_cffi

Downloads posts, stories, reels, and tagged posts directly via Instagram's
GraphQL and REST APIs. Uses curl_cffi with browser TLS fingerprinting
(no Playwright, no Cloudflare). Approximately 10-20x faster than ImgInn.

Posts use the public GraphQL API (no auth needed).
Stories, reels, and tagged use Instagram's REST API with session cookies.

File naming matches ImgInn format for full compatibility:
- Posts: {profile}_{YYYYMMDD_HHMMSS}_{media_id}{ext}
- Carousel: {profile}_{YYYYMMDD_HHMMSS}_{media_id}_{slide_index}{ext}
- Stories: {profile}_{YYYYMMDD_HHMMSS}_{media_id_full}_story{index}{ext}
- Tagged: {poster_username}_{YYYYMMDD_HHMMSS}_{media_id}{ext}
"""

import os
import re
import time
import random
import pickle
import subprocess
import platform
import json
from pathlib import Path
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any

from modules.base_module import LoggingMixin
from modules.instagram_rate_limiter import rate_limiter as ig_rate_limiter


def _cf_browser():
    """Resolve a supported curl_cffi browser impersonation string."""
    from curl_cffi.requests import Session
    for b in ("chrome131", "chrome136", "chrome"):
        try:
            Session(impersonate=b).close()
            return b
        except Exception:
            continue
    return "chrome"
from modules.instagram_utils import (
    extract_instagram_media_id,
    scan_existing_files_for_media_ids,
    record_instagram_download,
    is_instagram_downloaded
)


class InstagramClientDownloader(LoggingMixin):
    """Instagram downloader using direct API access via curl_cffi"""

    TIMELINE_DOC_ID = '33590632547249004'
    IG_APP_ID = '936619743392459'

    def __init__(self,
                 show_progress: bool = True,
                 use_database: bool = True,
                 log_callback=None,
                 unified_db=None):
        """Initialize the Instagram Client downloader.

        Args:
            show_progress: Whether to show download progress
            use_database: Whether to use database for dedup
            log_callback: Optional logging callback
            unified_db: UnifiedDatabase instance
        """
        self._init_logger('Instagram', log_callback, default_module='Download')

        self.scraper_id = 'instagram_client'
        self.show_progress = show_progress
        self.use_database = use_database
        self.download_count = 0
        self.pending_downloads = []
        self.downloaded_files = set()
        self.auth_failed = False
        self.user_id_failed = False
        self.invalid_owner = False

        # Session (lazy-initialized)
        self._session = None
        self._authenticated_session = None
        self._csrf_token = None
        self._user_id_cache = {}

        # Database
        if unified_db and use_database:
            self.unified_db = unified_db
        else:
            self.unified_db = None
            self.use_database = False

        # Load persistent user ID cache from database
        self._load_user_id_cache()

        # Activity status manager
        from modules.activity_status import get_activity_manager
        self.activity_manager = get_activity_manager(unified_db)

    # =========================================================================
    # SESSION MANAGEMENT
    # =========================================================================

    def _get_session(self, authenticated=False):
        """Get or create a curl_cffi session.

        Args:
            authenticated: If True, load Instaloader session cookies

        Returns:
            curl_cffi.requests.Session instance
        """
        from curl_cffi.requests import Session

        if authenticated:
            if self._authenticated_session is None:
                self._authenticated_session = Session(impersonate=_cf_browser())
                self._setup_session_headers(self._authenticated_session)
                # Load cookies: scrapers first, then pickle (legacy)
                if not self._load_db_cookies(self._authenticated_session):
                    self._visit_instagram(self._authenticated_session)
                    self._load_pickle_cookies(self._authenticated_session)
            return self._authenticated_session
        else:
            if self._session is None:
                self._session = Session(impersonate=_cf_browser())
                self._setup_session_headers(self._session)
                # Load DB cookies without sessionid to avoid per-account rate limits.
                # This gives us real Instagram cookies (mid, datr, csrftoken etc.)
                # without the homepage fetch that burns rate limits per-subprocess.
                if not self._load_db_cookies_public(self._session):
                    self._visit_instagram(self._session)
            return self._session

    def _setup_session_headers(self, session):
        """Set common Instagram headers on a session."""
        session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47',
            'X-IG-App-ID': self.IG_APP_ID,
            'X-Requested-With': 'XMLHttpRequest',
            'Referer': 'https://www.instagram.com/',
            'Origin': 'https://www.instagram.com',
            'Accept': '*/*',
            'Accept-Language': 'en-US,en;q=0.9',
        })

    def _visit_instagram(self, session):
        """Visit instagram.com to get CSRF token and initial cookies."""
        try:
            resp = session.get('https://www.instagram.com/', timeout=15)
            # Extract CSRF token from cookies
            for cookie in session.cookies.jar:
                if cookie.name == 'csrftoken':
                    self._csrf_token = cookie.value
                    session.headers['X-CSRFToken'] = cookie.value
                    break
            self.log("Initialized Instagram session", "debug")
        except Exception as e:
            self.log(f"Failed to initialize Instagram session: {e}", "warning")

    def _load_db_cookies(self, session) -> bool:
        """Load Instagram cookies from the scrapers table in the database."""
        try:
            import json as _json
            if self.unified_db and hasattr(self.unified_db, 'get_connection'):
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute(
                        "SELECT cookies_json FROM scrapers WHERE id = 'instagram'"
                    )
                    row = cursor.fetchone()
                    if not row or not row[0]:
                        return False

                    data = _json.loads(row[0])
                    cookie_list = data.get('cookies', data) if isinstance(data, dict) else data
                    if not isinstance(cookie_list, list) or not cookie_list:
                        return False

                    loaded = 0
                    csrftoken = None
                    has_sessionid = False
                    for cookie in cookie_list:
                        name = cookie.get('name', '')
                        value = cookie.get('value', '')
                        domain = cookie.get('domain', '.instagram.com')
                        if name and value:
                            session.cookies.set(name, value, domain=domain)
                            loaded += 1
                            if name == 'sessionid':
                                has_sessionid = True
                            if name == 'csrftoken':
                                csrftoken = value

                    if not has_sessionid:
                        self.log("No sessionid in database cookies", "debug")
                        return False

                    if csrftoken:
                        session.headers['X-CSRFToken'] = csrftoken
                        self._csrf_token = csrftoken

                    self.log(f"Loaded {loaded} cookies from database", "debug")
                    return True
        except Exception as e:
            self.log(f"Failed to load cookies from database: {e}", "debug")
        return False

    def _load_db_cookies_public(self, session) -> bool:
        """Load Instagram cookies from DB excluding sessionid.

        Provides real Instagram cookies (mid, datr, csrftoken) needed for
        public API endpoints without making the request authenticated.
        """
        try:
            import json as _json
            if self.unified_db and hasattr(self.unified_db, 'get_connection'):
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute(
                        "SELECT cookies_json FROM scrapers WHERE id = 'instagram'"
                    )
                    row = cursor.fetchone()
                    if not row or not row[0]:
                        return False

                    data = _json.loads(row[0])
                    cookie_list = data.get('cookies', data) if isinstance(data, dict) else data
                    if not isinstance(cookie_list, list) or not cookie_list:
                        return False

                    loaded = 0
                    csrftoken = None
                    for cookie in cookie_list:
                        name = cookie.get('name', '')
                        value = cookie.get('value', '')
                        domain = cookie.get('domain', '.instagram.com')
                        # Only load non-account cookies to avoid 401
                        public_cookies = {'csrftoken', 'mid', 'datr', 'ig_did', 'ig_nrcb'}
                        if name and value and name in public_cookies:
                            session.cookies.set(name, value, domain=domain)
                            loaded += 1
                            if name == 'csrftoken':
                                csrftoken = value

                    if csrftoken:
                        session.headers['X-CSRFToken'] = csrftoken
                        self._csrf_token = csrftoken

                    if loaded > 0:
                        self.log(f"Loaded {loaded} public cookies from database", "debug")
                        return True
        except Exception as e:
            self.log(f"Failed to load public cookies from database: {e}", "debug")
        return False

    def _load_pickle_cookies(self, session) -> bool:
        """Load cookies from Instaloader pickle session file (legacy fallback)."""
        try:
            session_file = self._get_session_file_path()
            if not session_file or not Path(session_file).exists():
                return False

            with open(session_file, 'rb') as f:
                session_data = pickle.load(f)

            cookies_dict = {}
            if isinstance(session_data, dict):
                cookies_dict = session_data
            elif hasattr(session_data, 'cookies'):
                cookies_dict = dict(session_data.cookies)

            if not cookies_dict.get('sessionid'):
                return False

            loaded = 0
            for name, value in cookies_dict.items():
                if value:
                    session.cookies.set(name, str(value), domain='.instagram.com')
                    loaded += 1

            csrftoken = cookies_dict.get('csrftoken')
            if csrftoken:
                session.headers['X-CSRFToken'] = csrftoken
                self._csrf_token = csrftoken

            self.log(f"Loaded {loaded} cookies from session file", "debug")
            return True
        except Exception as e:
            self.log(f"Failed to load pickle session: {e}", "debug")
            return False

    def _persist_cookies(self):
        """Write current authenticated session cookies back to the scrapers table."""
        if not self._authenticated_session or not self.unified_db:
            return
        if self.auth_failed:
            return
        try:
            cookies = []
            for cookie in self._authenticated_session.cookies.jar:
                cookies.append({
                    'name': cookie.name,
                    'value': cookie.value,
                    'domain': cookie.domain or '.instagram.com',
                    'path': cookie.path or '/',
                })
            if cookies and any(c['name'] == 'sessionid' for c in cookies):
                self.unified_db.save_scraper_cookies('instagram', cookies, merge=True)
                self.log(f"Persisted {len(cookies)} cookies to scrapers table", 'debug')
        except Exception as e:
            self.log(f"Failed to persist cookies: {e}", 'debug')

    def _get_session_file_path(self) -> Optional[str]:
        """Get the Instaloader session file path from config.

        Reads the instagram config from SettingsManager to find the
        session_file path and login username.
        """
        try:
            from modules.settings_manager import SettingsManager
            sm = SettingsManager('/opt/media-downloader/database/media_downloader.db')
            ig_outer = sm.get_category('instagram')
            # SettingsManager wraps in category key: {'instagram': {...actual config...}}
            ig_config = ig_outer.get('instagram', ig_outer)

            session_file = ig_config.get('session_file')
            if session_file:
                # Resolve relative paths against base directory
                if not session_file.startswith('/'):
                    session_file = f'/opt/media-downloader/{session_file}'
                return session_file

            # Fallback: construct from username
            username = ig_config.get('username')
            if username:
                return f'/opt/media-downloader/sessions/session-{username}'

            return None
        except Exception as e:
            self.log(f"Error reading session file path: {e}", "debug")
            return None

    def _has_valid_session(self) -> bool:
        """Check if a valid Instagram session is available (DB or pickle)."""
        # Check database first
        try:
            import json as _json
            if self.unified_db and hasattr(self.unified_db, 'get_connection'):
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute(
                        "SELECT cookies_json FROM scrapers WHERE id = 'instagram'"
                    )
                    row = cursor.fetchone()
                    if row and row[0]:
                        data = _json.loads(row[0])
                        cookie_list = data.get('cookies', data) if isinstance(data, dict) else data
                        if isinstance(cookie_list, list):
                            for c in cookie_list:
                                if c.get('name') == 'sessionid' and c.get('value'):
                                    return True
        except Exception:
            pass

        # Fall back to pickle file
        session_file = self._get_session_file_path()
        if not session_file or not Path(session_file).exists():
            return False
        try:
            with open(session_file, 'rb') as f:
                session_data = pickle.load(f)
            if isinstance(session_data, dict):
                return bool(session_data.get('sessionid'))
            elif hasattr(session_data, 'cookies'):
                return bool(dict(session_data.cookies).get('sessionid'))
        except Exception:
            pass
        return False

    # =========================================================================
    # USER ID / PROFILE
    # =========================================================================

    def _fetch_user_id(self, username: str) -> Optional[str]:
        """Fetch Instagram user ID for a username.

        Tries HTML profile scraping first (no auth needed, most reliable),
        then falls back to web_profile_info API.

        Args:
            username: Instagram username

        Returns:
            User ID string or None
        """
        if username in self._user_id_cache:
            return self._user_id_cache[username]

        # Throttle uncached fetches to avoid hitting Instagram rate limits
        if not hasattr(self, '_last_user_id_fetch'):
            self._last_user_id_fetch = 0
        elapsed = time.time() - self._last_user_id_fetch
        if elapsed < 2:
            time.sleep(2 - elapsed)

        # Primary: scrape user_id from profile HTML (works without auth)
        user_id = self._fetch_user_id_from_html(username)
        if user_id:
            return user_id

        # Fallback: try web_profile_info API (may need auth)
        session = self._get_session(authenticated=self._has_valid_session())
        for attempt in range(3):
            try:
                ig_rate_limiter.wait_if_needed()
                resp = session.get(
                    f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
                    timeout=15
                )
                self._last_user_id_fetch = time.time()
                if resp.status_code == 200:
                    data = resp.json()
                    user_id = data.get('data', {}).get('user', {}).get('id')
                    if user_id:
                        self._user_id_cache[username] = user_id
                        self._save_user_id_cache()
                        self.log(f"Fetched user ID for @{username}: {user_id}", "debug")
                        return user_id
                elif resp.status_code == 429:
                    self._handle_rate_limit(attempt)
                    continue
                elif resp.status_code == 404:
                    self.log(f"User @{username} not found", "warning")
                    return None
                else:
                    self.log(f"Failed to fetch user ID for @{username}: HTTP {resp.status_code}", "warning")
                    return None
            except Exception as e:
                self.log(f"Error fetching user ID for @{username}: {e}", "warning")
                return None
        return None

    def _fetch_user_id_from_html(self, username: str) -> Optional[str]:
        """Scrape user ID from the Instagram profile HTML page.

        Tries authenticated session first (Instagram requires auth to embed
        user_id in profile HTML), falls back to unauthenticated.
        """
        # Try authenticated first, then unauthenticated
        for use_auth in ([True, False] if self._has_valid_session() else [False]):
            try:
                session = self._get_session(authenticated=use_auth)
                ig_rate_limiter.wait_if_needed()
                resp = session.get(
                    f'https://www.instagram.com/{username}/',
                    timeout=15
                )
                self._last_user_id_fetch = time.time()
                if resp.status_code == 200:
                    # Try multiple patterns — Instagram changes these
                    for pattern in [r'"user_id":"(\d+)"', r'"profilePage_(\d+)"']:
                        match = re.search(pattern, resp.text)
                        if match:
                            user_id = match.group(1)
                            self._user_id_cache[username] = user_id
                            self._save_user_id_cache()
                            self.log(f"Fetched user ID for @{username} from HTML: {user_id}", "debug")
                            return user_id
                elif resp.status_code == 404:
                    self.log(f"User @{username} not found", "warning")
                    return None
                else:
                    self.log(f"Profile page for @{username} returned HTTP {resp.status_code}", "warning")
            except Exception as e:
                self.log(f"Error fetching user ID from HTML for @{username}: {e}", "warning")
        return None

    def _load_user_id_cache(self):
        """Load persistent user ID cache from database."""
        if self.unified_db:
            try:
                cached = self.unified_db.get_setting('instagram_user_id_cache')
                if cached:
                    self._user_id_cache = json.loads(cached)
                    self.log(f"Loaded {len(self._user_id_cache)} cached user IDs", "debug")
            except Exception:
                pass

    def _save_user_id_cache(self):
        """Persist user ID cache to database."""
        if self.unified_db:
            try:
                self.unified_db.set_setting('instagram_user_id_cache', json.dumps(self._user_id_cache))
            except Exception:
                pass

    def get_profile_info(self, username: str) -> Optional[Dict]:
        """Get profile info for a username.

        Returns:
            Dict with avatar_url, bio, display_name or None
        """
        session = self._get_session(authenticated=self._has_valid_session())
        try:
            resp = session.get(
                f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
                timeout=15
            )
            if resp.status_code == 200:
                user = resp.json().get('data', {}).get('user', {})
                return {
                    'avatar_url': user.get('profile_pic_url_hd') or user.get('profile_pic_url'),
                    'bio': user.get('biography', ''),
                    'display_name': user.get('full_name', username),
                }
        except Exception:
            pass
        return None

    def fetch_profile(self, username: str) -> Optional[Dict]:
        """Fetch full profile data from Instagram REST API.

        Returns the raw user object from web_profile_info including id,
        profile_pic_url_hd, biography, full_name, media_count, follower/
        following counts, verification, business status, etc.

        Returns:
            Full user dict or None
        """
        if not hasattr(self, '_last_profile_fetch'):
            self._last_profile_fetch = 0
        elapsed = time.time() - self._last_profile_fetch
        if elapsed < 2:
            time.sleep(2 - elapsed)

        session = self._get_session(authenticated=self._has_valid_session())
        try:
            ig_rate_limiter.wait_if_needed()
            resp = session.get(
                f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
                timeout=15
            )
            self._last_profile_fetch = time.time()
            if resp.status_code == 200:
                user = resp.json().get('data', {}).get('user')
                if user:
                    # Cache user_id as a side effect
                    uid = user.get('id')
                    if uid and username not in self._user_id_cache:
                        self._user_id_cache[username] = uid
                        self._save_user_id_cache()
                    return user
            elif resp.status_code == 429:
                self.log(f"Rate limited on profile for @{username}", "warning")
            elif resp.status_code == 404:
                self.log(f"Profile not found for @{username}", "warning")
            else:
                self.log(f"Profile API returned {resp.status_code} for @{username}", "warning")
        except Exception as e:
            self.log(f"Error fetching profile for @{username}: {e}", "warning")
        return None

    # =========================================================================
    # FETCH METHODS (return raw API data, no file downloads)
    # =========================================================================

    def fetch_posts(self, username: str, days_back: int = 14,
                    max_posts: int = 50, phrase_config: Dict = None,
                    max_age_hours: int = None,
                    progress_callback=None) -> List[Dict]:
        """Fetch post metadata without downloading files.

        Resolves user_id, paginates through timeline API, applies date cutoff
        and phrase filtering, returns raw v1-style nodes.

        Args:
            username: Instagram username
            days_back: How many days back to fetch
            max_posts: Maximum posts to return
            phrase_config: Phrase search config
            max_age_hours: Max age in hours (alternative to days_back)
            progress_callback: Optional callback(current_count) called after each page

        Returns:
            List of raw v1-style post nodes
        """
        user_id = self._fetch_user_id(username)
        if not user_id:
            self.log(f"Could not find user ID for @{username}", "error")
            return []

        cutoff = datetime.now() - timedelta(days=days_back)
        if max_age_hours:
            cutoff = datetime.now() - timedelta(hours=max_age_hours)

        posts = []
        self._last_pinned_posts = {}
        cursor = None
        consecutive_old = 0
        use_authenticated = False
        is_first_page = True

        while len(posts) < max_posts:
            if use_authenticated:
                result = self._fetch_timeline_page_authenticated(user_id, cursor)
                if not result:
                    break
                nodes = result.get('items', [])
                if not nodes:
                    break
                next_cursor = result.get('next_max_id')
            else:
                connection = self._fetch_timeline_page(user_id, cursor)
                if connection == 'INVALID_OWNER':
                    self.invalid_owner = True
                    use_authenticated = True
                    cursor = None
                    continue
                if not connection:
                    break
                edges = connection.get('edges', [])
                if not edges:
                    break
                nodes = [edge.get('node', {}) for edge in edges]
                page_info = connection.get('page_info', {})
                next_cursor = page_info.get('end_cursor') if page_info.get('has_next_page') else None

            # Detect pinned posts on the first page. Instagram places pinned
            # posts at the start of the feed, out of chronological order.
            # With the v1 REST API (authenticated), nodes have an explicit
            # timeline_pinned_user_ids field. For the GraphQL path (logged-out),
            # that field is absent, so we detect them by finding the timestamp
            # "jump" where chronological order resumes after the pinned block.
            if is_first_page:
                is_first_page = False
                # First try the explicit v1 API field (authenticated path)
                has_explicit = any(
                    node.get('timeline_pinned_user_ids')
                    for node in nodes[:6]
                )
                if has_explicit:
                    for node in nodes:
                        pinned_ids = node.get('timeline_pinned_user_ids')
                        if pinned_ids and isinstance(pinned_ids, list):
                            code = node.get('code', '')
                            if code:
                                self._last_pinned_posts[code] = {'pinned_at': None}
                else:
                    # GraphQL path: detect by out-of-order timestamps
                    # Pinned posts appear first but are older than subsequent posts
                    timestamps = [n.get('taken_at', 0) for n in nodes]
                    pinned_count = 0
                    for idx in range(len(timestamps) - 1):
                        if timestamps[idx + 1] > timestamps[idx]:
                            pinned_count = idx + 1
                            break
                    for node in nodes[:pinned_count]:
                        code = node.get('code', '')
                        if code:
                            self._last_pinned_posts[code] = {'pinned_at': None}

            for node in nodes:
                if len(posts) >= max_posts:
                    break

                taken_at = node.get('taken_at')
                if not taken_at:
                    continue

                post_date = datetime.fromtimestamp(taken_at)
                if post_date < cutoff:
                    consecutive_old += 1
                    if consecutive_old >= 5:
                        break
                    continue
                else:
                    consecutive_old = 0

                # Phrase filtering
                if phrase_config:
                    caption_text = ''
                    caption = node.get('caption')
                    if isinstance(caption, dict):
                        caption_text = caption.get('text', '')
                    elif isinstance(caption, str):
                        caption_text = caption
                    if not self._check_post_phrases(caption_text, phrase_config):
                        continue

                posts.append(node)

            if progress_callback:
                progress_callback(len(posts))

            if consecutive_old >= 5:
                break

            if next_cursor:
                cursor = next_cursor
                time.sleep(random.uniform(3.0, 5.0))
            else:
                break

        self.log(f"Fetched {len(posts)} posts for @{username}", "info")
        return posts

    def fetch_stories(self, username: str) -> List[Dict]:
        """Fetch story metadata without downloading files.

        Calls /api/v1/feed/user/{user_id}/story/ and returns raw story items.
        Sets auth_failed on 401/403.

        Args:
            username: Instagram username

        Returns:
            List of raw story item dicts
        """
        user_id = self._fetch_user_id(username)
        if not user_id:
            self.log(f"Could not find user ID for @{username}", "error")
            return []

        session = self._get_session(authenticated=True)
        try:
            ig_rate_limiter.wait_if_needed()
            resp = session.get(
                f'https://www.instagram.com/api/v1/feed/user/{user_id}/story/',
                timeout=20
            )

            if resp.status_code == 429:
                self.log("Rate limited on stories fetch", "warning")
                self._handle_rate_limit()
                return []

            if resp.status_code in (400, 401, 403):
                if self._has_valid_session():
                    self.log(f"Stories fetch failed: session expired or invalid (HTTP {resp.status_code})", "warning")
                    self.auth_failed = True
                else:
                    self.log(f"Stories fetch failed: no valid session (HTTP {resp.status_code})", "debug")
                return []

            if resp.status_code != 200:
                self.log(f"Stories fetch failed: HTTP {resp.status_code}", "warning")
                return []

            body = resp.text
            if not body or not body.strip():
                self.log(f"Stories fetch returned empty body for @{username}", "warning")
                return []

            data = resp.json()
            reel = data.get('reel', {})
            items = reel.get('items', []) if reel else []

        except Exception as e:
            self.log(f"Error fetching stories for @{username}: {e}", "warning")
            return []

        self._persist_cookies()

        if items:
            self.log(f"Fetched {len(items)} stories for @{username}", "info")
        else:
            self.log(f"No stories found for @{username}", "info")

        return items

    def fetch_reels(self, username: str, days_back: int = 14,
                    max_reels: int = 50, phrase_config: Dict = None) -> List[Dict]:
        """Fetch reel metadata without downloading files.

        First tries the unauthenticated GraphQL timeline (filters for
        product_type='clips'). Falls back to authenticated /api/v1/clips/user/
        if the timeline returns INVALID_OWNER or yields no results.

        Args:
            username: Instagram username
            days_back: How many days back to fetch
            max_reels: Maximum reels to return
            phrase_config: Phrase search config

        Returns:
            List of raw reel media dicts
        """
        # Try unauthenticated timeline first
        reels = self._fetch_reels_from_timeline(username, days_back, max_reels, phrase_config)
        if reels is not None:
            return reels

        # Fallback to authenticated clips API
        self.log(f"Falling back to authenticated reels API for @{username}", "info")
        return self._fetch_reels_authenticated(username, days_back, max_reels, phrase_config)

    def _fetch_reels_from_timeline(self, username: str, days_back: int,
                                    max_reels: int, phrase_config: Dict = None) -> Optional[List[Dict]]:
        """Extract reels from unauthenticated GraphQL timeline.

        Paginates through the logged-out timeline and picks out nodes with
        product_type='clips'. Returns None if the account blocks logged-out
        access (INVALID_OWNER), so the caller can fall back to authenticated API.
        """
        user_id = self._fetch_user_id(username)
        if not user_id:
            self.log(f"Could not find user ID for @{username}", "error")
            return []

        cutoff = datetime.now() - timedelta(days=days_back)
        reels = []
        cursor = None
        consecutive_old = 0

        while len(reels) < max_reels:
            connection = self._fetch_timeline_page(user_id, cursor)

            if connection == 'INVALID_OWNER':
                # Account blocks logged-out access — signal caller to use auth
                self.invalid_owner = True
                return None

            if not connection:
                break

            edges = connection.get('edges', [])
            if not edges:
                break

            for edge in edges:
                if len(reels) >= max_reels:
                    break

                node = edge.get('node', {})
                taken_at = node.get('taken_at')
                if not taken_at:
                    continue

                post_date = datetime.fromtimestamp(taken_at)
                if post_date < cutoff:
                    consecutive_old += 1
                    if consecutive_old >= 5:
                        self.log(f"Fetched {len(reels)} reels from timeline for @{username}", "info")
                        return reels
                    continue
                else:
                    consecutive_old = 0

                # Only keep reels (product_type='clips')
                if node.get('product_type') != 'clips':
                    continue

                if phrase_config:
                    caption = node.get('caption', {})
                    caption_text = caption.get('text', '') if isinstance(caption, dict) else ''
                    if not self._check_post_phrases(caption_text, phrase_config):
                        continue

                reels.append(node)

            if consecutive_old >= 5:
                break

            page_info = connection.get('page_info', {})
            if page_info.get('has_next_page') and page_info.get('end_cursor'):
                cursor = page_info['end_cursor']
                time.sleep(random.uniform(2.0, 4.0))
            else:
                break

        self.log(f"Fetched {len(reels)} reels from timeline for @{username}", "info")
        return reels

    def _fetch_reels_authenticated(self, username: str, days_back: int,
                                    max_reels: int, phrase_config: Dict = None) -> List[Dict]:
        """Fetch reels via authenticated /api/v1/clips/user/ endpoint."""
        user_id = self._fetch_user_id(username)
        if not user_id:
            return []

        session = self._get_session(authenticated=True)
        cutoff = datetime.now() - timedelta(days=days_back)

        reels = []
        max_id = None

        while len(reels) < max_reels:
            try:
                post_data = {
                    'target_user_id': user_id,
                    'page_size': 12,
                }
                if max_id:
                    post_data['max_id'] = max_id

                ig_rate_limiter.wait_if_needed()
                resp = session.post(
                    'https://www.instagram.com/api/v1/clips/user/',
                    data=post_data,
                    timeout=20
                )

                if resp.status_code == 429:
                    self.log("Rate limited on reels fetch", "warning")
                    self._handle_rate_limit()
                    break

                if resp.status_code in (400, 401, 403):
                    if self._has_valid_session():
                        self.log(f"Reels fetch failed: session expired or invalid (HTTP {resp.status_code})", "warning")
                        self.auth_failed = True
                    else:
                        self.log(f"Reels fetch failed: no valid session (HTTP {resp.status_code})", "debug")
                    break

                if resp.status_code != 200:
                    self.log(f"Reels fetch failed: HTTP {resp.status_code}", "warning")
                    break

                data = resp.json()
                items = data.get('items', [])

            except Exception as e:
                self.log(f"Error fetching reels for @{username}: {e}", "warning")
                break

            if not items:
                break

            consecutive_old = 0
            for item_wrapper in items:
                if len(reels) >= max_reels:
                    break

                media = item_wrapper.get('media', {})
                taken_at = media.get('taken_at')
                if not taken_at:
                    continue

                post_date = datetime.fromtimestamp(taken_at)
                if post_date < cutoff:
                    consecutive_old += 1
                    if consecutive_old >= 5:
                        self._persist_cookies()
                        self.log(f"Fetched {len(reels)} reels for @{username}", "info")
                        return reels
                    continue
                else:
                    consecutive_old = 0

                if phrase_config:
                    caption = media.get('caption', {})
                    caption_text = caption.get('text', '') if isinstance(caption, dict) else ''
                    if not self._check_post_phrases(caption_text, phrase_config):
                        continue

                reels.append(media)

            paging_info = data.get('paging_info', {})
            if paging_info.get('more_available') and paging_info.get('max_id'):
                max_id = paging_info['max_id']
                time.sleep(random.uniform(2.0, 4.0))
            else:
                break

        self._persist_cookies()
        self.log(f"Fetched {len(reels)} reels for @{username}", "info")
        return reels

    # =========================================================================
    # MAIN ENTRY POINT
    # =========================================================================

    def download(self, username: str, content_type: str = "posts",
                 days_back: int = 14, max_downloads: int = 50,
                 output_dir: str = None, phrase_config: Dict = None,
                 defer_database: bool = False) -> int:
        """Download Instagram content for a username.

        This matches the ImgInn interface exactly.

        Args:
            username: Instagram username
            content_type: One of 'posts', 'stories', 'reels', 'tagged'
            days_back: How many days back to download
            max_downloads: Maximum number of items to download
            output_dir: Output directory (temp dir)
            phrase_config: Optional phrase search config
            defer_database: If True, don't record to DB (caller will do it)

        Returns:
            Number of files downloaded
        """
        self.log(f"Starting {content_type} download for @{username} (days_back={days_back})", "info")

        # Check user ID resolution upfront
        user_id = self._fetch_user_id(username)
        if not user_id:
            self.log(f"Could not resolve user ID for @{username}", "warning")
            self.user_id_failed = True
            return 0

        if content_type == 'posts':
            results = self.download_posts(
                username, days_back=days_back, max_posts=max_downloads,
                output_dir=output_dir, phrase_config=phrase_config,
                defer_database=defer_database
            )
        elif content_type == 'stories':
            if not self._has_valid_session():
                self.log("Stories require authentication - no valid session found, skipping", "warning")
                return 0
            results = self.download_stories(
                username, days_back=days_back, max_stories=max_downloads,
                output_dir=output_dir, defer_database=defer_database
            )
        elif content_type == 'reels':
            results = self.download_reels(
                username, days_back=days_back, max_reels=max_downloads,
                output_dir=output_dir, phrase_config=phrase_config,
                defer_database=defer_database
            )
        elif content_type == 'tagged':
            if not self._has_valid_session():
                self.log("Tagged posts require authentication - no valid session found, skipping", "warning")
                return 0
            results = self.download_tagged(
                username, days_back=days_back, max_posts=max_downloads,
                output_dir=output_dir, phrase_config=phrase_config,
                defer_database=defer_database
            )
        else:
            self.log(f"Unknown content type: {content_type}", "error")
            return 0

        count = len(results) if results else 0
        self.download_count += count
        return count

    # =========================================================================
    # POSTS (public, no auth needed)
    # =========================================================================

    def download_posts(self, username: str, days_back: int = 14,
                       max_posts: int = 50, specific_post_url: str = None,
                       output_dir: str = None, phrase_config: Dict = None,
                       skip_database: bool = False, max_age_hours: int = None,
                       defer_database: bool = False) -> list:
        """Download posts for a username via GraphQL API (no auth needed).

        Uses fetch_posts() to get metadata, then downloads each item.

        Args:
            username: Instagram username
            days_back: How many days back to fetch
            max_posts: Maximum posts to download
            specific_post_url: Download a specific post URL
            output_dir: Output directory
            phrase_config: Phrase search config
            skip_database: Skip database checks
            max_age_hours: Max age in hours (alternative to days_back)
            defer_database: Defer database recording

        Returns:
            List of downloaded file info dicts
        """
        if output_dir:
            out_path = Path(output_dir) / username
            out_path.mkdir(parents=True, exist_ok=True)
        else:
            out_path = Path('.')

        self.activity_manager.update_status("Fetching posts", progress_current=0, progress_total=0)

        nodes = self.fetch_posts(username, days_back=days_back, max_posts=max_posts,
                                 phrase_config=phrase_config, max_age_hours=max_age_hours)

        downloaded = []
        posts_processed = 0

        for node in nodes:
            taken_at = node.get('taken_at')
            if not taken_at:
                continue

            post_date = datetime.fromtimestamp(taken_at)
            media_type = node.get('media_type')
            posts_processed += 1
            media_label = 'video' if media_type == 2 else 'image'

            self.activity_manager.update_status(
                "Downloading posts",
                progress_current=posts_processed,
                progress_total=len(nodes)
            )

            if media_type == 8:
                carousel_media = node.get('carousel_media', [])
                self.log(f"Post {posts_processed}: carousel with {len(carousel_media)} files ({post_date.strftime('%Y-%m-%d')})", "info")
                for idx, cm in enumerate(carousel_media):
                    result = self._download_v1_media(
                        cm, username, post_date, out_path,
                        skip_database=skip_database,
                        defer_database=defer_database,
                        content_type='posts',
                        carousel_index=idx + 1
                    )
                    if result:
                        downloaded.append(result)
            else:
                self.log(f"Post {posts_processed}: downloading {media_label} ({post_date.strftime('%Y-%m-%d')})", "info")
                result = self._download_v1_media(
                    node, username, post_date, out_path,
                    skip_database=skip_database,
                    defer_database=defer_database,
                    content_type='posts'
                )
                if result:
                    downloaded.append(result)

        self.log(f"Downloaded {len(downloaded)} files from {posts_processed} posts for @{username}", "info")
        return downloaded

    def _fetch_timeline_page(self, user_id: str, cursor: str = None):
        """Fetch one page of timeline posts via GraphQL (no auth needed).

        Uses OwnerToTimelineMediaLoggedOutQuery. Response is at
        data.xdt_get_owner_to_timeline_media_logged_out.

        Args:
            user_id: Instagram user ID
            cursor: Pagination cursor (end_cursor from previous page)

        Returns:
            Connection dict with edges and page_info, 'INVALID_OWNER' string
            if the account blocks logged-out access, or None on failure
        """
        import json as _json
        session = self._get_session(authenticated=False)

        variables = {
            'owner_id': user_id,
            'first': 50,
        }
        if cursor:
            variables['after'] = cursor

        try:
            resp = session.post(
                'https://www.instagram.com/graphql/query',
                data={
                    'doc_id': self.TIMELINE_DOC_ID,
                    'variables': _json.dumps(variables),
                },
                timeout=30
            )

            if resp.status_code == 400:
                self.log(
                    "GraphQL returned 400 - doc_id may be outdated. "
                    "Update TIMELINE_DOC_ID in instagram_client_module.py.",
                    'error'
                )
                return None

            if resp.status_code == 429:
                self.log("Rate limited on timeline fetch, backing off", "warning")
                self._handle_rate_limit()
                return None

            if resp.status_code != 200:
                self.log(f"Timeline fetch failed: HTTP {resp.status_code}", "warning")
                return None

            data = resp.json()

            # Check for errors - but still try to extract data since
            # some errors are non-critical (e.g., individual field errors)
            if data.get('errors'):
                # Only fail if there's no data at all
                if not data.get('data'):
                    # Check for "Invalid owner id" - account may block logged-out access
                    err_descriptions = [e.get('description', '') for e in data['errors']]
                    if any('Invalid owner id' in d for d in err_descriptions):
                        self.log(
                            f"GraphQL 'Invalid owner id' for user {user_id} - "
                            "account blocks logged-out timeline, will try authenticated API",
                            'warning'
                        )
                        return 'INVALID_OWNER'
                    self.log(
                        f"GraphQL errors with no data: {data['errors'][:2]} - "
                        "doc_id may be outdated. Update TIMELINE_DOC_ID.",
                        'error'
                    )
                    return None
                self.log(f"GraphQL returned {len(data['errors'])} partial errors (continuing)", "debug")

            connection = (data.get('data') or {}).get(
                'xdt_get_owner_to_timeline_media_logged_out'
            )
            if not connection:
                self.log(
                    "GraphQL returned empty data - doc_id may be outdated. "
                    "Update TIMELINE_DOC_ID in instagram_client_module.py.",
                    'error'
                )
                return None

            return connection

        except Exception as e:
            self.log(f"Error fetching timeline: {e}", "warning")
            return None

    def _fetch_timeline_page_authenticated(self, user_id: str, max_id: str = None) -> Optional[Dict]:
        """Fetch one page of timeline posts via authenticated REST API.

        Fallback for accounts where OwnerToTimelineMediaLoggedOutQuery returns
        'Invalid owner id'. Uses /api/v1/feed/user/{user_id}/ which requires auth.

        Args:
            user_id: Instagram user ID
            max_id: Pagination cursor (next_max_id from previous page)

        Returns:
            Dict with 'items' list and 'next_max_id', or None on failure
        """
        session = self._get_session(authenticated=True)

        url = f'https://www.instagram.com/api/v1/feed/user/{user_id}/'
        params = {'count': 33}
        if max_id:
            params['max_id'] = max_id

        try:
            ig_rate_limiter.wait_if_needed()
            resp = session.get(url, params=params, timeout=30)

            if resp.status_code == 429:
                self.log("Rate limited on authenticated timeline fetch, backing off", "warning")
                self._handle_rate_limit()
                return None

            if resp.status_code in (400, 401, 403):
                # Only flag auth_failed if we actually had a session — otherwise
                # there's nothing to expire and no notification should fire
                if self._has_valid_session():
                    self.log(f"Authenticated timeline fetch failed: session expired or invalid (HTTP {resp.status_code})", "warning")
                    self.auth_failed = True
                else:
                    self.log(f"Authenticated timeline fetch failed: no valid session (HTTP {resp.status_code})", "debug")
                return None

            if resp.status_code != 200:
                self.log(f"Authenticated timeline fetch failed: HTTP {resp.status_code}", "warning")
                return None

            data = resp.json()
            items = data.get('items', [])
            next_max_id = data.get('next_max_id')
            more_available = data.get('more_available', False)

            return {
                'items': items,
                'next_max_id': next_max_id if more_available else None,
            }

        except Exception as e:
            self.log(f"Error fetching authenticated timeline: {e}", "warning")
            return None

    # =========================================================================
    # STORIES (requires auth)
    # =========================================================================

    def download_stories(self, username: str, days_back: int = 1,
                         max_stories: int = 50, output_dir: str = None,
                         skip_database: bool = False,
                         defer_database: bool = False) -> list:
        """Download stories for a username.

        Uses fetch_stories() to get metadata, then downloads each item.

        Args:
            username: Instagram username
            days_back: How many days back (stories are typically 24h)
            max_stories: Maximum stories to download
            output_dir: Output directory
            skip_database: Skip database checks
            defer_database: Defer database recording

        Returns:
            List of downloaded file info dicts
        """
        if output_dir:
            out_path = Path(output_dir) / username
            out_path.mkdir(parents=True, exist_ok=True)
        else:
            out_path = Path('.')

        cutoff = datetime.now() - timedelta(days=days_back)

        items = self.fetch_stories(username)
        if not items:
            return []

        self.activity_manager.update_status(
            "Downloading stories",
            progress_current=0,
            progress_total=len(items)
        )

        downloaded = []
        story_num = 0
        for idx, item in enumerate(items):
            if len(downloaded) >= max_stories:
                break

            taken_at = item.get('taken_at')
            if not taken_at:
                continue

            post_date = datetime.fromtimestamp(taken_at)
            if post_date < cutoff:
                continue

            media_type = item.get('media_type')
            story_num += 1
            media_label = 'video' if media_type == 2 else 'image'
            self.log(f"Story {story_num}/{len(items)}: downloading {media_label} ({post_date.strftime('%Y-%m-%d %H:%M')})", "info")
            self.activity_manager.update_status(
                "Downloading stories",
                progress_current=story_num,
                progress_total=len(items)
            )

            if media_type == 2:  # Video
                video_versions = item.get('video_versions', [])
                if video_versions:
                    best = max(video_versions, key=lambda v: v.get('width', 0) * v.get('height', 0))
                    url = best.get('url')
                    ext = '.mp4'
                else:
                    continue
            else:  # Image
                candidates = item.get('image_versions2', {}).get('candidates', [])
                if candidates:
                    best = max(candidates, key=lambda c: c.get('width', 0) * c.get('height', 0))
                    url = best.get('url')
                    ext = '.jpg'
                else:
                    continue

            media_id = str(item.get('pk', item.get('id', '')))
            media_id_full = media_id

            if not skip_database and self._is_already_downloaded(media_id):
                self.log(f"Story already downloaded: {media_id}", "debug")
                continue

            date_str = post_date.strftime('%Y%m%d_%H%M%S')
            filename = f"{username}_{date_str}_{media_id_full}_story{idx + 1}{ext}"

            filepath = out_path / filename
            if self._download_file(url, filepath):
                self._update_file_timestamps(filepath, post_date)

                result = {
                    'filename': filename,
                    'media_id': media_id,
                    'username': username,
                    'content_type': 'stories',
                    'post_date': post_date.isoformat(),
                    'url': url,
                }
                downloaded.append(result)

                if defer_database:
                    self.pending_downloads.append(result)
                elif not skip_database:
                    self._record_download(
                        media_id=media_id, username=username,
                        content_type='stories', filename=filename,
                        url=url, post_date=post_date,
                        file_path=str(filepath)
                    )

                time.sleep(random.uniform(1.0, 2.0))

        self.log(f"Downloaded {len(downloaded)} stories for @{username}", "info")
        return downloaded

    # =========================================================================
    # REELS (requires auth)
    # =========================================================================

    def download_reels(self, username: str, days_back: int = 14,
                       max_reels: int = 50, output_dir: str = None,
                       phrase_config: Dict = None,
                       defer_database: bool = False) -> list:
        """Download reels for a username.

        Uses fetch_reels() to get metadata, then downloads each item.

        Args:
            username: Instagram username
            days_back: How many days back to fetch
            max_reels: Maximum reels to download
            output_dir: Output directory
            phrase_config: Phrase search config
            defer_database: Defer database recording

        Returns:
            List of downloaded file info dicts
        """
        if output_dir:
            out_path = Path(output_dir) / username
            out_path.mkdir(parents=True, exist_ok=True)
        else:
            out_path = Path('.')

        media_items = self.fetch_reels(username, days_back=days_back,
                                       max_reels=max_reels, phrase_config=phrase_config)

        downloaded = []
        reel_num = 0

        self.activity_manager.update_status(
            "Downloading reels",
            progress_current=0,
            progress_total=len(media_items)
        )

        for media in media_items:
            if len(downloaded) >= max_reels:
                break

            taken_at = media.get('taken_at')
            if not taken_at:
                continue

            post_date = datetime.fromtimestamp(taken_at)

            video_versions = media.get('video_versions', [])
            if not video_versions:
                continue

            best = max(video_versions, key=lambda v: v.get('width', 0) * v.get('height', 0))
            url = best.get('url')
            media_id = str(media.get('pk', media.get('id', '')))

            if self._is_already_downloaded(media_id):
                self.log(f"Reel already downloaded: {media_id}", "debug")
                continue

            reel_num += 1
            self.log(f"Reel {reel_num}: downloading video ({post_date.strftime('%Y-%m-%d')})", "info")
            self.activity_manager.update_status(
                "Downloading reels",
                progress_current=reel_num,
                progress_total=len(media_items)
            )

            date_str = post_date.strftime('%Y%m%d_%H%M%S')
            filename = f"{username}_{date_str}_{media_id}.mp4"

            filepath = out_path / filename
            if self._download_file(url, filepath):
                self._update_file_timestamps(filepath, post_date)

                result = {
                    'filename': filename,
                    'media_id': media_id,
                    'username': username,
                    'content_type': 'reels',
                    'post_date': post_date.isoformat(),
                    'url': url,
                }
                downloaded.append(result)

                if defer_database:
                    self.pending_downloads.append(result)
                else:
                    self._record_download(
                        media_id=media_id, username=username,
                        content_type='reels', filename=filename,
                        url=url, post_date=post_date,
                        file_path=str(filepath)
                    )

                time.sleep(random.uniform(1.0, 2.0))

        self.log(f"Downloaded {len(downloaded)} reels for @{username}", "info")
        return downloaded

    # =========================================================================
    # TAGGED (requires auth)
    # =========================================================================

    def download_tagged(self, username: str, days_back: int = 14,
                        max_posts: int = 50, output_dir: str = None,
                        phrase_config: Dict = None,
                        defer_database: bool = False) -> list:
        """Download tagged posts for a username.

        Args:
            username: Instagram username (the tagged user)
            days_back: How many days back to fetch
            max_posts: Maximum posts to download
            output_dir: Output directory
            phrase_config: Phrase search config
            defer_database: Defer database recording

        Returns:
            List of downloaded file info dicts
        """
        user_id = self._fetch_user_id(username)
        if not user_id:
            self.log(f"Could not find user ID for @{username}", "error")
            return []

        if output_dir:
            out_path = Path(output_dir) / username
            out_path.mkdir(parents=True, exist_ok=True)
        else:
            out_path = Path('.')

        session = self._get_session(authenticated=True)
        cutoff = datetime.now() - timedelta(days=days_back)

        downloaded = []
        posts_processed = 0
        max_id = None
        page_num = 0

        while posts_processed < max_posts:
            page_num += 1
            self.log(f"Fetching tagged page {page_num} for @{username}", "debug")

            try:
                params = {}
                if max_id:
                    params['max_id'] = max_id

                ig_rate_limiter.wait_if_needed()
                resp = session.get(
                    f'https://www.instagram.com/api/v1/usertags/{user_id}/feed/',
                    params=params,
                    timeout=20
                )

                if resp.status_code == 429:
                    self.log("Rate limited on tagged fetch", "warning")
                    self._handle_rate_limit()
                    break

                if resp.status_code in (400, 401, 403):
                    if self._has_valid_session():
                        self.log(f"Tagged fetch failed: session expired or invalid (HTTP {resp.status_code}) - please re-login from the Scrapers page", "warning")
                        self.auth_failed = True
                    else:
                        self.log(f"Tagged fetch failed: no valid session (HTTP {resp.status_code})", "debug")
                    break

                if resp.status_code != 200:
                    self.log(f"Tagged fetch failed: HTTP {resp.status_code}", "warning")
                    break

                body = resp.text
                if not body or not body.strip():
                    self.log(f"Tagged fetch returned empty body for @{username} (status={resp.status_code}, content-type={resp.headers.get('content-type', 'N/A')})", "warning")
                    break

                data = resp.json()
                items = data.get('items', [])

            except Exception as e:
                self.log(f"Error fetching tagged for @{username}: {e}", "warning")
                break

            if not items:
                break

            consecutive_old = 0
            for item in items:
                if posts_processed >= max_posts:
                    break

                taken_at = item.get('taken_at')
                if not taken_at:
                    continue

                post_date = datetime.fromtimestamp(taken_at)
                if post_date < cutoff:
                    consecutive_old += 1
                    if consecutive_old >= 5:
                        break
                    continue
                else:
                    consecutive_old = 0

                # Check phrase filter
                if phrase_config:
                    caption = item.get('caption', {})
                    caption_text = caption.get('text', '') if isinstance(caption, dict) else ''
                    if not self._check_post_phrases(caption_text, phrase_config):
                        continue

                # Poster username (who posted, not the tagged user)
                poster_username = item.get('user', {}).get('username', username)

                # Handle carousel vs single media
                carousel_media = item.get('carousel_media', [])
                media_type = item.get('media_type')
                posts_processed += 1
                media_label = 'video' if media_type == 2 else 'image'

                self.activity_manager.update_status(
                    "Downloading tagged",
                    progress_current=posts_processed,
                    progress_total=max_posts
                )

                if carousel_media:
                    self.log(f"Tagged {posts_processed}: carousel with {len(carousel_media)} files by @{poster_username} ({post_date.strftime('%Y-%m-%d')})", "info")
                    for idx, cm in enumerate(carousel_media):
                        result = self._download_tagged_media(
                            cm, poster_username, post_date, out_path,
                            defer_database=defer_database,
                            carousel_index=idx + 1
                        )
                        if result:
                            downloaded.append(result)
                else:
                    self.log(f"Tagged {posts_processed}: downloading {media_label} by @{poster_username} ({post_date.strftime('%Y-%m-%d')})", "info")
                    result = self._download_tagged_media(
                        item, poster_username, post_date, out_path,
                        defer_database=defer_database
                    )
                    if result:
                        downloaded.append(result)

            if consecutive_old >= 5:
                break

            # Check pagination
            if data.get('more_available') and data.get('next_max_id'):
                max_id = data['next_max_id']
                time.sleep(random.uniform(2.0, 4.0))
            else:
                break

        self._persist_cookies()
        self.log(f"Downloaded {len(downloaded)} files from {posts_processed} tagged posts for @{username}", "info")
        return downloaded

    def _download_tagged_media(self, media: Dict, poster_username: str,
                               post_date: datetime, out_path: Path,
                               defer_database: bool = False,
                               carousel_index: int = None) -> Optional[Dict]:
        """Download a single tagged media item.

        Uses poster_username in filename (not the tagged user).
        """
        media_type = media.get('media_type')
        media_id = str(media.get('pk', media.get('id', '')))

        # Check if already downloaded
        if self._is_already_downloaded(media_id):
            return None

        if media_type == 2:  # Video
            video_versions = media.get('video_versions', [])
            if not video_versions:
                return None
            best = max(video_versions, key=lambda v: v.get('width', 0) * v.get('height', 0))
            url = best.get('url')
            ext = '.mp4'
        else:  # Image
            candidates = media.get('image_versions2', {}).get('candidates', [])
            if not candidates:
                return None
            best = max(candidates, key=lambda c: c.get('width', 0) * c.get('height', 0))
            url = best.get('url')
            ext = '.jpg'

        date_str = post_date.strftime('%Y%m%d_%H%M%S')
        if carousel_index:
            filename = f"{poster_username}_{date_str}_{media_id}_{carousel_index}{ext}"
        else:
            filename = f"{poster_username}_{date_str}_{media_id}{ext}"

        filepath = out_path / filename
        if self._download_file(url, filepath):
            self._update_file_timestamps(filepath, post_date)

            result = {
                'filename': filename,
                'media_id': media_id,
                'username': poster_username,
                'content_type': 'tagged',
                'post_date': post_date.isoformat(),
                'url': url,
            }

            if defer_database:
                self.pending_downloads.append(result)
            else:
                self._record_download(
                    media_id=media_id, username=poster_username,
                    content_type='tagged', filename=filename,
                    url=url, post_date=post_date,
                    file_path=str(filepath)
                )

            time.sleep(random.uniform(1.0, 2.0))
            return result

        return None

    # =========================================================================
    # MEDIA DOWNLOAD HELPERS
    # =========================================================================

    def _download_v1_media(self, media: Dict, username: str,
                           post_date: datetime, out_path: Path,
                           skip_database: bool = False,
                           defer_database: bool = False,
                           content_type: str = 'posts',
                           carousel_index: int = None) -> Optional[Dict]:
        """Download media from a v1 API response item.

        Handles image_versions2/video_versions format used by all v1 endpoints.
        """
        media_type = media.get('media_type')
        media_id = str(media.get('pk', media.get('id', '')))

        if not media_id:
            return None

        # Check if already downloaded
        if not skip_database and self._is_already_downloaded(media_id):
            self.log(f"Already downloaded: {media_id}", "debug")
            return None

        if media_type == 2:  # Video
            video_versions = media.get('video_versions', [])
            if not video_versions:
                return None
            best = max(video_versions, key=lambda v: v.get('width', 0) * v.get('height', 0))
            url = best.get('url')
            ext = '.mp4'
        else:  # Image (type 1) or other
            candidates = media.get('image_versions2', {}).get('candidates', [])
            if not candidates:
                return None
            best = max(candidates, key=lambda c: c.get('width', 0) * c.get('height', 0))
            url = best.get('url')
            ext = '.jpg'

        if not url:
            return None

        # Build filename
        date_str = post_date.strftime('%Y%m%d_%H%M%S')
        if carousel_index:
            filename = f"{username}_{date_str}_{media_id}_{carousel_index}{ext}"
        else:
            filename = f"{username}_{date_str}_{media_id}{ext}"

        filepath = out_path / filename
        if self._download_file(url, filepath):
            self._update_file_timestamps(filepath, post_date)

            result = {
                'filename': filename,
                'media_id': media_id,
                'username': username,
                'content_type': content_type,
                'post_date': post_date.isoformat(),
                'url': url,
            }

            if defer_database:
                self.pending_downloads.append(result)
            elif not skip_database:
                self._record_download(
                    media_id=media_id, username=username,
                    content_type=content_type, filename=filename,
                    url=url, post_date=post_date,
                    file_path=str(filepath)
                )

            # Rate limit between CDN downloads
            time.sleep(random.uniform(1.0, 2.0))
            return result

        return None

    def _update_file_timestamps(self, filepath: Path, post_date: datetime):
        """Update all timestamps for a file to match the post date"""
        try:
            timestamp = post_date.timestamp()

            # 1. Update file system timestamps (access time and modification time)
            os.utime(filepath, (timestamp, timestamp))
            self.log(f"Updated file timestamps to {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug")

            # 2. Try to update creation time (platform-specific)
            if platform.system() == 'Darwin':
                date_str = post_date.strftime('%m/%d/%Y %H:%M:%S')
                try:
                    subprocess.run(
                        ['SetFile', '-d', date_str, str(filepath)],
                        capture_output=True, text=True
                    )
                except (subprocess.SubprocessError, FileNotFoundError, OSError):
                    pass

            # 3. Update EXIF data for images
            if str(filepath).lower().endswith(('.jpg', '.jpeg', '.png', '.heic')):
                self._update_exif_timestamps(filepath, post_date)

        except Exception as e:
            self.log(f"Error updating timestamps: {e}", "warning")

    def _update_exif_timestamps(self, filepath: Path, post_date: datetime):
        """Update EXIF timestamps in image files"""
        try:
            result = subprocess.run(['which', 'exiftool'], capture_output=True, text=True)
            if result.returncode == 0:
                exif_date = post_date.strftime('%Y:%m:%d %H:%M:%S')
                cmd = [
                    'exiftool', '-overwrite_original', '-quiet',
                    f'-AllDates={exif_date}',
                    f'-MetadataDate={exif_date}',
                    '-HistoryWhen=',
                    f'-FileModifyDate={exif_date}',
                    str(filepath)
                ]
                subprocess.run(cmd, capture_output=True, text=True)
                self.log(f"Updated EXIF timestamps", "debug")
        except Exception:
            pass

    def _download_file(self, url: str, filepath: Path, max_retries: int = 2) -> bool:
        """Download a file from URL to filepath with retry on timeout.

        Uses the curl_cffi session for CDN downloads.

        Args:
            url: URL to download
            filepath: Path to save the file
            max_retries: Number of retries on timeout

        Returns:
            True if successful
        """
        filepath.parent.mkdir(parents=True, exist_ok=True)
        session = self._get_session()
        timeouts = [30, 60, 90]

        for attempt in range(max_retries + 1):
            try:
                timeout = timeouts[min(attempt, len(timeouts) - 1)]
                resp = session.get(url, timeout=timeout)

                if resp.status_code != 200:
                    self.log(f"Download failed: HTTP {resp.status_code} for {filepath.name}", "warning")
                    return False

                filepath.write_bytes(resp.content)

                if self.show_progress:
                    size_mb = len(resp.content) / (1024 * 1024)
                    self.log(f"Downloaded: {filepath.name} ({size_mb:.1f}MB)", "debug")

                return True

            except Exception as e:
                if attempt < max_retries and 'timed out' in str(e).lower():
                    self.log(f"Timeout downloading {filepath.name}, retrying ({attempt + 1}/{max_retries})...", "warning")
                    time.sleep(2)
                    continue
                self.log(f"Error downloading {filepath.name}: {e}", "warning")
                return False

    # =========================================================================
    # PHRASE MATCHING
    # =========================================================================

    def _check_post_phrases(self, caption_text: str, phrase_config: Dict) -> bool:
        """Check if a post caption matches the phrase filter.

        Args:
            caption_text: Post caption text
            phrase_config: Dict with 'phrases', 'case_sensitive', 'match_all', 'download_all'

        Returns:
            True if the post matches (or no filter active)
        """
        if not phrase_config or not phrase_config.get('enabled'):
            return True

        if phrase_config.get('download_all'):
            return True

        phrases = phrase_config.get('phrases', [])
        if not phrases:
            return True

        case_sensitive = phrase_config.get('case_sensitive', False)
        match_all = phrase_config.get('match_all', False)

        text = caption_text if case_sensitive else caption_text.lower()

        if match_all:
            return all(
                (p if case_sensitive else p.lower()) in text
                for p in phrases
            )
        else:
            return any(
                (p if case_sensitive else p.lower()) in text
                for p in phrases
            )

    # =========================================================================
    # DATABASE
    # =========================================================================

    def _is_already_downloaded(self, media_id: str) -> bool:
        """Check if media has already been downloaded."""
        if not self.use_database or not self.unified_db:
            return media_id in self.downloaded_files

        return is_instagram_downloaded(self.unified_db, media_id)

    def _record_download(self, media_id: str, username: str,
                         content_type: str, filename: str,
                         url: str = None, post_date: datetime = None,
                         file_path: str = None):
        """Record a download to the database."""
        self.downloaded_files.add(media_id)

        if not self.use_database or not self.unified_db:
            return

        record_instagram_download(
            db=self.unified_db,
            media_id=media_id,
            username=username,
            content_type=content_type,
            filename=filename,
            url=url,
            post_date=post_date,
            file_path=file_path,
            method='instagram_client'
        )

    def _get_processed_posts(self, username: str) -> set:
        """Get set of already processed media IDs for a username."""
        if not self.use_database or not self.unified_db:
            return self.downloaded_files

        processed = set()
        try:
            if hasattr(self.unified_db, 'get_connection'):
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute('''
                        SELECT media_id FROM downloads
                        WHERE platform = 'instagram'
                        AND file_path IS NOT NULL AND file_path != ''
                    ''')
                    for row in cursor.fetchall():
                        if row[0]:
                            processed.add(row[0])
        except Exception as e:
            self.log(f"Error fetching processed posts: {e}", "warning")

        return processed

    def get_pending_downloads(self) -> list:
        """Get list of pending downloads (for deferred recording)."""
        return self.pending_downloads

    def clear_pending_downloads(self):
        """Clear the pending downloads list."""
        self.pending_downloads = []

    # =========================================================================
    # RATE LIMITING
    # =========================================================================

    def _handle_rate_limit(self, attempt: int = 0):
        """Handle 429 rate limit with exponential backoff.

        Args:
            attempt: Current retry attempt (0-based)
        """
        # Inform the shared rate limiter that we hit a 429
        ig_rate_limiter.track_call()

        backoff_times = [60, 120, 300]
        max_attempts = len(backoff_times)

        if attempt >= max_attempts:
            self.log("Max rate limit retries exceeded", "error")
            return

        wait_time = backoff_times[min(attempt, max_attempts - 1)]
        self.log(f"Rate limited - waiting {wait_time}s (attempt {attempt + 1}/{max_attempts})", "warning")
        time.sleep(wait_time)
