"""
Instagram Adapter for Paid Content — fetches Instagram posts, stories,
highlights, and reels via a combination of Instagram GraphQL API,
authenticated browser cookies, and ImgInn scraping.

Converts raw data to the Post/Attachment models used by the paid content system.
"""

import asyncio
import re
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Set

from modules.base_module import LoggingMixin
from .models import Creator, Post, Attachment


class InstagramAdapter(LoggingMixin):
    """Async adapter for Instagram paid content (posts, stories, highlights, reels)."""

    SERVICE_ID = 'instagram'
    PLATFORM = 'instagram'

    def __init__(self, unified_db=None, log_callback=None):
        self._init_logger('PaidContent', log_callback, default_module='Instagram')
        self.unified_db = unified_db
        self._client = None
        self._last_fetched_posts: List[Post] = []  # Cache for reels reuse

    def _get_client(self):
        """Lazily create the ImgInnAPIDownloader."""
        if self._client is None:
            from modules.imginn_api_module import ImgInnAPIDownloader
            self._client = ImgInnAPIDownloader(
                show_progress=False,
                use_database=True,
                log_callback=self.log_callback if hasattr(self, 'log_callback') else None,
                unified_db=self.unified_db,
            )
        return self._client

    def cleanup(self):
        """Clean up resources."""
        self._client = None

    # ── Static helpers ──────────────────────────────────────────

    @staticmethod
    def extract_username(url: str) -> Optional[str]:
        match = re.search(r'instagram\.com/([a-zA-Z0-9_.]+)/?', url)
        if match:
            username = match.group(1).lower()
            non_usernames = {
                'explore', 'reels', 'stories', 'p', 'tv', 'accounts',
                'direct', 'about', 'legal', 'developer', 'privacy',
                'terms', 'help', 'api', 'reel', 'tags'
            }
            if username not in non_usernames:
                return username
        return None

    @staticmethod
    def normalize_creator_url(username: str) -> str:
        if username.startswith('http://') or username.startswith('https://'):
            return username
        return f"https://www.instagram.com/{username}"

    # ── Count parsing ─────────────────────────────────────────────

    @staticmethod
    def _parse_count(value) -> Optional[int]:
        """Parse ImgInn string counts like '1.2M', '3,456', '500K' to int."""
        if value is None:
            return None
        if isinstance(value, int):
            return value
        if isinstance(value, float):
            return int(value)
        s = str(value).strip().replace(',', '')
        if not s:
            return None
        multiplier = 1
        if s[-1].upper() == 'M':
            multiplier = 1_000_000
            s = s[:-1]
        elif s[-1].upper() == 'K':
            multiplier = 1_000
            s = s[:-1]
        try:
            return int(float(s) * multiplier)
        except (ValueError, TypeError):
            return None

    # ── Profile ───────────────────────────────────────────────────

    async def get_profile_info(self, username: str) -> Optional[Dict]:
        client = self._get_client()
        profile = await asyncio.to_thread(client.get_user_profile, username)
        if not profile:
            return None

        return {
            'user_id': profile.get('user_id'),
            'avatar_url': profile.get('avatar_url'),
            'bio': profile.get('bio'),
            'display_name': profile.get('display_name') or username,
            'post_count': self._parse_count(profile.get('posts_count')) or 0,
            'followers_count': self._parse_count(profile.get('followers')),
            'following_count': self._parse_count(profile.get('following')),
            'is_verified': profile.get('verified', False),
            'is_private': False,
            'external_url': None,
            'external_links': None,
            'is_business': False,
            'is_professional': False,
            'category_name': None,
            'highlight_count': 0,
        }

    async def get_creator_info(self, url: str) -> Optional[Dict]:
        username = self.extract_username(url)
        if not username:
            return None

        profile = await self.get_profile_info(username)
        if profile:
            return {
                'creator_id': username,
                'creator_name': profile.get('display_name') or username,
                'creator_url': self.normalize_creator_url(username),
                'avatar_url': profile.get('avatar_url'),
                'bio': profile.get('bio'),
                'display_name': profile.get('display_name'),
                'post_count': profile.get('post_count', 0),
            }

        return {
            'creator_id': username,
            'creator_name': username,
            'creator_url': self.normalize_creator_url(username),
        }

    async def get_creator(self, url: str) -> Optional[Creator]:
        info = await self.get_creator_info(url)
        if not info:
            return None

        username = info.get('creator_id', '')
        return Creator(
            creator_id=username,
            service_id='instagram',
            platform='instagram',
            username=username,
            display_name=info.get('display_name') or info.get('creator_name'),
            profile_image_url=info.get('avatar_url'),
            bio=info.get('bio'),
            post_count=info.get('post_count', 0),
        )

    # ── Posts ─────────────────────────────────────────────────────

    async def get_posts(self, url_or_username, since_date=None,
                        progress_callback=None, known_post_ids=None,
                        post_callback=None,
                        use_authenticated_api=False,
                        paginate_all=False) -> List[Post]:
        """Fetch posts for a creator.

        Args:
            post_callback: Optional callback invoked per-post as each one is ready
                           (detail URL + tags fetched). Signature: (post: Post) -> None.
                           When provided, each post is yielded immediately so the caller
                           can upsert/queue downloads before CDN URLs expire.
            use_authenticated_api: If True, fall back to browser cookies when GraphQL
                                   returns stale data.
        """
        username = self.extract_username(url_or_username) or url_or_username

        since_ts = None
        if since_date:
            try:
                since_ts = int(datetime.fromisoformat(str(since_date)).timestamp())
            except (ValueError, TypeError):
                pass

        posts = await asyncio.to_thread(
            self._fetch_all_posts, username, since_ts, known_post_ids,
            progress_callback, post_callback, use_authenticated_api,
            paginate_all
        )

        if progress_callback:
            progress_callback(len(posts), len(posts))

        self._last_fetched_posts = posts
        self.log(f"Found {len(posts)} posts for @{username}", 'info')
        return posts

    def _fetch_all_posts(self, username: str, since_ts: Optional[int],
                         known_post_ids: Optional[Set[str]],
                         progress_callback=None,
                         post_callback=None,
                         use_authenticated_api=False,
                         paginate_all=False) -> List[Post]:
        """Fetch posts for an Instagram creator.

        When use_authenticated_api is True, uses browser cookies as the primary
        fetch method and falls back to unauthenticated GraphQL if cookies are
        expired.  When False, uses unauthenticated GraphQL only.

        When post_callback is provided, each post is emitted as soon as it's
        converted, so the caller can upsert/queue downloads immediately.
        """
        # Always create ig_client — needed for user_id resolution and GraphQL fallback
        from modules.instagram_client_module import InstagramClientDownloader
        ig_client = InstagramClientDownloader(
            show_progress=False, use_database=False,
            log_callback=self.log_callback if hasattr(self, 'log_callback') else None,
            unified_db=self.unified_db,
        )

        new_items = []

        # ── Authenticated path (primary when enabled, forced for backfill) ──
        auth_succeeded = False
        if use_authenticated_api or paginate_all:
            self.log(f"Fetching posts for @{username} via authenticated API...", 'info')
            ig_user_id = getattr(ig_client, '_user_id_cache', {}).get(username)
            if not ig_user_id:
                try:
                    ig_user_id = ig_client._fetch_user_id(username)
                except Exception:
                    pass
            if ig_user_id:
                result = self._fetch_via_browser_cookies(
                    username, ig_user_id, since_ts, known_post_ids,
                    paginate_all=paginate_all,
                    progress_callback=progress_callback)
                if result is not None:
                    # API call succeeded (even if 0 new posts)
                    auth_succeeded = True
                    new_items = result
                    if progress_callback:
                        progress_callback(len(new_items))
                else:
                    self.log(f"Authenticated API failed for @{username}, falling back to GraphQL...", 'info')

        # ── Unauthenticated GraphQL (primary when disabled, fallback when auth fails) ──
        if not auth_succeeded and not new_items:
            self.log(f"Fetching posts for @{username} via Instagram GraphQL API...", 'info')
            try:
                if since_ts:
                    ig_days_back = max(1, (datetime.now() - datetime.fromtimestamp(since_ts)).days + 1)
                else:
                    ig_days_back = 365 * 15

                def _scan_progress(count):
                    if progress_callback:
                        progress_callback(count)

                ig_posts = ig_client.fetch_posts(username, days_back=ig_days_back, max_posts=15000,
                                                 progress_callback=_scan_progress)

                # Copy pinned posts data so the scraper can call update_pinned_posts()
                if hasattr(ig_client, '_last_pinned_posts'):
                    self._last_pinned_posts = ig_client._last_pinned_posts

                pinned_codes = self._last_pinned_posts if hasattr(self, '_last_pinned_posts') else {}
                for node in ig_posts:
                    code = node.get('code', '')
                    if not code:
                        continue
                    if known_post_ids and code in known_post_ids:
                        continue
                    item = self._ig_node_to_item(node)
                    if code in pinned_codes:
                        item['isPind'] = True
                    new_items.append(item)

                self.log(f"Instagram GraphQL returned {len(ig_posts)} posts ({len(new_items)} new)", 'info')
            except Exception as e:
                self.log(f"Instagram GraphQL fetch failed: {e}", 'error')

            if progress_callback:
                progress_callback(len(new_items))

            if not new_items:
                return []

        # Phase 2: Process posts — GraphQL items already have full-res URLs
        # and tags, so they skip detail page fetching entirely.
        # Non-GraphQL items (if any) still go through ImgInn detail pages.
        import html as html_mod
        import json as json_mod
        import time
        import requests as std_requests

        total = len(new_items)
        self.log(f"Processing {total} posts...", 'info')

        posts = []
        fs_session_id = None
        ig_session = None
        ig_stopped = False
        base = 'https://imginn.com'

        # Proxy rotation state for Instagram tag fetching
        _nordvpn_creds = 'Dc9mgrpJnFnkTtc5iQkGNwLM:fKd2ZEjBUJ3YDQ5hhcoTKsnW'
        _nordvpn_servers = [
            'us.socks.nordhold.net', 'nl.socks.nordhold.net',
            'se.socks.nordhold.net', 'amsterdam.nl.socks.nordhold.net',
            'atlanta.us.socks.nordhold.net', 'chicago.us.socks.nordhold.net',
            'dallas.us.socks.nordhold.net', 'los-angeles.us.socks.nordhold.net',
            'new-york.us.socks.nordhold.net', 'stockholm.se.socks.nordhold.net',
        ]
        _server_idx = 0

        try:
            # Create FlareSolverr session for detail pages
            try:
                resp = std_requests.post('http://localhost:8191/v1', json={
                    'cmd': 'sessions.create'
                }, timeout=30)
                data = resp.json()
                if data.get('status') == 'ok':
                    fs_session_id = data.get('session')
                    self.log(f"FlareSolverr session: {fs_session_id[:8]}...", 'debug')
                else:
                    self.log("Failed to create FlareSolverr session, using API URLs", 'warning')
            except Exception as e:
                self.log(f"FlareSolverr session error: {e}", 'warning')

            # Create Instagram session for tag fetching
            try:
                from curl_cffi.requests import Session as CurlSession
                for _browser in ("chrome131", "chrome136", "chrome"):
                    try:
                        ig_session = CurlSession(impersonate=_browser)
                        break
                    except Exception:
                        continue
                else:
                    ig_session = CurlSession()
                ig_session.headers.update({
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                    'Accept-Language': 'en-US,en;q=0.9',
                    'Referer': 'https://www.instagram.com/',
                })
            except ImportError:
                self.log("curl_cffi not available for Instagram tag fetching", 'debug')

            for i, item in enumerate(new_items):
                code = item.get('code', '')
                if not code:
                    continue

                # Items from Instagram API already have full-res URLs and tags —
                # skip the slow detail page + tag fetching for these.
                if item.get('_from_ig_api'):
                    post = self._api_item_to_post(item, username)
                    if post:
                        posts.append(post)
                        if post_callback:
                            post_callback(post)
                    continue

                if progress_callback:
                    progress_callback(i + 1, total)

                # --- Fetch detail page for full-res URLs ---
                detail = None
                if fs_session_id:
                    try:
                        resp = std_requests.post('http://localhost:8191/v1', json={
                            'cmd': 'request.get',
                            'url': f'{base}/p/{code}/',
                            'session': fs_session_id,
                            'maxTimeout': 60000,
                        }, timeout=70)
                        page_data = resp.json()
                        if page_data.get('status') == 'ok':
                            html = page_data.get('solution', {}).get('response', '')
                            if html:
                                srcs = self._parse_detail_srcs(html)
                                tagged_users = self._parse_tagged_users(html)
                                if srcs:
                                    detail = {
                                        'srcs': srcs,
                                        'isSidecar': len(srcs) > 1,
                                        'tagged_users': tagged_users,
                                    }
                                    date_match = re.search(r'data-created="(\d+)"', html)
                                    if date_match:
                                        detail['date'] = int(date_match.group(1))
                                    cap_match = re.search(
                                        r'<meta\s+property="og:description"\s+content="([^"]*)"', html)
                                    if cap_match:
                                        detail['alt'] = html_mod.unescape(cap_match.group(1))
                    except Exception as e:
                        self.log(f"Detail fetch failed for {code}: {e}", 'debug')

                # --- Merge detail data ---
                if detail:
                    merge = {'tagged_users': detail.get('tagged_users', [])}
                    if 'srcs' in detail:
                        merge['srcs'] = detail['srcs']
                        merge['isSidecar'] = detail.get('isSidecar', False)
                    if not item.get('date') and detail.get('date'):
                        merge['date'] = detail['date']
                    if not item.get('alt') and detail.get('alt'):
                        merge['alt'] = detail['alt']
                    item = {**item, **merge}

                # --- Fetch tags from Instagram if detail page had none ---
                if not item.get('tagged_users') and ig_session and not ig_stopped:
                    try:
                        time.sleep(2)
                        server = _nordvpn_servers[_server_idx % len(_nordvpn_servers)]
                        proxy = {'https': f'socks5://{_nordvpn_creds}@{server}:1080'}

                        ig_resp = None
                        for attempt in range(3):
                            try:
                                ig_resp = ig_session.get(
                                    f'https://www.instagram.com/p/{code}/',
                                    timeout=20, proxies=proxy)
                                break
                            except Exception:
                                _server_idx += 1
                                server = _nordvpn_servers[_server_idx % len(_nordvpn_servers)]
                                proxy = {'https': f'socks5://{_nordvpn_creds}@{server}:1080'}
                                if attempt == 2:
                                    raise

                        if ig_resp and ig_resp.status_code == 429:
                            self.log("Instagram rate limited, stopping tag fallback", 'warning')
                            ig_stopped = True
                            _server_idx += 1
                        elif ig_resp and ig_resp.status_code == 200:
                            ig_html = ig_resp.text
                            idx_pos = ig_html.find('xdt_api__v1__media__shortcode__web_info')
                            if idx_pos >= 0:
                                items_start = ig_html.find('{"items":[{', idx_pos)
                                if items_start >= 0:
                                    depth = 0
                                    end = items_start
                                    for ci in range(items_start, min(items_start + 500000, len(ig_html))):
                                        if ig_html[ci] == '{':
                                            depth += 1
                                        elif ig_html[ci] == '}':
                                            depth -= 1
                                            if depth == 0:
                                                end = ci + 1
                                                break
                                    ig_data = json_mod.loads(ig_html[items_start:end])
                                    ig_items = ig_data.get('items', [])
                                    if ig_items:
                                        tagged = []
                                        ig_item = ig_items[0]
                                        for tag in (ig_item.get('usertags') or {}).get('in', []):
                                            uname = (tag.get('user') or {}).get('username')
                                            if uname and uname not in tagged:
                                                tagged.append(uname)
                                        for cm in ig_item.get('carousel_media') or []:
                                            for tag in (cm.get('usertags') or {}).get('in', []):
                                                uname = (tag.get('user') or {}).get('username')
                                                if uname and uname not in tagged:
                                                    tagged.append(uname)
                                        if tagged:
                                            item = {**item, 'tagged_users': tagged}
                                            self.log(f"IG tags: {code} -> {tagged}", 'debug')
                    except Exception as e:
                        self.log(f"IG tag fetch failed for {code}: {e}", 'debug')

                # --- Build Post and emit via callback ---
                post = self._api_item_to_post(item, username)
                if post:
                    posts.append(post)
                    if post_callback:
                        post_callback(post)

        finally:
            if fs_session_id:
                try:
                    std_requests.post('http://localhost:8191/v1', json={
                        'cmd': 'sessions.destroy',
                        'session': fs_session_id,
                    }, timeout=10)
                except Exception:
                    pass

        self.log(f"Processed {len(posts)}/{total} posts", 'info')
        return posts

    def _fetch_via_browser_cookies(self, username: str, user_id: str,
                                    since_ts, known_post_ids,
                                    paginate_all=False,
                                    progress_callback=None) -> Optional[List[Dict]]:
        """Fetch timeline via authenticated browser cookies.

        Loads cookies from the 'instagram_browser' scraper entry in the DB,
        makes /api/v1/feed/user/ calls, and writes updated cookies back to
        the DB.

        When paginate_all is False (default), makes a single API call to
        minimise fingerprint.  When True, paginates the full timeline using
        max_id cursor to find all missing posts (backfill mode).

        Returns None on failure (no cookies, expired, error) so the caller
        can distinguish from an empty list (API worked, nothing new).
        """
        import json as _json
        import time as _time

        if not self.unified_db:
            return None

        # Load cookies from DB
        try:
            with self.unified_db.get_connection() as conn:
                cursor = conn.cursor()
                cursor.execute(
                    "SELECT cookies_json FROM scrapers WHERE id = 'instagram_browser'"
                )
                row = cursor.fetchone()
                if not row or not row[0]:
                    return None
                cookie_data = _json.loads(row[0])
                # Handle both formats: flat array or {"cookies": [...]}
                if isinstance(cookie_data, dict):
                    cookie_list = cookie_data.get('cookies', [])
                elif isinstance(cookie_data, list):
                    cookie_list = cookie_data
                else:
                    return None
                if not cookie_list:
                    return None
                # Must have sessionid
                has_session = any(c.get('name') == 'sessionid' and c.get('value')
                                  for c in cookie_list)
                if not has_session:
                    return None
        except Exception as e:
            self.log(f"Authenticated API: failed to load cookies: {e}", 'debug')
            return None

        self.log(f"Authenticated API: fetching feed for @{username}{'  (full backfill)' if paginate_all else ''}...", 'info')

        try:
            import random as _random
            from curl_cffi.requests import Session as CurlSession
            session = CurlSession(impersonate='chrome136')

            # Load cookies first so we can extract csrftoken for the header
            csrf_token = ''
            for c in cookie_list:
                name = c.get('name', '')
                value = c.get('value', '')
                domain = c.get('domain', '.instagram.com')
                if name and value:
                    session.cookies.set(name, value, domain=domain)
                    if name == 'csrftoken':
                        csrf_token = value

            # Extract ig_www_claim from cookies if present (set by IG in responses)
            ig_claim = '0'
            for c in cookie_list:
                if c.get('name') == 'ig_www_claim':
                    ig_claim = c.get('value', '0')
                    break

            # Override UA/platform headers to match Edge on macOS (where
            # the cookies were created).  The chrome136 TLS fingerprint is
            # identical to Edge 136 since both use Chromium's BoringSSL.
            session.headers.update({
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0',
                'Sec-CH-UA': '"Microsoft Edge";v="136", "Chromium";v="136", "Not_A Brand";v="24"',
                'Sec-CH-UA-Mobile': '?0',
                'Sec-CH-UA-Platform': '"macOS"',
                'X-CSRFToken': csrf_token,
                'X-IG-App-ID': '936619743392459',
                'X-IG-WWW-Claim': ig_claim,
                'X-ASBD-ID': '129477',
                'X-Requested-With': 'XMLHttpRequest',
                'Referer': f'https://www.instagram.com/{username}/',
                'Origin': 'https://www.instagram.com',
                'Accept': '*/*',
                'Accept-Language': 'en-US,en;q=0.9',
                'Sec-Fetch-Dest': 'empty',
                'Sec-Fetch-Mode': 'cors',
                'Sec-Fetch-Site': 'same-origin',
            })

            def _save_cookies():
                try:
                    updated_cookies = []
                    for c in session.cookies:
                        updated_cookies.append({
                            'name': c.name,
                            'value': c.value,
                            'domain': c.domain or '.instagram.com',
                        })
                    if updated_cookies:
                        with self.unified_db.get_connection() as conn:
                            cursor = conn.cursor()
                            cursor.execute(
                                "UPDATE scrapers SET cookies_json = ?, cookies_updated_at = ? WHERE id = 'instagram_browser'",
                                (_json.dumps(updated_cookies), datetime.now().isoformat())
                            )
                            conn.commit()
                except Exception as e:
                    self.log(f"Authenticated API: failed to save cookies: {e}", 'debug')

            def _update_claim_header(resp):
                """Update X-IG-WWW-Claim from response headers (IG rotates this)."""
                claim = resp.headers.get('x-ig-set-www-claim')
                if claim:
                    session.headers['X-IG-WWW-Claim'] = claim

            # ── Single-page mode (normal sync) ──
            if not paginate_all:
                # Randomized pre-request delay to avoid machine-like timing
                _time.sleep(_random.uniform(1.0, 3.0))
                resp = session.get(
                    f'https://www.instagram.com/api/v1/feed/user/{user_id}/',
                    params={'count': 12},
                    timeout=15
                )
                _update_claim_header(resp)
                _save_cookies()

                if resp.status_code == 401:
                    self.log(f"Authenticated API: 401 Unauthorized — cookies expired for @{username}", 'warning')
                    self._notify_cookie_expired()
                    return None

                if resp.status_code != 200:
                    self.log(f"Authenticated API: HTTP {resp.status_code}", 'warning')
                    return None

                data = resp.json()
                nodes = data.get('items', [])
                self._notify_cookie_ok()
                if not nodes:
                    return []

                self.log(f"Authenticated API: got {len(nodes)} posts for @{username}", 'info')

                cutoff = datetime.fromtimestamp(since_ts) if since_ts else None
                new_items = []
                consecutive_old = 0
                for node in nodes:
                    code = node.get('code', '')
                    if not code:
                        continue
                    if known_post_ids and code in known_post_ids:
                        continue
                    taken_at = node.get('taken_at')
                    if not taken_at:
                        continue
                    post_date = datetime.fromtimestamp(taken_at)
                    if cutoff and post_date < cutoff:
                        consecutive_old += 1
                        if consecutive_old >= 5:
                            break
                        continue
                    else:
                        consecutive_old = 0
                    item = self._ig_node_to_item(node)
                    item['_from_ig_api'] = True
                    new_items.append(item)

                self.log(f"Authenticated API: {len(new_items)} new posts for @{username}", 'info')
                return new_items

            # ── Full pagination mode (backfill) ──
            max_id = None
            total_fetched = 0
            total_new = 0
            page = 0
            consecutive_errors = 0
            consecutive_known_pages = 0
            all_new_items = []
            # Mutable set to track known IDs including newly discovered ones
            seen_ids = set(known_post_ids) if known_post_ids else set()

            while True:
                page += 1
                params = {'count': 12}
                if max_id:
                    params['max_id'] = max_id

                # Human-like delay between pages (randomized)
                if page > 1:
                    _time.sleep(_random.uniform(2.0, 5.0))
                else:
                    _time.sleep(_random.uniform(1.0, 3.0))

                try:
                    resp = session.get(
                        f'https://www.instagram.com/api/v1/feed/user/{user_id}/',
                        params=params,
                        timeout=15
                    )
                    _update_claim_header(resp)
                except Exception as e:
                    self.log(f"Backfill page {page}: request error: {e}", 'warning')
                    consecutive_errors += 1
                    if consecutive_errors >= 3:
                        self.log("Backfill: too many consecutive errors, stopping.", 'warning')
                        break
                    _time.sleep(_random.uniform(5.0, 10.0))
                    continue

                if resp.status_code == 401:
                    self.log(f"Backfill: 401 Unauthorized — cookies expired for @{username}", 'warning')
                    self._notify_cookie_expired()
                    _save_cookies()
                    break

                if resp.status_code == 429:
                    self.log("Backfill: rate limited, waiting 60-120s...", 'warning')
                    _time.sleep(_random.uniform(60, 120))
                    continue

                if resp.status_code != 200:
                    self.log(f"Backfill page {page}: HTTP {resp.status_code}", 'warning')
                    consecutive_errors += 1
                    if consecutive_errors >= 3:
                        self.log("Backfill: too many consecutive errors, stopping.", 'warning')
                        break
                    _time.sleep(_random.uniform(5.0, 10.0))
                    continue

                consecutive_errors = 0
                if page == 1:
                    self._notify_cookie_ok()
                data = resp.json()
                items = data.get('items', [])
                more = data.get('more_available', False)
                next_max_id = data.get('next_max_id')

                if not items:
                    self.log(f"Backfill page {page}: no items returned, done.", 'info')
                    break

                total_fetched += len(items)
                page_new = 0

                for node in items:
                    code = node.get('code', '')
                    if not code:
                        continue
                    if code in seen_ids:
                        continue
                    seen_ids.add(code)
                    item = self._ig_node_to_item(node)
                    item['_from_ig_api'] = True
                    all_new_items.append(item)
                    page_new += 1
                    total_new += 1

                self.log(f"Backfill page {page}: {len(items)} items, {page_new} new (total: {total_fetched} fetched, {total_new} new)", 'info')

                if progress_callback:
                    progress_callback(total_fetched, total_new)

                # Early stop: if 3 consecutive pages had zero new posts, we've
                # reached content we already have — no need to scan further.
                if page_new == 0 and known_post_ids:
                    consecutive_known_pages += 1
                    if consecutive_known_pages >= 3:
                        self.log("Backfill: 3 consecutive pages with no new posts, stopping early.", 'info')
                        break
                else:
                    consecutive_known_pages = 0

                if not more or not next_max_id:
                    self.log("Backfill: no more pages available.", 'info')
                    break

                max_id = next_max_id

            _save_cookies()
            self.log(f"Backfill complete: {total_fetched} fetched, {total_new} new posts for @{username}", 'info')
            return all_new_items

        except Exception as e:
            self.log(f"Authenticated API failed: {e}", 'warning')
            return None

    def _notify_cookie_ok(self):
        """Mark instagram_browser cookies as OK after a successful API call."""
        try:
            if self.unified_db:
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute(
                        "UPDATE scrapers SET last_test_status = 'ok', "
                        "last_test_message = 'Authenticated API working' "
                        "WHERE id = 'instagram_browser'"
                    )
                    conn.commit()
        except Exception as e:
            self.log(f"Failed to update scraper status: {e}", 'debug')

    def _notify_cookie_expired(self):
        """Mark instagram_browser cookies as failed and broadcast a health alert."""
        import json as _json
        try:
            if self.unified_db:
                with self.unified_db.get_connection() as conn:
                    cursor = conn.cursor()
                    cursor.execute(
                        "UPDATE scrapers SET last_test_status = 'failed', "
                        "last_test_message = 'Cookies expired (401 from Instagram API)' "
                        "WHERE id = 'instagram_browser'"
                    )
                    conn.commit()
        except Exception as e:
            self.log(f"Failed to update scraper status: {e}", 'debug')

        try:
            from web.backend.core.dependencies import get_app_state
            from web.backend.routers.health import _broadcast_cookie_alert
            app_state = get_app_state()
            _broadcast_cookie_alert(
                app_state,
                "scraper:instagram_browser",
                "Instagram (Authenticated)",
                "expired",
            )
        except Exception as e:
            self.log(f"Failed to broadcast cookie alert: {e}", 'debug')

    @staticmethod
    def _parse_tagged_users(html: str) -> List[str]:
        """Extract tagged Instagram usernames from a post detail page HTML."""
        tagged = []
        idx = html.find('class="tagged-user-list"')
        if idx < 0:
            return tagged
        chunk = html[idx:idx + 5000]
        for m in re.finditer(r'class="name">\s*(\S+)\s*</div>', chunk):
            username = m.group(1).strip()
            if re.match(r'^[a-zA-Z0-9_.]{1,30}$', username):
                tagged.append(username)
        return tagged

    @staticmethod
    def _parse_detail_srcs(html: str) -> List[str]:
        """Extract full-res CDN URLs from a post detail page HTML."""
        import html as html_mod

        # Swiper slide data-src (carousel posts)
        slide_pattern = re.compile(
            r'class="swiper-slide[^"]*"[^>]*data-src="([^"]+)"', re.DOTALL)
        slide_srcs = slide_pattern.findall(html)

        # Fallback: download button links
        if not slide_srcs:
            dl_pattern = re.compile(
                r'class="[^"]*downloads[^"]*"[^>]*href="(https://scontent[^"]+)"', re.DOTALL)
            slide_srcs = dl_pattern.findall(html)
            if not slide_srcs:
                slide_srcs = re.findall(r'href="(https://scontent[^"]*dl=1[^"]*)"', html)

        # Clean and deduplicate
        urls = []
        seen = set()
        for src in slide_srcs:
            src = html_mod.unescape(src)
            if 'scontent' not in src and 'cdninstagram' not in src:
                continue
            base = src.split('?')[0]
            if base in seen:
                continue
            seen.add(base)
            urls.append(src)

        return urls

    # ── Stories ───────────────────────────────────────────────────

    async def get_stories(self, url_or_username, progress_callback=None) -> List[Post]:
        username = self.extract_username(url_or_username) or url_or_username

        posts = await asyncio.to_thread(self._fetch_stories, username)

        if progress_callback:
            progress_callback(len(posts), len(posts))

        self.log(f"Found {len(posts)} stories for @{username}", 'info')
        return posts

    def _fetch_stories(self, username: str) -> List[Post]:
        """Fetch stories via ImgInn HTML page (sync, runs in thread).

        The API endpoint (/api/story/) often returns stale/cached results,
        while the HTML page (/stories/{username}/) has the current stories.
        """
        client = self._get_client()

        # Prefer HTML parsing — the API often returns stale data
        items = client._parse_stories_from_html(username)

        if not items:
            # Fall back to API
            params = client._get_stories_params(username)
            if params:
                data = client._call_api('/api/story/', params)
                if data and data.get('items'):
                    items = data['items']

        if not items:
            return []

        posts = []
        for idx, item in enumerate(items):
            post = self._story_item_to_post(item, username, idx)
            if post:
                posts.append(post)

        return posts

    # ── Highlights ────────────────────────────────────────────────

    async def get_highlights(self, url_or_username, known_post_ids=None,
                             progress_callback=None) -> List[Post]:
        username = self.extract_username(url_or_username) or url_or_username
        posts = await asyncio.to_thread(
            self._fetch_highlights, username, known_post_ids, progress_callback
        )
        self.log(f"Found {len(posts)} highlights for @{username}", 'info')
        return posts

    def _fetch_highlights(self, username: str, known_post_ids=None,
                          progress_callback=None) -> List[Post]:
        """Fetch highlights via ImgInn (sync, runs in thread).

        Each highlight group becomes a single Post with multiple attachments,
        similar to a carousel post.
        """
        client = self._get_client()

        # Step 1: Discover highlights from stories page
        highlights = client._parse_highlights_from_html(username)
        if not highlights:
            return []

        self.log(f"Found {len(highlights)} highlights for @{username}", 'info')

        # Step 2: Fetch items for each highlight via /api/media/?id=
        posts = []
        for hi_idx, highlight in enumerate(highlights):
            highlight_id = highlight['id']
            post_id = f"highlight_{highlight_id}"
            title = highlight['title'] or f'Highlight {hi_idx + 1}'

            # Skip already-known highlights
            if known_post_ids and post_id in known_post_ids:
                if progress_callback:
                    progress_callback(hi_idx + 1, len(highlights))
                continue

            data = client._call_api('/api/media/', {'id': highlight_id})
            if not data or not data.get('items'):
                if progress_callback:
                    progress_callback(hi_idx + 1, len(highlights))
                continue

            # Build attachments from all items in this highlight
            attachments = []
            earliest_time = None
            for item_idx, item in enumerate(data['items']):
                att = self._highlight_item_to_attachment(
                    item, highlight_id, item_idx
                )
                if att:
                    attachments.append(att)
                item_time = item.get('time')
                if item_time and (earliest_time is None or item_time < earliest_time):
                    earliest_time = item_time

            if attachments:
                published_at = (datetime.fromtimestamp(earliest_time).isoformat()
                                if earliest_time else None)
                posts.append(Post(
                    post_id=post_id,
                    service_id='instagram',
                    platform='instagram',
                    creator_id=username,
                    title=title,
                    content='',
                    published_at=published_at,
                    attachments=attachments,
                ))

            if progress_callback:
                progress_callback(hi_idx + 1, len(highlights))

        return posts

    def _highlight_item_to_attachment(self, item: Dict, highlight_id: str,
                                      item_idx: int) -> Optional[Attachment]:
        """Convert a single ImgInn highlight item to an Attachment."""
        src = item.get('src', '')
        proxy = item.get('proxy', '')
        download_url = src or proxy
        if not download_url:
            return None

        cdn_path = download_url.split('?')[0]
        ext = '.mp4' if '.mp4' in cdn_path else '.jpg'
        file_type = 'video' if ext == '.mp4' else 'image'

        return Attachment(
            name=f"highlight_{highlight_id}_{item_idx}{ext}",
            file_type=file_type,
            extension=ext,
            server_path=f"https://www.instagram.com/stories/highlights/{highlight_id}/{item_idx}",
            download_url=download_url,
        )

    async def refresh_failed_highlight_urls(self, failed_attachments: List[Dict]) -> Dict[int, str]:
        """Re-fetch highlight API to get fresh CDN URLs for failed attachments.

        Args:
            failed_attachments: List of attachment dicts with 'id', 'name', 'download_url'
                                where name matches pattern highlight_{id}_{idx}.ext

        Returns:
            Dict mapping attachment_id -> new_download_url for URLs that changed
        """
        return await asyncio.to_thread(
            self._refresh_failed_highlight_urls_sync, failed_attachments
        )

    def _refresh_failed_highlight_urls_sync(self, failed_attachments: List[Dict]) -> Dict[int, str]:
        """Sync implementation of highlight URL refresh."""
        import re as _re
        import time

        # Group by highlight_id
        highlights: Dict[str, List[Dict]] = {}
        for att in failed_attachments:
            m = _re.match(r'highlight_(\d+)_(\d+)\.\w+', att.get('name', ''))
            if m:
                h_id = m.group(1)
                highlights.setdefault(h_id, []).append({
                    'att_id': att['id'],
                    'item_idx': int(m.group(2)),
                    'old_url': att.get('download_url', ''),
                })

        if not highlights:
            return {}

        client = self._get_client()
        updated = {}
        now = time.time()

        for h_id, atts in highlights.items():
            data = client._call_api('/api/media/', {'id': h_id})
            if not data or not data.get('items'):
                continue

            items = data['items']
            for att_info in atts:
                idx = att_info['item_idx']
                if idx >= len(items):
                    continue

                item = items[idx]
                new_url = item.get('src', '') or item.get('proxy', '')
                if not new_url:
                    continue

                # Only update if URL actually changed
                if new_url != att_info['old_url']:
                    # Check if the new URL is fresh
                    oe_match = _re.search(r'[&?]oe=([0-9A-Fa-f]+)', new_url)
                    if oe_match:
                        try:
                            expiry = int(oe_match.group(1), 16)
                            if expiry < now:
                                continue  # New URL is also expired, skip
                        except (ValueError, OverflowError):
                            pass
                    updated[att_info['att_id']] = new_url

        if updated:
            self.log(f"Refreshed {len(updated)} highlight URLs with fresh CDN links", 'info')

        return updated

    # ── Reels ─────────────────────────────────────────────────────

    async def get_reels(self, url_or_username, since_date=None,
                        max_reels=50, progress_callback=None) -> List[Post]:
        username = self.extract_username(url_or_username) or url_or_username

        # Reuse posts from get_posts() if available (avoid duplicate browser session)
        if self._last_fetched_posts:
            all_posts = self._last_fetched_posts
            self.log(f"Reusing {len(all_posts)} cached posts for reels filter", 'debug')
        else:
            since_ts = None
            if since_date:
                try:
                    since_ts = int(datetime.fromisoformat(str(since_date)).timestamp())
                except (ValueError, TypeError):
                    pass
            all_posts = await asyncio.to_thread(
                self._fetch_all_posts, username, since_ts, None
            )

        reels = []
        for post in all_posts:
            if any(a.file_type == 'video' for a in post.attachments):
                reels.append(post)
                if len(reels) >= max_reels:
                    break

        if progress_callback:
            progress_callback(len(reels), len(reels))

        self.log(f"Found {len(reels)} reels for @{username}", 'info')
        return reels

    # ── Download ──────────────────────────────────────────────────

    async def download_media(self, download_url: str, output_path) -> Dict:
        try:
            output_path = Path(output_path)
            output_path.parent.mkdir(parents=True, exist_ok=True)
            result = await asyncio.to_thread(
                self._download_media_sync, download_url, output_path
            )
            return result
        except Exception as e:
            self.log(f"Error downloading media: {e}", 'error')
            return {'success': False, 'error': str(e)}

    def _download_media_sync(self, download_url: str, output_path: Path) -> Dict:
        client = self._get_client()
        try:
            # Download directly via session — skip ImgInn rate limiter since
            # CDN downloads go to Instagram's servers, not ImgInn
            resp = client.session.get(
                download_url,
                headers={'Referer': f'{client.IMGINN_BASE}/'},
                timeout=120,
            )
            if resp.status_code != 200:
                return {'success': False, 'error': f'HTTP {resp.status_code}'}

            output_path.parent.mkdir(parents=True, exist_ok=True)
            with open(output_path, 'wb') as f:
                f.write(resp.content)

            file_size = output_path.stat().st_size
            if file_size < 1000:
                output_path.unlink()
                return {'success': False, 'error': f'File too small ({file_size}B)'}

            return {
                'success': True,
                'file_path': str(output_path),
                'filename': output_path.name,
                'file_size': file_size,
            }
        except Exception as e:
            return {'success': False, 'error': str(e)}

    def _get_session(self):
        """Expose session for avatar downloads (used by scraper)."""
        return self._get_client().session

    # ── Conversion helpers ────────────────────────────────────────

    def _api_item_to_post(self, item: Dict, username: str) -> Optional[Post]:
        """Convert an ImgInn API post item to a Post with Attachments.

        API item format: {code, date, alt, srcs[], isSidecar, isPind}
        """
        code = item.get('code', '')
        if not code:
            return None

        timestamp = item.get('date', 0)
        published_at = datetime.fromtimestamp(timestamp).isoformat() if timestamp else None

        caption = item.get('alt', '') or ''

        srcs = item.get('srcs', [])
        if not srcs:
            src = item.get('src', '')
            if src:
                srcs = [src]

        if not srcs:
            return None

        # Deduplicate by base URL (without query params)
        seen_bases = set()
        unique_srcs = []
        for src_url in srcs:
            base = src_url.split('?')[0]
            if base not in seen_bases:
                seen_bases.add(base)
                unique_srcs.append(src_url)
        srcs = unique_srcs

        attachments = []
        for idx, src_url in enumerate(srcs):
            ext = '.mp4' if '.mp4' in src_url.split('?')[0] else '.jpg'
            file_type = 'video' if ext == '.mp4' else 'image'

            attachment = Attachment(
                name=f"{code}_{idx}{ext}",
                file_type=file_type,
                extension=ext,
                server_path=f"https://www.instagram.com/p/{code}/?img_index={idx + 1}",
                download_url=src_url,
            )
            attachments.append(attachment)

        if not attachments:
            return None

        return Post(
            post_id=code,
            service_id='instagram',
            platform='instagram',
            creator_id=username,
            title=None,
            content=caption,
            published_at=published_at,
            attachments=attachments,
            is_pinned=item.get('isPind', False),
            tagged_users=item.get('tagged_users', []),
        )

    def _ig_node_to_item(self, node: Dict) -> Dict:
        """Convert an Instagram v1 API node to an ImgInn-style item dict.

        Used when falling back to Instagram GraphQL API after ImgInn pagination
        exhausts. The returned dict matches the format expected by _api_item_to_post.
        """
        code = node.get('code', '')
        taken_at = node.get('taken_at', 0)

        caption_obj = node.get('caption')
        caption = caption_obj.get('text', '') if isinstance(caption_obj, dict) else ''

        srcs = []
        media_type = node.get('media_type', 1)

        if media_type == 8 and node.get('carousel_media'):
            for child in node['carousel_media']:
                url = self._best_media_url(child)
                if url:
                    srcs.append(url)
        else:
            url = self._best_media_url(node)
            if url:
                srcs.append(url)

        tagged_users = []
        for tag in (node.get('usertags') or {}).get('in', []):
            uname = (tag.get('user') or {}).get('username')
            if uname and uname not in tagged_users:
                tagged_users.append(uname)
        for cm in node.get('carousel_media') or []:
            for tag in (cm.get('usertags') or {}).get('in', []):
                uname = (tag.get('user') or {}).get('username')
                if uname and uname not in tagged_users:
                    tagged_users.append(uname)

        return {
            'code': code,
            'date': taken_at,
            'alt': caption,
            'srcs': srcs,
            'isSidecar': len(srcs) > 1,
            'isPind': False,
            'tagged_users': tagged_users,
            '_from_ig_api': True,
        }

    @staticmethod
    def _best_media_url(node: Dict) -> Optional[str]:
        """Get the best quality URL from a v1 media node."""
        media_type = node.get('media_type', 1)
        if media_type == 2 and node.get('video_versions'):
            best = max(node['video_versions'], key=lambda v: v.get('width', 0) * v.get('height', 0))
            return best.get('url', '')
        candidates = node.get('image_versions2', {}).get('candidates', [])
        if candidates:
            best = max(candidates, key=lambda c: c.get('width', 0) * c.get('height', 0))
            return best.get('url', '')
        return None

    def _story_item_to_post(self, item: Dict, username: str, idx: int) -> Optional[Post]:
        """Convert an ImgInn story item to a Post.

        Story item format: {src, time} or {src, proxy}
        """
        src = item.get('src', '')
        if not src:
            src = item.get('proxy', '')
        if not src:
            return None

        story_time = item.get('time')
        published_at = datetime.fromtimestamp(story_time).isoformat() if story_time else None

        # Derive stable story_id from CDN filename
        cdn_path = src.split('?')[0]
        cdn_filename = cdn_path.split('/')[-1]
        story_id = cdn_filename.rsplit('.', 1)[0] if '.' in cdn_filename else cdn_filename
        if not story_id:
            story_id = f"story_{username}_{idx}"

        ext = '.mp4' if '.mp4' in cdn_path else '.jpg'
        file_type = 'video' if ext == '.mp4' else 'image'

        attachment = Attachment(
            name=f"story_{story_id}{ext}",
            file_type=file_type,
            extension=ext,
            server_path=f"https://www.instagram.com/stories/{username}/",
            download_url=src,
        )

        return Post(
            post_id=f'story_{story_id}',
            service_id='instagram',
            platform='instagram',
            creator_id=username,
            title=f'Story by @{username}',
            content='',
            published_at=published_at,
            attachments=[attachment],
        )
