#!/usr/bin/env python3 """ ImgInn downloader module with FastDL-compatible file naming Format: {profile}_{YYYYMMDD_HHMMSS}_{media_id}{ext} """ import os import json import time import random import re import subprocess import platform import requests from pathlib import Path from datetime import datetime, timedelta from modules.base_module import LoggingMixin from modules.cloudflare_handler import ( CloudflareHandler, SiteStatus, get_flaresolverr_user_agent, get_playwright_context_options, get_playwright_stealth_scripts ) from modules.instagram_utils import ( extract_instagram_media_id, scan_existing_files_for_media_ids, record_instagram_download, is_instagram_downloaded ) from typing import Dict, Optional from playwright.sync_api import sync_playwright class ImgInnDownloader(LoggingMixin): """ImgInn downloader with FastDL-compatible naming""" def __init__(self, headless: bool = True, cookie_file: str = "/opt/media-downloader/cookies/imginn_cookies.json", show_progress: bool = True, use_database: bool = True, log_callback=None, unified_db=None, ): """Initialize downloader compatible with media-downloader system""" # Initialize logging via mixin self._init_logger('Instagram', log_callback, default_module='Download') self.headless = headless self.downloaded_files = set() # Track downloaded media IDs self.show_progress = show_progress self.use_database = use_database self.download_count = 0 self.unified_db = unified_db # Store for scraper config access self.scraper_id = 'imginn' # Scraper ID in database self.pending_downloads = [] # Track downloads for deferred database recording # Rate limiting - track last scrape time to avoid hitting Cloudflare self._last_scrape_time = None self._min_scrape_interval = 15 # Minimum seconds between scrape types # Track transient page load failures per session self._page_load_failures = 0 self._page_load_failure_threshold = 5 # Escalate to error after this many # Browser reuse across profiles self.playwright = None self.browser = None self.context = None self.page = None # Use unified database directly (no adapter needed) if unified_db and use_database: self.unified_db = unified_db else: self.unified_db = None self.use_database = False # Initialize activity status manager for real-time updates from modules.activity_status import get_activity_manager self.activity_manager = get_activity_manager(unified_db) # Load scraper configuration from database if available self.proxy_url = None self.cookie_file = None # Default to None (use database) if unified_db: scraper_config = unified_db.get_scraper(self.scraper_id) if scraper_config: # Get proxy configuration if scraper_config.get('proxy_enabled') and scraper_config.get('proxy_url'): self.proxy_url = scraper_config['proxy_url'] self.log(f"Using proxy: {self.proxy_url}", "info") # Fall back to cookie file if no database if not unified_db: self.cookie_file = Path(cookie_file) self.cookie_file.parent.mkdir(parents=True, exist_ok=True) # User-Agent to match FlareSolverr (dynamically fetched for consistency) self.user_agent = get_flaresolverr_user_agent() # Initialize universal Cloudflare handler # Pass proxy_url if configured, and cookie_file=None for database storage self.cf_handler = CloudflareHandler( module_name="ImgInn", cookie_file=str(self.cookie_file) if self.cookie_file else None, user_agent=self.user_agent, logger=self.logger, aggressive_expiry=True, # Refresh cookies expiring within 7 days proxy_url=self.proxy_url # Pass proxy to FlareSolverr ) # Keep for backwards compatibility self.flaresolverr_url = self.cf_handler.flaresolverr_url self.flaresolverr_enabled = self.cf_handler.flaresolverr_enabled # Load cookies from database if available self._load_cookies_from_db() def _load_cookies_from_db(self): """Load cookies from database if available""" if not self.unified_db: return try: cookies = self.unified_db.get_scraper_cookies(self.scraper_id) if cookies: # Load into CloudflareHandler self.cf_handler._cookies = cookies self.log(f"Loaded {len(cookies)} cookies from database", "debug") except Exception as e: self.log(f"Error loading cookies from database: {e}", "warning") def _save_cookies_to_db(self, cookies: list, user_agent: str = None): """Save cookies to database Args: cookies: List of cookie dictionaries user_agent: User agent to associate with cookies (important for cf_clearance). If not provided, uses self.user_agent as fallback. """ if not self.unified_db: return try: # Use provided user_agent or fall back to self.user_agent ua = user_agent or self.user_agent self.unified_db.save_scraper_cookies( self.scraper_id, cookies, user_agent=ua, merge=True ) self.log(f"Saved {len(cookies)} cookies to database (UA: {ua[:50]}...)", "debug") except Exception as e: self.log(f"Error saving cookies to database: {e}", "warning") def _cookies_expired(self): """Check if cookies are expired - delegates to CloudflareHandler""" return self.cf_handler.cookies_expired() def _get_cookies_for_requests(self): """Get cookies in format for requests library - delegates to CloudflareHandler""" return self.cf_handler.get_cookies_dict() def _get_cookies_via_flaresolverr(self, url="https://imginn.com/", max_retries=2): """Use FlareSolverr to bypass Cloudflare - delegates to CloudflareHandler Args: url: URL to fetch max_retries: Maximum number of retry attempts (default: 2) Returns: True if cookies obtained successfully, False otherwise """ success = self.cf_handler.get_cookies_via_flaresolverr(url, max_retries) # Save cookies to database if successful if success and self.unified_db: cookies_list = self.cf_handler.get_cookies_list() if cookies_list: # CRITICAL: Get the user_agent from FlareSolverr solution, not self.user_agent # cf_clearance cookies are fingerprinted to the browser that solved the challenge flaresolverr_ua = self.cf_handler.get_user_agent() self._save_cookies_to_db(cookies_list, user_agent=flaresolverr_ua) return success def _enforce_rate_limit(self, scrape_type: str = "scrape"): """Enforce rate limiting between scrape operations to avoid Cloudflare blocks. Args: scrape_type: Type of scrape (posts, stories, tagged) for logging """ import random if self._last_scrape_time is not None: elapsed = time.time() - self._last_scrape_time if elapsed < self._min_scrape_interval: # Add random jitter (5-15 seconds) to the delay jitter = random.uniform(5, 15) wait_time = self._min_scrape_interval - elapsed + jitter self.log(f"Rate limiting: waiting {wait_time:.1f}s before {scrape_type} (Cloudflare avoidance)", "info") time.sleep(wait_time) self._last_scrape_time = time.time() def _has_valid_cookies(self): """Check if we have valid cookies (either in file or database)""" if self.unified_db: cookies = self.unified_db.get_scraper_cookies(self.scraper_id) return cookies and len(cookies) > 0 elif self.cookie_file: return self.cookie_file.exists() return False def _start_browser(self): """Start browser if not already running (reusable across profiles)""" # Try to get fresh cookies via FlareSolverr if we don't have them or they're old # Do this BEFORE the browser reuse check so cookies are always checked if not self._has_valid_cookies() or self._cookies_expired(): self.log("Cookies missing or expired, attempting FlareSolverr bypass...", "info") if self._get_cookies_via_flaresolverr(): self.log("Successfully got fresh cookies from FlareSolverr", "info") else: self.log("FlareSolverr unavailable, will try with Playwright", "warning") if self.browser is not None: self.log("Browser already running, reusing...", "debug") return import os # Use environment variable if set, otherwise use standard location if 'PLAYWRIGHT_BROWSERS_PATH' not in os.environ: os.environ['PLAYWRIGHT_BROWSERS_PATH'] = '/root/.cache/ms-playwright' os.environ['DISPLAY'] = ':100' # Use Xvfb virtual display self.log("Starting browser (Chromium)...", "info") self.playwright = sync_playwright().start() self.browser = self.playwright.chromium.launch( headless=self.headless, args=[ '--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding' ] ) # CRITICAL: Browser fingerprint must match FlareSolverr for cookies to work # Get dynamic fingerprint settings from FlareSolverr context_options = get_playwright_context_options() # IMPORTANT: If cookies have a stored user_agent, use THAT user_agent # Cloudflare cf_clearance cookies are fingerprinted to the browser that solved the challenge try: stored_user_agent = self.unified_db.get_scraper_cookies_user_agent(self.scraper_id) if stored_user_agent: self.log(f"Using stored cookie user_agent: {stored_user_agent[:50]}...", "debug") context_options['user_agent'] = stored_user_agent else: self.log(f"Using fingerprint: UA={context_options['user_agent'][:50]}...", "debug") except Exception as e: self.log(f"Error getting stored user_agent, using default: {e}", "debug") self.context = self.browser.new_context(**context_options) # Load cookies self.load_cookies(self.context) self.page = self.context.new_page() # Add comprehensive anti-detection scripts (dynamically from cloudflare_handler) self.page.add_init_script(get_playwright_stealth_scripts()) self.log("Browser started and ready", "info") def _stop_browser(self): """Stop the browser safely with proper error handling""" # Close context first if self.context: try: self.context.close() self.log("Browser context closed", "debug") except Exception as e: self.log(f"Error closing browser context: {e}", "warning") finally: self.context = None # Close browser if self.browser: try: self.browser.close() self.log("Browser closed", "debug") except Exception as e: self.log(f"Error closing browser: {e}", "warning") finally: self.browser = None # Stop playwright if self.playwright: try: self.playwright.stop() except Exception as e: self.log(f"Error stopping playwright: {e}", "warning") finally: self.playwright = None self.page = None def __del__(self): """Cleanup browser when instance is destroyed""" self._stop_browser() def __enter__(self): """Context manager entry - allows using 'with' statement""" return self def __exit__(self, exc_type, exc_val, exc_tb): """Context manager exit - ensures browser cleanup""" self._stop_browser() return False # Don't suppress exceptions def get_profile_info(self, username: str) -> Optional[Dict]: """Extract profile info (avatar URL, bio, display name) from imginn profile page. Returns dict with keys: avatar_url, bio, display_name, or None on failure. """ import time as _time import random as _random self._enforce_rate_limit("posts") self._start_browser() page = self.page if not page: return None try: url = f"https://imginn.com/{username.lower()}/?ref=index" self.log(f"Fetching profile info for @{username} from imginn", "info") page.goto(url, wait_until='domcontentloaded') wait_time = 5 + _random.uniform(0, 2) _time.sleep(wait_time) if not self.wait_for_cloudflare(page): self.log("Page didn't load for profile info extraction", "warning") return None self.save_cookies(self.context) _time.sleep(2) # Use JavaScript to extract profile info with multiple selector strategies profile_info = page.evaluate("""() => { const result = {}; // --- Avatar --- // Strategy 1: img inside a profile/user info section const avatarSelectors = [ '.profile-avatar img', '.user-avatar img', '.avatar img', '.profile-info img', '.info img:first-of-type', 'header img', '.user img', ]; for (const sel of avatarSelectors) { const el = document.querySelector(sel); if (el && el.src && !el.src.includes('lazy') && !el.src.includes('data:')) { result.avatar_url = el.src; break; } } // Strategy 2: find small/round img with scontent or profile in src if (!result.avatar_url) { const imgs = document.querySelectorAll('img'); for (const img of imgs) { const src = img.src || ''; if ((src.includes('scontent') || src.includes('profile') || src.includes('avatar') || src.includes('imginn.com')) && !src.includes('lazy') && !src.includes('data:')) { const rect = img.getBoundingClientRect(); if (rect.width > 20 && rect.width < 250) { result.avatar_url = src; break; } } } } // Clean avatar URL: strip query params (imginn CDN works without them // and the full URL often has malformed double-? from Instagram CDN paths) if (result.avatar_url && result.avatar_url.includes('?')) { result.avatar_url = result.avatar_url.split('?')[0]; } // --- Bio --- const bioSelectors = [ '.biography', '.bio', '.user-bio', '.profile-bio', '.profile-info .description', '.info .bio', ]; for (const sel of bioSelectors) { const el = document.querySelector(sel); if (el && el.textContent.trim().length > 2) { result.bio = el.textContent.trim(); break; } } // --- Display Name --- const nameSelectors = [ '.fullname', '.display-name', '.profile-name', '.name', '.user-info h1', 'h1', ]; for (const sel of nameSelectors) { const el = document.querySelector(sel); if (el && el.textContent.trim().length > 1 && el.textContent.trim().length < 100) { result.display_name = el.textContent.trim(); break; } } return result; }""") # Save debug screenshot for future selector tuning try: screenshot_path = Path(f"/tmp/imginn_profile_{username}.png") page.screenshot(path=str(screenshot_path)) self.log(f"Profile screenshot saved to {screenshot_path}", "debug") except Exception: pass if profile_info and any(profile_info.values()): self.log(f"Extracted profile info: avatar={'yes' if profile_info.get('avatar_url') else 'no'}, " f"bio={'yes' if profile_info.get('bio') else 'no'}, " f"name={profile_info.get('display_name', 'no')}", "info") return profile_info else: # Save page HTML for debugging try: html_path = Path(f"/tmp/imginn_profile_{username}.html") html_path.write_text(page.content()[:50000]) self.log(f"No profile info found - HTML saved to {html_path}", "warning") except Exception: pass return None except Exception as e: self.log(f"Error getting profile info for @{username}: {e}", "error") return None def _extract_media_id_from_url(self, url: str) -> str: """Extract Instagram media ID from URL""" # URL format: https://imginn.com/p/MEDIA_ID/ # or just /p/MEDIA_ID/ match = re.search(r'/p/([^/]+)/?', url) if match: return match.group(1) return None def _update_file_timestamps(self, filepath: Path, post_date: datetime): """Update all timestamps for a file to match the post date""" try: # Convert datetime to timestamp timestamp = post_date.timestamp() # 1. Update file system timestamps (access time and modification time) os.utime(filepath, (timestamp, timestamp)) self.log(f"Updated file timestamps to {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug") # 2. Try to update creation time (platform-specific) if platform.system() == 'Darwin': # macOS # Use SetFile command on macOS date_str = post_date.strftime('%m/%d/%Y %H:%M:%S') try: subprocess.run( ['SetFile', '-d', date_str, str(filepath)], capture_output=True, text=True ) except (subprocess.SubprocessError, FileNotFoundError, OSError): pass # SetFile not available on this system elif platform.system() == 'Windows': # On Windows, use PowerShell with proper escaping to prevent injection filepath_escaped = str(filepath).replace("'", "''") date_escaped = post_date.isoformat().replace("'", "''") ps_command = f"(Get-Item -LiteralPath '{filepath_escaped}').CreationTime = Get-Date '{date_escaped}'" try: subprocess.run( ['powershell', '-Command', ps_command], capture_output=True, text=True ) except (subprocess.SubprocessError, FileNotFoundError, OSError): pass # PowerShell command failed # Linux doesn't support changing creation time # 3. Update EXIF data for images if str(filepath).lower().endswith(('.jpg', '.jpeg', '.png', '.heic')): self._update_exif_timestamps(filepath, post_date) except Exception as e: self.log(f"Error updating timestamps: {e}", "warning") def _update_exif_timestamps(self, filepath: Path, post_date: datetime): """Update EXIF timestamps in image files""" try: # Check if exiftool is available result = subprocess.run(['which', 'exiftool'], capture_output=True, text=True) if result.returncode == 0: # Format date for EXIF exif_date = post_date.strftime('%Y:%m:%d %H:%M:%S') # Update all date fields in EXIF including MetadataDate for Immich cmd = [ 'exiftool', '-overwrite_original', '-quiet', f'-AllDates={exif_date}', f'-MetadataDate={exif_date}', '-HistoryWhen=', f'-FileModifyDate={exif_date}', str(filepath) ] subprocess.run(cmd, capture_output=True, text=True) self.log(f"Updated EXIF timestamps", "debug") except Exception: # Silently skip if exiftool not available pass def _extract_post_date(self, page) -> datetime: """Try to extract post date from page""" try: # Wait a moment for dynamic content to load page.wait_for_timeout(500) # FIRST: Look for data-created attribute (Unix timestamp) elements_with_data_created = page.locator('[data-created]').all() self.log(f"Found {len(elements_with_data_created)} elements with data-created attribute", "debug") for elem in elements_with_data_created: timestamp_str = elem.get_attribute('data-created') if timestamp_str: try: # Convert Unix timestamp to datetime timestamp = int(timestamp_str) post_date = datetime.fromtimestamp(timestamp) self.log(f"Found data-created timestamp: {timestamp} -> {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug") return post_date except Exception as e: self.log(f"Failed to parse timestamp {timestamp_str}: {e}", "debug") pass # If no data-created found, wait a bit more and try again if len(elements_with_data_created) == 0: self.log("No data-created elements found, waiting for dynamic content...", "debug") # Try to wait for the element to appear try: page.wait_for_selector('[data-created]', timeout=2000) elements_with_data_created = page.locator('[data-created]').all() self.log(f"After waiting for selector: found {len(elements_with_data_created)} elements with data-created", "debug") except Exception: # Still try one more time with a longer wait page.wait_for_timeout(1500) elements_with_data_created = page.locator('[data-created]').all() self.log(f"After timeout wait: found {len(elements_with_data_created)} elements with data-created", "debug") for elem in elements_with_data_created: timestamp_str = elem.get_attribute('data-created') if timestamp_str: try: timestamp = int(timestamp_str) post_date = datetime.fromtimestamp(timestamp) self.log(f"Found data-created timestamp after wait: {timestamp} -> {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug") return post_date except Exception as e: self.log(f"Failed to parse timestamp {timestamp_str}: {e}", "debug") # Fallback: Look for other date elements date_selectors = [ 'time[datetime]', 'time', '.date', '[datetime]', 'span.date', 'div.date' ] for selector in date_selectors: elem = page.locator(selector).first if elem.count() > 0: # Try datetime attribute first datetime_str = elem.get_attribute('datetime') if datetime_str: # Parse ISO format for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d']: try: return datetime.strptime(datetime_str.split('.')[0].replace('Z', ''), fmt) except Exception: continue # Try text content text = elem.text_content() if text: # Parse various date formats # Could be "2 days ago", "September 6, 2025", etc. if "ago" in text.lower(): # Handle relative dates if "hour" in text: hours = int(re.search(r'(\d+)', text).group(1)) return datetime.now() - timedelta(hours=hours) elif "day" in text: days = int(re.search(r'(\d+)', text).group(1)) return datetime.now() - timedelta(days=days) elif "week" in text: weeks = int(re.search(r'(\d+)', text).group(1)) return datetime.now() - timedelta(weeks=weeks) else: # Try parsing absolute date for fmt in ['%B %d, %Y', '%b %d, %Y', '%Y-%m-%d']: try: return datetime.strptime(text, fmt) except Exception: continue except Exception as e: self.log(f"Error extracting date: {e}", "debug") return None def _scan_existing_files(self, output_dir: Path, profile_name: str): """Scan directory for existing files and extract media IDs""" self.downloaded_files = scan_existing_files_for_media_ids( output_dir, profile_name, min_file_size=20000, recursive=False ) if self.downloaded_files: self.log(f"Found {len(self.downloaded_files)} existing media IDs for {profile_name}", "debug") def _is_already_downloaded(self, media_id: str) -> bool: """Check if media_id has already been downloaded (uses centralized function)""" if not self.use_database or not self.unified_db: return False # Use centralized function for consistent cross-module detection return is_instagram_downloaded(self.unified_db, media_id) def _record_download(self, media_id: str, username: str, filename: str, url: str = None, post_date=None, file_path: str = None, content_type: str = 'post', metadata: dict = None, deferred: bool = False): """Record a successful download in the database (uses centralized function) Args: deferred: If True, don't record to database now - add to pending_downloads list for later recording after file move is complete """ # If deferred, store for later recording instead of recording now if deferred: self.pending_downloads.append({ 'media_id': media_id, 'username': username, 'filename': filename, 'url': url, 'post_date': post_date.isoformat() if post_date else None, 'file_path': file_path, 'content_type': content_type, 'metadata': metadata }) self.log(f"Deferred recording for {media_id}", "debug") return True if not self.use_database or not self.unified_db: return False try: # Use centralized function for consistent cross-module storage result = record_instagram_download( db=self.unified_db, media_id=media_id, username=username, content_type=content_type, filename=filename, url=url, post_date=post_date, file_path=file_path, method='imginn', extra_metadata=metadata ) if result: self.log(f"Recorded download for {media_id}", "debug") return result except Exception as e: self.log(f"Failed to record download: {e}", "debug") return False def get_pending_downloads(self): """Get list of downloads that were deferred for later recording Returns: List of download metadata dicts ready for database recording """ return self.pending_downloads.copy() def clear_pending_downloads(self): """Clear the pending downloads list after they've been recorded""" self.pending_downloads = [] def _get_processed_posts(self, username: str) -> set: """Get set of post/story IDs that have been processed from database NOTE: Checks ALL Instagram posts globally, not just this user's, because the same post can appear on multiple profiles (shared posts, tags, reposts) """ processed = set() if not self.unified_db: return processed try: with self.unified_db.get_connection() as conn: cursor = conn.cursor() # Get all Instagram posts globally (same post can appear on multiple profiles) cursor.execute(''' SELECT url, filename, metadata FROM downloads WHERE platform = 'instagram' ''') for row in cursor.fetchall(): url, filename, metadata_str = row # Add full URL to processed set if url: processed.add(url) # Also extract and add post ID from URL for backward compatibility if url and '/p/' in url: match = re.search(r'/p/([^/]+)/', url) if match: processed.add(match.group(1)) # For stories, extract media_id from filename if filename and '_story' in filename: # Extract the long media ID before _story # Format: username_date_MEDIAID_storyN.ext parts = filename.split('_story') if len(parts) >= 2: # Get everything before _story, then get the media ID (last underscore-separated part) pre_story = parts[0] # Split by underscore and skip first 3 parts (username_YYYYMMDD_HHMMSS) id_parts = pre_story.split('_') if len(id_parts) > 3: # Join everything after date as the media_id media_id_full = '_'.join(id_parts[3:]) processed.add(media_id_full) # Also add the extracted Instagram media ID (18-digit number) normalized_id = extract_instagram_media_id(media_id_full) if normalized_id and normalized_id != media_id_full: processed.add(normalized_id) # Also check metadata for media_id if metadata_str: try: metadata = json.loads(metadata_str) if 'post_id' in metadata: processed.add(metadata['post_id']) if 'media_id' in metadata: media_id = metadata['media_id'] processed.add(media_id) # Also add the extracted Instagram media ID normalized_id = extract_instagram_media_id(media_id) if normalized_id and normalized_id != media_id: processed.add(normalized_id) if 'media_id_full' in metadata: processed.add(metadata['media_id_full']) except Exception: pass if processed: self.log(f"Found {len(processed)} processed posts in database for {username}", "debug") except Exception as e: self.log(f"Error loading processed posts from database: {e}", "debug") return processed def save_cookies(self, context): """Save cookies to database or file""" cookies = context.cookies() # Save to database if available if self.unified_db: try: # CRITICAL: Include user_agent for cf_clearance cookies to work self.unified_db.save_scraper_cookies( self.scraper_id, cookies, user_agent=self.user_agent, merge=True ) self.log(f"Saved {len(cookies)} cookies to database", "debug") return except Exception as e: self.log(f"Error saving cookies to database: {e}", "warning") # Fallback to file-based storage if self.cookie_file: storage_data = { 'cookies': cookies, 'timestamp': datetime.now().isoformat() } with open(self.cookie_file, 'w') as f: json.dump(storage_data, f, indent=2) self.log(f"Saved {len(cookies)} cookies to file", "debug") def load_cookies(self, context): """Load saved cookies from database or file""" # Try loading from database first if self.unified_db: try: cookies = self.unified_db.get_scraper_cookies(self.scraper_id) if cookies: # Clean cookies - remove unsupported properties and convert expiry->expires cleaned_cookies = [] for cookie in cookies: cleaned = {k: v for k, v in cookie.items() if k not in ['partitionKey', '_crHasCrossSiteAncestor']} # FlareSolverr uses 'expiry' but Playwright uses 'expires' if 'expiry' in cleaned and 'expires' not in cleaned: cleaned['expires'] = cleaned.pop('expiry') cleaned_cookies.append(cleaned) # CRITICAL: Clear existing cookies first to ensure new cf_clearance takes effect # Otherwise old cookies may override new ones from FlareSolverr try: context.clear_cookies() self.log("Cleared existing browser cookies", "debug") except Exception as e: self.log(f"Could not clear cookies: {e}", "debug") context.add_cookies(cleaned_cookies) self.log(f"Loaded {len(cleaned_cookies)} cookies from database", "info") return True except Exception as e: self.log(f"Error loading cookies from database: {e}", "warning") # Fallback to file-based cookies if not self.cookie_file or not self.cookie_file.exists(): return False try: with open(self.cookie_file, 'r') as f: data = json.load(f) # Check age (24 hours) saved_time = datetime.fromisoformat(data['timestamp']) if datetime.now() - saved_time > timedelta(hours=24): self.log("Cookies expired", "debug") return False # Clean cookies - remove unsupported properties and convert expiry->expires cleaned_cookies = [] for cookie in data['cookies']: # Remove Chrome-specific properties that Playwright doesn't support cleaned = {k: v for k, v in cookie.items() if k not in ['partitionKey', '_crHasCrossSiteAncestor']} # FlareSolverr uses 'expiry' but Playwright uses 'expires' if 'expiry' in cleaned and 'expires' not in cleaned: cleaned['expires'] = cleaned.pop('expiry') cleaned_cookies.append(cleaned) context.add_cookies(cleaned_cookies) self.log(f"Loaded {len(cleaned_cookies)} cookies from file", "info") return True except Exception as e: self.log(f"Failed to load cookies: {e}", "warning") return False def wait_for_cloudflare(self, page): """Wait for Cloudflare to auto-solve or page to load - uses CloudflareHandler with ImgInn-specific checks""" self.log("Waiting for page to load...", "debug") max_wait = 120 # Extended wait - Cloudflare challenges can take up to 120 seconds flaresolverr_attempts = 0 max_flaresolverr_attempts = 3 for i in range(max_wait): time.sleep(1) # Check current URL and content try: current_url = page.url content = page.content().lower() except Exception as e: # Page is still navigating, wait and try again if "navigating" in str(e).lower(): self.log("Page still navigating, waiting...", "debug") continue else: # Some other error, re-raise it raise # First check if the actual content is visible (not Cloudflare) # ImgInn pages will have profile content when loaded if 'imginn' in current_url.lower() and ('posts' in content or 'followers' in content or 'following' in content): # We have actual content, not a challenge self.log(f"Page loaded successfully after {i+1} seconds", "info") return True # Check for actual Cloudflare challenge or server error # NOTE: 'challenge-platform' is NOT a reliable indicator - it's embedded JS that stays on the page # even after successful bypass. Only check for visible interstitial text. challenge_indicators = ['checking your browser', 'just a moment', 'verify you are human', 'enable javascript'] error_indicators = ['internal server error', 'error code 500', 'error code 502', 'error code 503'] has_challenge = any(indicator in content for indicator in challenge_indicators) has_error = any(indicator in content for indicator in error_indicators) if has_error: self.log("Server error detected (500/502/503) - site is likely down", "error") # Save screenshot for debugging try: debug_dir = Path("debug") debug_dir.mkdir(exist_ok=True) screenshot_path = debug_dir / f"server_error_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" page.screenshot(path=str(screenshot_path)) self.log(f"Screenshot saved to {screenshot_path}", "debug") except Exception: pass return False if has_challenge: # Try FlareSolverr at specific intervals (0s, 15s, 30s) # Note: Turnstile checkbox clicking doesn't work - it's designed to block automation if i == 0 or (i in [15, 30] and flaresolverr_attempts < max_flaresolverr_attempts): flaresolverr_attempts += 1 self.log(f"Cloudflare challenge detected, attempting FlareSolverr bypass (attempt {flaresolverr_attempts})...", "info") # Get current browser user_agent for comparison current_browser_ua = None try: current_browser_ua = page.evaluate('() => navigator.userAgent') except Exception: pass # Try to get fresh cookies via FlareSolverr if self._get_cookies_via_flaresolverr(page.url): self.log("Got fresh cookies, reloading page...", "info") # Check if user_agent changed - if so, restart browser new_ua = None try: new_ua = self.unified_db.get_scraper_cookies_user_agent(self.scraper_id) self.log(f"Stored cookie UA: {new_ua[:60] if new_ua else 'None'}...", "debug") self.log(f"Browser UA: {current_browser_ua[:60] if current_browser_ua else 'None'}...", "debug") except Exception as e: self.log(f"Error getting stored UA: {e}", "debug") if new_ua and current_browser_ua and new_ua != current_browser_ua: self.log("User-agent changed, restarting browser with new fingerprint...", "info") self._stop_browser() self._start_browser() page = self.page try: page.goto(current_url, wait_until='domcontentloaded', timeout=30000) except Exception as e: self.log(f"Error navigating after browser restart: {e}", "debug") else: # Reload cookies in browser context try: self.load_cookies(self.context) # Reload the page with new cookies page.reload(wait_until='domcontentloaded', timeout=10000) # CRITICAL: Wait for Cloudflare background JS validation (5-7 seconds) wait_time = 5 + random.uniform(0, 2) self.log(f"Waiting {wait_time:.1f}s for Cloudflare background validation...", "debug") time.sleep(wait_time) except Exception as e: self.log(f"Error reloading page with new cookies: {e}", "debug") else: self.log("FlareSolverr failed, waiting for challenge to resolve...", "warning") # Continue waiting for challenge to resolve continue # Check if we're on the correct page with content if '/p/' in current_url: # Post page # Look for download button or image if 'download' in content or 'data-created' in content: self.log(f"Post page loaded after {i+1} seconds", "info") return True elif '/stories/' in current_url: # Stories page # Stories pages have swiper, reels, or story content if 'swiper' in content or 'data-uid' in content or 'reel' in content: self.log(f"Stories page loaded after {i+1} seconds", "info") return True # Also check for counter/profile info which is on stories pages too if 'counter-item' in content or ('posts' in content and 'followers' in content): self.log(f"Stories page loaded after {i+1} seconds", "info") return True elif '/tagged/' in current_url: # Tagged page # Tagged pages have items grid if 'class="item"' in content or 'data-uid' in content: self.log(f"Tagged page loaded after {i+1} seconds", "info") return True if 'posts' in content and 'followers' in content: self.log(f"Tagged page loaded after {i+1} seconds", "info") return True else: # Profile page # Check if profile content is visible - ImgInn specific if 'imginn' in current_url.lower(): if ('posts' in content and 'followers' in content) or 'following' in content: self.log(f"Profile page loaded after {i+1} seconds", "info") return True # Also check for actual post links if 'href="/p/' in content or 'class="item"' in content: self.log(f"Profile page loaded after {i+1} seconds", "info") return True # Debug: Log what we're seeing if we've been waiting a while if i == 15: self.log(f"Debug: URL={current_url[:50]}, has posts={('posts' in content)}, has swiper={('swiper' in content)}", "debug") # Status updates (only if we haven't detected content yet) if i == 10: self.log("Still waiting (10s)... page loading", "debug") elif i == 20: self.log("Still waiting (20s)... page not ready yet", "info") elif i == 30: self.log("Still waiting (30s)... slow response from server", "info") elif i == 45: self.log("Still waiting (45s)... checking if blocked", "info") elif i == 60: self.log("Still waiting (60s)... page load is slow", "warning") elif i == 90: self.log("Still waiting (90s)... this is taking too long", "warning") # Timeout reached - page didn't load self._page_load_failures += 1 level = "error" if self._page_load_failures >= self._page_load_failure_threshold else "warning" self.log(f"Page load timeout ({self._page_load_failures}x this session). URL: {page.url}", level) # Save screenshot for debugging try: debug_dir = Path("debug") debug_dir.mkdir(exist_ok=True) screenshot_path = debug_dir / f"cloudflare_block_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" page.screenshot(path=str(screenshot_path)) self.log(f"Screenshot saved to {screenshot_path}", "debug") except Exception: pass return False def _dismiss_consent_dialog(self, page): """Dismiss cookie consent / GDPR overlay if present (Google FundingChoices).""" try: # Google FundingChoices consent dialog consent_btn = page.locator( 'button.fc-cta-consent, ' # "Consent" button 'button.fc-cta-do-not-consent, ' # "Do not consent" button 'button[aria-label="Consent"], ' 'button.fc-dismiss-button, ' # Dismiss/close button '.fc-dialog button.fc-primary-button' ).first if consent_btn.count() > 0 and consent_btn.is_visible(): consent_btn.click(force=True) self.log("Dismissed consent dialog", "debug") time.sleep(0.5) return # Fallback: remove the overlay via JS if buttons aren't found overlay = page.locator('.fc-consent-root, .fc-dialog-overlay').first if overlay.count() > 0: page.evaluate("document.querySelectorAll('.fc-consent-root, .fc-dialog-overlay, .fc-dialog-container').forEach(el => el.remove())") self.log("Removed consent overlay via JS", "debug") except Exception: pass def _safe_go_back(self, page, username: str, tagged: bool = False): """Navigate back to profile page safely with timeout handling. Tries go_back() first with a short timeout, falls back to direct navigation. """ try: page.go_back(timeout=10000) except Exception: self.log("go_back timed out, navigating directly to profile", "debug") try: suffix = f"/tagged/?ref=index" if tagged else "/?ref=index" page.goto(f"https://imginn.com/{username}{suffix}", timeout=15000) except Exception as nav_err: self.log(f"Direct navigation back also failed: {nav_err}", "warning") def _is_cloudflare_challenge(self, page) -> bool: """Check if current page is a Cloudflare challenge page. Returns: True if Cloudflare challenge detected, False otherwise """ try: title = page.title().lower() content = page.content().lower()[:2000] # Check first 2000 chars challenge_indicators = ['just a moment', 'checking your browser', 'verify you are human', 'enable javascript', 'cloudflare'] # Check title first (most reliable) if any(indicator in title for indicator in challenge_indicators): return True # Check content if any(indicator in content for indicator in challenge_indicators): return True return False except Exception: return False def _handle_cloudflare_on_post(self, page, post_url: str, max_retries: int = 2) -> bool: """Handle Cloudflare challenge on a post page by getting fresh cookies and retrying. Args: page: Playwright page object post_url: URL of the post to retry max_retries: Maximum number of retry attempts Returns: True if page loaded successfully (no Cloudflare), False if still blocked """ if not self._is_cloudflare_challenge(page): return True # No challenge, page is good self.log(f"Cloudflare challenge detected on post page, attempting bypass...", "warning") for attempt in range(max_retries): # Wait before FlareSolverr attempt - give Cloudflare time to cool down if attempt == 0: wait_time = random.uniform(15, 25) else: wait_time = random.uniform(30, 60) self.log(f"Waiting {wait_time:.1f}s before FlareSolverr attempt {attempt + 1}...", "info") time.sleep(wait_time) # Get fresh cookies via FlareSolverr using the post URL if self._get_cookies_via_flaresolverr(post_url): self.log(f"Got fresh cookies (attempt {attempt + 1}), reloading post...", "info") # Check if user_agent changed - if so, restart browser try: current_browser_ua = page.evaluate('() => navigator.userAgent') new_ua = self.unified_db.get_scraper_cookies_user_agent(self.scraper_id) if new_ua and current_browser_ua and new_ua != current_browser_ua: self.log("User-agent changed, restarting browser...", "info") self._stop_browser() self._start_browser() page = self.page except Exception as e: self.log(f"Error checking user_agent: {e}", "debug") # Reload cookies into browser context try: self.load_cookies(self.context) except Exception as e: self.log(f"Error loading cookies: {e}", "debug") # Navigate directly to the post URL try: page.goto(post_url, wait_until='domcontentloaded', timeout=30000) # Wait for Cloudflare background JS validation (5-7 seconds) wait_time = 5 + random.uniform(0, 2) self.log(f"Waiting {wait_time:.1f}s for Cloudflare background validation...", "debug") time.sleep(wait_time) # Check if still blocked if not self._is_cloudflare_challenge(page): self.log("Cloudflare bypass successful on post page", "info") # IMPORTANT: Save browser cookies after successful bypass # This captures any cookies set by Cloudflare's JS validation try: self.save_cookies(self.context) self.log("Saved browser cookies after successful bypass", "debug") except Exception as e: self.log(f"Error saving cookies after bypass: {e}", "debug") return True else: self.log(f"Still blocked after retry {attempt + 1}", "warning") except Exception as e: self.log(f"Navigation failed after cookie refresh: {e}", "warning") else: self.log(f"FlareSolverr failed (attempt {attempt + 1})", "warning") self.log("Failed to bypass Cloudflare on post page after all retries", "error") return False def _check_post_phrases(self, page, phrase_config: dict) -> bool: """Check if post contains required phrases Args: page: Playwright page object phrase_config: Phrase search configuration { 'phrases': list of phrases to search for, 'case_sensitive': bool, 'match_all': bool (True = all phrases must match, False = any phrase) } Returns: True if post matches phrase criteria, False otherwise """ try: # Get post caption/text caption_selectors = [ '.caption', '.post-caption', 'meta[property="og:description"]', 'meta[name="description"]', '.content', 'div[class*="caption"]', 'span[class*="caption"]' ] post_text = "" for selector in caption_selectors: try: element = page.locator(selector).first if element.count() > 0: text = element.text_content() or element.get_attribute('content') or "" if text: post_text += " " + text except Exception: continue # Also check visible text in the main content area try: main_content = page.locator('main, article, .post-content, div[role="main"]').first if main_content.count() > 0: post_text += " " + (main_content.text_content() or "") except Exception: pass if not post_text: self.log("Could not extract post text for phrase matching", "warning") return False # Clean up text post_text = ' '.join(post_text.split()) # Normalize whitespace phrases = phrase_config.get('phrases', []) if not phrases: return True # No phrases to match = match all case_sensitive = phrase_config.get('case_sensitive', False) match_all = phrase_config.get('match_all', False) if not case_sensitive: post_text = post_text.lower() phrases = [p.lower() for p in phrases] self.log(f"Checking post text ({len(post_text)} chars) for phrases: {phrases}", "debug") # Check phrase matching matches = [] for phrase in phrases: if phrase in post_text: matches.append(phrase) self.log(f"Found phrase: '{phrase}'", "debug") if match_all: # All phrases must be found result = len(matches) == len(phrases) if not result: missing = [p for p in phrases if p not in matches] self.log(f"Missing required phrases: {missing}", "debug") else: # At least one phrase must be found result = len(matches) > 0 if not result: self.log(f"No matching phrases found", "debug") return result except Exception as e: self.log(f"Error checking post phrases: {e}", "error") return False def download(self, username: str, content_type: str = "posts", days_back: int = 14, max_downloads: int = 50, output_dir: str = None, phrase_config: dict = None, defer_database: bool = False): """Download content from a user - compatible with media-downloader interface Args: username: Instagram username content_type: Type of content ("posts", "stories", or "tagged") days_back: How many days back to search max_downloads: Maximum posts to download output_dir: Output directory phrase_config: Optional phrase search configuration { 'enabled': bool, 'phrases': list of phrases to search for, 'case_sensitive': bool, 'match_all': bool (True = all phrases must match, False = any phrase) } defer_database: If True, defer database recording to pending_downloads list for later recording after file move is complete """ # Clear downloaded_files cache between accounts to prevent memory growth self.downloaded_files.clear() # Clear pending downloads for fresh batch self.pending_downloads = [] # Set output directory if output_dir: output_path = Path(output_dir) / username else: output_path = Path(f"/opt/media-downloader/downloads/{username}") # Route to appropriate download method if content_type == "posts": files = self.download_posts( username=username, days_back=days_back, max_posts=max_downloads, output_dir=output_path, phrase_config=phrase_config, defer_database=defer_database ) elif content_type == "stories": files = self.download_stories( username=username, days_back=days_back, max_stories=max_downloads, output_dir=output_path, defer_database=defer_database ) elif content_type == "tagged": files = self.download_tagged( username=username, days_back=days_back, max_posts=max_downloads, output_dir=output_path, phrase_config=phrase_config, defer_database=defer_database ) else: self.log(f"ImgInn does not support content type: {content_type}", "warning") return 0 return len(files) def download_posts(self, username: str, days_back: int = 14, max_posts: int = 50, specific_post_url: str = None, output_dir: Path = None, phrase_config: dict = None, skip_database: bool = False, max_age_hours: int = None, defer_database: bool = False): """Download posts from a user with FastDL naming Args: username: Instagram username days_back: How many days back to search max_posts: Maximum posts to check specific_post_url: Download a specific post output_dir: Output directory phrase_config: Optional phrase search configuration skip_database: If True, don't record downloads in database (for temporary processing) max_age_hours: If specified, only download posts newer than N hours (overrides days_back) defer_database: If True, defer database recording to pending_downloads list for later recording after file move is complete """ # Rate limiting to avoid Cloudflare blocks self._enforce_rate_limit("posts") profile_name = username.lower() if output_dir is None: output_dir = Path(f"/opt/media-downloader/downloads/{profile_name}") output_dir.mkdir(parents=True, exist_ok=True) # Check site status before doing anything else self.log("Checking ImgInn site status...", "debug") site_status, error_msg = self.cf_handler.check_site_status("https://imginn.com/", timeout=10) if self.cf_handler.should_skip_download(site_status): self.log(f"Skipping download for @{profile_name} - ImgInn is unavailable: {error_msg}", "warning") self.activity_manager.update_status(f"Skipped - ImgInn unavailable ({error_msg})") return [] elif site_status == SiteStatus.CLOUDFLARE_CHALLENGE: self.log("Cloudflare challenge detected, will attempt bypass during download", "info") # Scan existing files self._scan_existing_files(output_dir, profile_name) # Get processed posts from database processed_posts = self._get_processed_posts(profile_name) self.log(f"Loaded {len(processed_posts)} processed posts for {profile_name} from database", "info") if len(processed_posts) > 0 and len(processed_posts) < 20: self.log(f"Processed post IDs: {processed_posts}", "debug") downloaded_files = [] # Use max_age_hours if specified, otherwise use days_back if max_age_hours is not None: cutoff_date = datetime.now() - timedelta(hours=max_age_hours) else: cutoff_date = datetime.now() - timedelta(days=days_back) # Update activity status if specific_post_url and profile_name == 'unknown': self.activity_manager.update_status(f"Fetching post...") else: self.activity_manager.update_status("Checking posts") # Start or reuse browser self._start_browser() page = self.page try: # If specific post URL provided, go directly to it if specific_post_url: self.log(f"Navigating to specific post", "info") page.goto(specific_post_url, wait_until='domcontentloaded') else: # Navigate to profile self.log(f"Navigating to @{username} profile", "info") page.goto(f"https://imginn.com/{username}/?ref=index", wait_until='domcontentloaded') # CRITICAL: Wait 5-7 seconds for Cloudflare background JS challenges to complete # Per browserless.io: "Allow 5+ seconds post-page load for background JavaScript challenges" import random wait_time = 5 + random.uniform(0, 2) # 5-7 seconds self.log(f"Waiting {wait_time:.1f}s for Cloudflare background validation...", "debug") time.sleep(wait_time) # Wait for page to load if not self.wait_for_cloudflare(page): self._page_load_failures += 1 level = "error" if self._page_load_failures >= self._page_load_failure_threshold else "warning" self.log(f"Page didn't load properly ({self._page_load_failures}x this session)", level) return [] # Save cookies self.save_cookies(self.context) # Wait for JavaScript to load posts (ImgInn loads posts dynamically) self.log("Waiting for posts to load via JavaScript...", "info") try: # Wait for post links to appear (up to 10 seconds) page.wait_for_selector('a[href*="/p/"]', timeout=10000) self.log("Posts loaded successfully", "info") except Exception: # Timeout - posts might not exist, or page structure changed self.log("Timeout waiting for posts to appear", "warning") time.sleep(2) # Give it a bit more time anyway # If specific post, process it directly if specific_post_url: self.log("Processing specific post", "info") # Extract media ID from URL media_id = self._extract_media_id_from_url(specific_post_url) if not media_id: self.log("Could not extract media ID", "warning") return [] self.log(f"URL Media ID: {media_id}", "debug") # Process this single post (bypass date filter for specific posts) post_links = [None] # Dummy list for iteration bypass_date_filter = True else: # Find posts on profile page self.log("Finding posts...", "info") # Debug: Check what's actually on the page page_content = page.content() if 'no posts' in page_content.lower() or 'page not found' in page_content.lower(): self.log("Page shows 'no posts' or 'not found'", "warning") post_links = page.locator('a[href*="/p/"]').all() self.log(f"Found {len(post_links)} posts", "info") if not post_links: # Debug: Save screenshot to see what's wrong try: screenshot_path = Path(f"/tmp/imginn_no_posts_{username}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png") page.screenshot(path=str(screenshot_path)) self.log(f"No posts found - screenshot saved to {screenshot_path}", "warning") except Exception: pass self.log("No posts found", "warning") return [] bypass_date_filter = False self.log(f"Processing posts (max {max_posts})", "info") # Collect all post URLs upfront to avoid stale element issues post_urls_to_process = [] if not specific_post_url: for idx, pl in enumerate(post_links[:max_posts]): try: href = pl.get_attribute('href', timeout=5000) if href: if not href.startswith('http'): href = f"https://imginn.com{href}" post_urls_to_process.append(href) except Exception as e: self.log(f"Post {idx+1}: Failed to get URL: {str(e)[:50]}", "debug") continue self.log(f"Collected {len(post_urls_to_process)} post URLs", "debug") # Track consecutive old posts to handle pinned posts consecutive_old_posts = 0 max_consecutive_old_posts = 5 # Allow up to 5 old posts (pinned) before stopping # Set initial progress so dashboard shows 0/N immediately total_posts = len(post_urls_to_process) if not specific_post_url else 1 self.activity_manager.update_status( "Downloading posts", progress_current=0, progress_total=total_posts ) for i, post_url in enumerate(post_urls_to_process if not specific_post_url else [specific_post_url]): # Update progress at start of each iteration (fires even on skips) self.activity_manager.update_status( "Downloading posts", progress_current=i + 1, progress_total=total_posts ) try: # Handle specific post vs regular posts if specific_post_url: # Already on the specific post page post_url = specific_post_url media_id = self._extract_media_id_from_url(post_url) else: # URL already collected and formatted media_id = self._extract_media_id_from_url(post_url) if not media_id: self.log(f"Post {i+1}: Could not extract media ID", "warning") continue # Check if post was already processed (from database) if media_id in processed_posts: # Skip if in database - trust the database tracking self.log(f"Post {i+1}: {media_id} already processed (database), skipping", "debug") continue # Rate limiting between post downloads to avoid Cloudflare blocks if i > 0: post_delay = random.uniform(3, 8) self.log(f"Rate limit: waiting {post_delay:.1f}s before post {i+1}", "debug") time.sleep(post_delay) self.log(f"Post {i+1}: Processing {media_id}", "info") # Navigate directly to post URL (more reliable than clicking which can timeout) try: page.goto(post_url, wait_until='domcontentloaded', timeout=30000) except Exception as nav_err: self.log(f"Post {i+1}: Navigation failed: {nav_err}", "warning") continue # Wait for page to load time.sleep(2) # Wait for navigation to complete try: page.wait_for_load_state('networkidle', timeout=5000) except Exception: # Continue even if network isn't idle - page might still be usable self.log("Network didn't idle, but continuing", "debug") # Check if on post page if "/p/" not in page.url: self.log(f"Not a downloadable post (URL: {page.url})", "warning") self._safe_go_back(page, username) continue # IMPORTANT: Wait for post page content to fully render # This ensures download buttons are from the POST PAGE, not profile page preview try: # Wait for the post container to be visible (imginn uses main-content now) page.wait_for_selector('div.main-content, div.post, div.content, div.single-post', timeout=3000) time.sleep(1) # Additional wait for download buttons to render except Exception: self.log("Post container not found, checking for Cloudflare...", "debug") # Check for Cloudflare challenge and handle it cloudflare_bypassed = False if self._is_cloudflare_challenge(page): self.log(f"Cloudflare challenge detected on post {media_id}", "warning") if not self._handle_cloudflare_on_post(page, post_url): # Cloudflare bypass failed - skip this post WITHOUT marking as processed # so it can be retried on next run self.log(f"Skipping post {media_id} due to Cloudflare block (will retry later)", "warning") try: page.goto(f"https://imginn.com/{username}/?ref=index") time.sleep(3) except Exception: pass continue cloudflare_bypassed = True self.log(f"Navigated to post page: {page.url}", "debug") self._dismiss_consent_dialog(page) # Extract actual username from post page if we don't have it (e.g., specific_post_url with unknown user) if profile_name == 'unknown' or specific_post_url: try: username_elem = page.locator('div.username a').first if username_elem.count() > 0: username_href = username_elem.get_attribute('href') if username_href: # Extract username from href like "/evalongoria/" -> "evalongoria" extracted_username = username_href.strip('/').lower() if extracted_username and extracted_username != 'unknown': profile_name = extracted_username self.log(f"Extracted username from post page: @{profile_name}", "info") # Update activity status with real username self.activity_manager.update_status("Downloading posts") except Exception as e: self.log(f"Could not extract username from post page: {e}", "debug") # Extract post date - ALWAYS extract for proper file naming post_date = self._extract_post_date(page) # Use post date for filename, or current date if post_date: date_str = post_date.strftime('%Y%m%d_%H%M%S') self.log(f"Original post date: {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug") else: date_str = datetime.now().strftime('%Y%m%d_%H%M%S') self.log(f"No original date found, using current time", "debug") # Check date filter AFTER extracting date (bypass for specific posts) if not bypass_date_filter and post_date and post_date < cutoff_date: consecutive_old_posts += 1 self.log(f"Post too old ({post_date.strftime('%Y-%m-%d')}), skipping (consecutive old: {consecutive_old_posts}/{max_consecutive_old_posts})", "info") # Mark this old post as checked in database to avoid re-checking # Only mark if doing phrase search (has phrase_config) if phrase_config and media_id: self._record_download( media_id=media_id, username=profile_name, filename=f"_old_post_{media_id}", url=post_url, post_date=post_date, content_type='post', metadata={'marker': True, 'reason': 'old_post'} ) self._safe_go_back(page, username) # Stop only after 5 consecutive old posts (handles pinned posts at top) if consecutive_old_posts >= max_consecutive_old_posts: self.log(f"Found {consecutive_old_posts} consecutive old posts - stopping", "info") break else: continue # Skip this old post but keep checking (might be pinned) # Reset consecutive old posts counter - we found a post within date range consecutive_old_posts = 0 # Check for phrase matching if configured if phrase_config and phrase_config.get('enabled'): if not self._check_post_phrases(page, phrase_config): self.log(f"Post does not match phrase criteria, skipping download", "info") # Mark this post as checked (but not downloaded) in database # This prevents re-checking the same post every run if media_id: self._record_download( media_id=media_id, username=profile_name, filename=f"_phrase_checked_{media_id}", url=post_url, post_date=post_date, content_type='post', metadata={'marker': True, 'reason': 'phrase_checked'} ) self._safe_go_back(page, username) continue else: self.log(f"Post matches phrase criteria, using high-res download", "info") # Check for carousel carousel_next = page.locator('div[role="button"][aria-label*="Next"], .swiper-button-next').first has_carousel = carousel_next.count() > 0 if has_carousel: self.log(f"Carousel detected - will download all carousel images", "info") self._dismiss_consent_dialog(page) # CRITICAL: Wait for POST PAGE carousel download buttons to be ready # This prevents downloading from the profile page preview try: # Wait for download buttons with POST PAGE URLs (have "scontent" or "post" in them) page.wait_for_selector('a.btn[href*="scontent"], a[download], a.download', timeout=3000) time.sleep(1.5) # Additional wait for all carousel images to load self.log("Carousel download buttons ready on post page", "debug") except Exception: self.log("Download buttons not found, but continuing", "debug") else: self.log("Single image post", "debug") # Handle downloads - always use download buttons from post page image_count = 0 max_images = 10 # Download images (carousel or single) if has_carousel: # First, let's find all carousel slides all_slides = page.locator('.swiper-slide').all() self.log(f"Found {len(all_slides)} carousel slides", "debug") # Download each slide's image for slide_index in range(min(len(all_slides), max_images)): self.log(f"Processing carousel slide {slide_index + 1}/{len(all_slides)}", "debug") # Get the current slide element to scope our searches current_slide = all_slides[slide_index] # Click next to navigate to this slide (except for first one) if slide_index > 0: next_btn = page.locator('div[role="button"][aria-label*="Next"], .swiper-button-next').first if next_btn.count() > 0 and next_btn.is_visible(): try: next_btn.click(force=True) except Exception: self.log(f"Carousel next button click timed out at slide {slide_index + 1}, stopping carousel", "warning") break time.sleep(2) # Wait for slide transition and image to load # First, try to find a download button for this carousel item # IMPORTANT: Search within CURRENT SLIDE only, not entire page download_btn = None download_url = None webp_fallback_url = None # Look for download button on the current slide - prefer high-res, fallback to .webp download_selectors = [ 'a.btn[href*="scontent"][href*=".jpg"]', # High-res jpg 'a.btn[href*="scontent"][href*=".mp4"]', # Video 'a.btn[href*="scontent"]', # Any scontent 'a[download][href*=".jpg"]', 'a[download][href*=".mp4"]', 'a.download', 'a[download]', 'a[href*="/post"]' ] # Search for download buttons - first try within slide, then try page-level # Imginn often has download buttons outside the .swiper-slide elements search_contexts = [current_slide, page] for search_context in search_contexts: if download_url: # Already found, skip other contexts break for selector in download_selectors: btn = search_context.locator(selector).first if btn.count() > 0: temp_url = btn.get_attribute('href') if temp_url and temp_url != '#' and temp_url != 'javascript:void(0)': if not temp_url.startswith('http'): temp_url = f"https://imginn.com{temp_url}" # Store .webp as fallback, but keep looking for better if '.webp' in temp_url.lower(): if not webp_fallback_url: webp_fallback_url = temp_url self.log(f"Found .webp link (fallback): {temp_url[:80]}...", "debug") continue # Found non-.webp link, use it download_btn = btn download_url = temp_url self.log(f"Found high-res download for carousel slide {slide_index + 1}: {download_url[:80]}...", "debug") break # Use .webp fallback if no high-res found used_webp_fallback = False if not download_url and webp_fallback_url: download_url = webp_fallback_url used_webp_fallback = True self.log(f"Using .webp fallback for carousel slide {slide_index + 1}", "info") # If we found a download button, use it for high-res if download_url: try: import requests from urllib.parse import urlparse, unquote response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename and media ID from the actual file url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] # The media ID is the filename without extension actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename for carousel image using actual media ID filename = f"{profile_name}_{date_str}_{actual_media_id}_{slide_index + 1}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib # Check for duplicate hash (hash blacklist persists even if original deleted) file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): # Duplicate hash found - content was already downloaded (prevents redownload of deleted content) self.log(f"āš  Duplicate content detected (hash match): {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") # Delete the duplicate regardless of whether original file still exists try: filepath.unlink() self.log(f"Deleted duplicate (hash blacklist): {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) # Log with appropriate quality label quality_label = "fallback" if used_webp_fallback else "high-res" self.log(f"Downloaded ({quality_label}): {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) image_count += 1 # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) if not skip_database or defer_database: unique_url = f"{post_url}#{filename}" self._record_download( media_id=actual_media_id, username=profile_name, filename=filename, url=unique_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) continue # Skip to next slide except Exception as e: self.log(f"Failed to download high-res carousel image {slide_index + 1}: {e}, falling back to standard res", "warning") # Fallback: Find the current slide's media (img or video) if no download button # current_slide already defined at top of loop # Try img first, then video media_src = None slide_img = current_slide.locator('img').first if slide_img.count() > 0: media_src = slide_img.get_attribute('src') # If it's a lazy placeholder, wait for it to load if media_src and 'lazy.jpg' in media_src: self.log(f"Slide {slide_index + 1} is lazy, waiting for load...", "debug") # Trigger load by making it visible current_slide.scroll_into_view_if_needed() time.sleep(1) # Get src again media_src = slide_img.get_attribute('src') else: # Check for video tag slide_video = current_slide.locator('video source, video').first if slide_video.count() > 0: media_src = slide_video.get_attribute('src') self.log(f"Found video for slide {slide_index + 1}", "debug") if media_src and 'lazy.jpg' not in media_src and '483011604' not in media_src: self.log(f"Downloading carousel media {slide_index + 1} (standard res): {media_src[:80]}...", "debug") # Download this media try: import requests from urllib.parse import urlparse, unquote if not media_src.startswith('http'): media_src = f"https:{media_src}" if media_src.startswith('//') else f"https://imginn.com{media_src}" response = requests.get(media_src, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename and media ID from the actual file url_path = urlparse(media_src).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] # The media ID is the filename without extension actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename for carousel image using actual media ID filename = f"{profile_name}_{date_str}_{actual_media_id}_{slide_index + 1}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): existing_path = PathLib(existing['file_path']) if existing_path.exists(): self.log(f"āš  Duplicate file detected: {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") try: filepath.unlink() self.log(f"Deleted duplicate: {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) self.log(f"Downloaded: {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) image_count += 1 # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) if not skip_database or defer_database: unique_url = f"{post_url}#{filename}" self._record_download( media_id=actual_media_id, username=profile_name, filename=filename, url=unique_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) except Exception as e: self.log(f"Failed to download carousel media {slide_index + 1}: {e}", "error") else: self.log(f"Slide {slide_index + 1} has no valid media (img/video)", "warning") # Skip the old carousel download logic pass # This duplicate block is not needed - single image logic is already handled above # OLD carousel logic removed - handled above if False: # Wait for carousel content to load time.sleep(1) # Find download button AND image elements # ImgInn sometimes has the full image in an img tag, not just download button download_selectors = [ 'a[download]', 'a.download-btn', 'a[href*="scontent"]', 'a[href*="s3.imginn.com"]', 'a.download', 'a[href*="/post"][href*=".jpg"]', 'a[href*="/post"][href*=".mp4"]', 'button.download', 'a.btn-download' ] # For carousel images, we need to find the actual post image, not the profile thumbnail # Look for images that are NOT the profile pic and NOT lazy placeholders img_src = None # Try to find the carousel image (exclude profile pic and lazy images) possible_images = page.locator('img[src*="post"], img[src*="scontent"]:not([src*="profile"])').all() for img_elem in possible_images: src = img_elem.get_attribute('src') if src and 'lazy.jpg' not in src and '483011604' not in src: # Exclude profile pic img_src = src self.log(f"Found carousel image src: {img_src[:100]}...", "debug") break # If no good image found, wait and try again if not img_src or 'lazy.jpg' in img_src: time.sleep(1) # Try once more after waiting main_image = page.locator('img[src*="post"]:not([src*="lazy"])').first if main_image.count() > 0: img_src = main_image.get_attribute('src') if img_src: self.log(f"Found carousel image after wait: {img_src[:100]}...", "debug") download_btn = None for selector in download_selectors: btn = page.locator(selector).first if btn.count() > 0: download_btn = btn break if download_btn and download_btn.count() > 0: try: # For ImgInn, we should click the download button to get the full-size image # The href often points to a thumbnail, not the full image download_url = download_btn.get_attribute('href') self.log(f"Download button href: {download_url[:100] if download_url else 'None'}...", "debug") # Try clicking the button for browser download first try: self.log(f"Attempting browser download (clicking button)", "debug") with page.expect_download(timeout=5000) as download_info: download_btn.click() download = download_info.value original_name = download.suggested_filename media_id_from_file = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' download_method = 'browser' response = None self.log(f"Browser download completed: {original_name}", "debug") except Exception: # Fallback to direct download if clicking doesn't work self.log(f"Browser download failed, trying direct download", "debug") # For carousels, if no download URL or it's invalid, use image src if has_carousel and (not download_url or download_url == "None" or download_url == "null"): if img_src: self.log(f"No download button for carousel, using image src", "debug") download_url = img_src # Be more lenient with download URLs - accept any https URL that looks like it could be an image/video if download_url and download_url.startswith('http'): # Make sure it's not just the post page URL if '/p/' not in download_url or download_url.endswith(('.jpg', '.jpeg', '.png', '.heic', '.mp4', '.webm')): import requests response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() self.log(f"Downloaded {len(response.content)} bytes", "debug") download_method = 'direct' # Extract filename from URL from urllib.parse import urlparse, unquote url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) # Remove 'post' prefix if present if original_name.startswith('post'): original_name = original_name[4:] media_id_from_file = Path(original_name).stem # This is the actual media ID ext = Path(original_name).suffix or '.jpg' else: # Try to use image src instead if img_src: self.log(f"Download URL is post page, using image src instead", "debug") download_url = img_src if not download_url.startswith('http'): download_url = f"https://imginn.com{download_url}" import requests response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() download_method = 'direct' from urllib.parse import urlparse, unquote url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] media_id_from_file = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' else: raise Exception("No valid download URL found") else: raise Exception("No valid download URL found") # Update our tracked media ID with the correct one from the file if media_id_from_file: media_id = media_id_from_file self.log(f"Media ID from file: {media_id}", "debug") # For carousels, if we don't get a unique media ID, generate one normalized_media_id = extract_instagram_media_id(media_id) if media_id else None if has_carousel and (not media_id or media_id in self.downloaded_files or (normalized_media_id and normalized_media_id in self.downloaded_files)): # Generate unique ID for this carousel image media_id = f"{media_id_base}_carousel_{carousel_index}" normalized_media_id = extract_instagram_media_id(media_id) self.log(f"Generated carousel media ID: {media_id}", "debug") # Check if this media ID is already downloaded (both original and normalized) if media_id in self.downloaded_files or (normalized_media_id and normalized_media_id in self.downloaded_files): self.log(f"Already have {media_id}, skipping download but continuing carousel", "debug") # Still count this as an image even if skipped image_count += 1 if has_carousel: carousel_index += 1 else: self.log(f"Downloading new file for {media_id}", "debug") # Build filename with FastDL format if has_carousel: # For carousel items, append index (simpler format) filename = f"{profile_name}_{date_str}_{media_id_base}_{carousel_index}{ext}" else: filename = f"{profile_name}_{date_str}_{media_id}{ext}" filepath = output_dir / filename # Save the downloaded content if download_method == 'direct': with open(filepath, 'wb') as f: f.write(response.content) else: download.save_as(filepath) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib # Check for duplicate hash (hash blacklist persists even if original deleted) file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): # Duplicate hash found - content was already downloaded (prevents redownload of deleted content) self.log(f"āš  Duplicate content detected (hash match): {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") # Delete the duplicate regardless of whether original file still exists try: filepath.unlink() self.log(f"Deleted duplicate (hash blacklist): {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update file timestamps to match post date if post_date: self._update_file_timestamps(filepath, post_date) self.log(f"Downloaded: {filename}", "info") downloaded_files.append(str(filepath)) image_count += 1 # Add to tracking self.downloaded_files.add(media_id) # Increment carousel index for next image if has_carousel: carousel_index += 1 # Mark as downloaded in database (or defer for later) # Use per-slide URL for carousels so each slide gets a unique url_hash record_url = f"{post_url}?img_index={carousel_index + 1}" if has_carousel else post_url if not skip_database or defer_database: self._record_download( media_id=media_id, username=profile_name, filename=filename, url=record_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) except Exception as e: self.log(f"Download failed: {e}", "error") import traceback self.log(f"Traceback: {traceback.format_exc()}", "debug") break else: # No download button found, try using the image src as fallback page_url = page.url self.log(f"No download button found on {page_url}, trying image src", "warning") # Use the image src we found earlier if img_src: try: self.log(f"Using image src as fallback: {img_src[:100]}...", "debug") import requests from urllib.parse import urlparse, unquote # Ensure full URL if not img_src.startswith('http'): img_src = f"https://imginn.com{img_src}" response = requests.get(img_src, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename from URL url_path = urlparse(img_src).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename with carousel index if needed if has_carousel and carousel_index > 1: filename = f"{profile_name}_{date_str}_{media_id}_{carousel_index}{ext}" else: filename = f"{profile_name}_{date_str}_{media_id}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded via image src: {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib # Check for duplicate hash (hash blacklist persists even if original deleted) file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): # Duplicate hash found - content was already downloaded (prevents redownload of deleted content) self.log(f"āš  Duplicate content detected (hash match): {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") # Delete the duplicate regardless of whether original file still exists try: filepath.unlink() self.log(f"Deleted duplicate (hash blacklist): {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) image_count += 1 self.downloaded_files.add(media_id) # Mark in database (or defer for later) # Use per-slide URL for carousels so each slide gets a unique url_hash record_url = f"{post_url}?img_index={carousel_index + 1}" if has_carousel else post_url if not skip_database or defer_database: self._record_download( media_id=media_id, username=profile_name, filename=filename, url=record_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) except Exception as e: self.log(f"Failed to download via image src: {e}", "error") # Don't break here - might be a temporary issue with one image if not has_carousel: break else: self.log(f"No image src available as fallback", "debug") # For carousels, we might still have more images after clicking next if not has_carousel: break # Check for next image in carousel if has_carousel and image_count < max_images: next_btn = page.locator('div[role="button"][aria-label*="Next"], .swiper-button-next').first if next_btn.count() > 0 and next_btn.is_visible(): # Store current image src to detect when it changes current_img_src = img_src if img_src else "" self.log(f"Clicking next for carousel image {carousel_index}", "debug") try: next_btn.click(force=True) except Exception: self.log(f"Carousel next button click timed out at image {carousel_index}, stopping carousel", "warning") break # Wait for the image to change time.sleep(2) # Give more time for slide transition and new image to load else: self.log("No more carousel images", "debug") break else: break else: # Single image - download from post page using download button download_url = None webp_fallback_url = None download_selectors = [ 'a.btn[href*="scontent"][href*=".jpg"]', # High-res jpg 'a.btn[href*="scontent"][href*=".mp4"]', # Video 'a.btn[href*="scontent"]', # Any scontent 'a[download][href*=".jpg"]', 'a[download][href*=".mp4"]', 'a.download', 'a[href*="/post"]' ] for selector in download_selectors: btn = page.locator(selector).first if btn.count() > 0: temp_url = btn.get_attribute('href') if temp_url and temp_url != '#' and temp_url != 'javascript:void(0)': if not temp_url.startswith('http'): temp_url = f"https://imginn.com{temp_url}" # Store .webp as fallback, but keep looking for better if '.webp' in temp_url.lower(): if not webp_fallback_url: webp_fallback_url = temp_url self.log(f"Found .webp link (fallback): {temp_url[:80]}...", "debug") continue # Found non-.webp link, use it download_url = temp_url self.log(f"Found high-res download for single image: {download_url[:80]}...", "debug") break # Use .webp fallback if no high-res found if not download_url and webp_fallback_url: download_url = webp_fallback_url self.log(f"Using .webp fallback for single image", "info") if download_url: try: import requests from urllib.parse import urlparse, unquote response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename and media ID from the actual file url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] # The media ID is the filename without extension actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename filename = f"{profile_name}_{date_str}_{actual_media_id}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded (high-res): {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): existing_path = PathLib(existing['file_path']) if existing_path.exists(): self.log(f"āš  Duplicate file detected: {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") try: filepath.unlink() self.log(f"Deleted duplicate: {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) image_count = 1 # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) if not skip_database or defer_database: self._record_download( media_id=actual_media_id, username=profile_name, filename=filename, url=post_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) except Exception as e: self.log(f"Failed to download single image: {e}", "warning") else: # No download button found - try video/image src as fallback self.log("No download button found, trying video/image src fallback", "debug") media_src = None # Try video first - multiple selectors for different page structures video_selectors = [ 'video source[src]', 'video[src]', 'video source[type*="mp4"]', '.video-container video', '.post-video video', 'div[class*="video"] video', 'video' ] for v_selector in video_selectors: video_elem = page.locator(v_selector).first if video_elem.count() > 0: # Try src attribute first, then check source child media_src = video_elem.get_attribute('src') if not media_src: source_elem = video_elem.locator('source').first if source_elem.count() > 0: media_src = source_elem.get_attribute('src') if media_src and media_src != '#': self.log(f"Found video src via '{v_selector}': {media_src[:80]}...", "debug") break # If no video found, wait a bit and try again (videos may lazy-load) if not media_src: time.sleep(2) for v_selector in video_selectors: video_elem = page.locator(v_selector).first if video_elem.count() > 0: media_src = video_elem.get_attribute('src') if not media_src: source_elem = video_elem.locator('source').first if source_elem.count() > 0: media_src = source_elem.get_attribute('src') if media_src and media_src != '#': self.log(f"Found video src after wait via '{v_selector}': {media_src[:80]}...", "debug") break # Try image if no video if not media_src: img_elem = page.locator('img[src*="scontent"]:not([src*="profile"]), img[src*="post"]').first if img_elem.count() > 0: media_src = img_elem.get_attribute('src') if media_src and 'lazy.jpg' not in media_src: self.log(f"Found image src: {media_src[:80]}...", "debug") else: media_src = None if media_src: try: import requests from urllib.parse import urlparse, unquote if not media_src.startswith('http'): media_src = f"https://imginn.com{media_src}" response = requests.get(media_src, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename from URL url_path = urlparse(media_src).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.mp4' filename = f"{profile_name}_{date_str}_{actual_media_id}{ext}" filepath = output_dir / filename with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded (fallback): {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) if post_date: self._update_file_timestamps(filepath, post_date) image_count = 1 self.downloaded_files.add(actual_media_id) if not skip_database or defer_database: self._record_download( media_id=actual_media_id, username=profile_name, filename=filename, url=post_url, post_date=post_date, file_path=str(filepath), content_type='post', deferred=defer_database ) except Exception as e: self.log(f"Failed to download via fallback: {e}", "error") else: self.log("No download button or media src found for single post", "warning") # Debug: capture screenshot and page content when download fails try: debug_dir = Path("debug") debug_dir.mkdir(exist_ok=True) timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') screenshot_path = debug_dir / f"no_media_{media_id}_{timestamp}.png" page.screenshot(path=str(screenshot_path)) self.log(f"Debug screenshot saved: {screenshot_path}", "debug") # Also log page title and some content title = page.title() self.log(f"Page title: {title}", "debug") # Check if this is a Cloudflare block - don't mark as processed if so if self._is_cloudflare_challenge(page): self.log(f"Cloudflare block detected - NOT marking {media_id} as processed (will retry later)", "warning") # Skip to next post without marking as processed try: page.goto(f"https://imginn.com/{username}/?ref=index") time.sleep(3) except Exception: pass continue except Exception as e: self.log(f"Failed to capture debug screenshot: {e}", "debug") # Mark post as processed in database even if no downloads # (might be already downloaded or failed - but NOT if Cloudflare blocked) if image_count == 0: # Still mark the post URL as processed to avoid re-checking self._record_download( media_id=media_id, username=profile_name, filename=f"{media_id}_skipped", url=post_url, post_date=post_date, content_type='post', metadata={'marker': True, 'reason': 'skipped'} ) # Go back to profile self._safe_go_back(page, username) # If we just bypassed Cloudflare, wait longer to let session stabilize if cloudflare_bypassed: cooldown = random.uniform(15, 25) self.log(f"Post-bypass cooldown: waiting {cooldown:.1f}s to stabilize session", "info") time.sleep(cooldown) else: time.sleep(random.uniform(1, 3)) # Check if back on profile if username not in page.url: page.goto(f"https://imginn.com/{username}/?ref=index") time.sleep(3) except Exception as e: self.log(f"Error processing post: {e}", "error") try: page.goto(f"https://imginn.com/{username}/?ref=index") time.sleep(3) except Exception: pass self.log(f"Downloaded {len(downloaded_files)} files", "info") except Exception as e: self.log(f"Error: {e}", "error") # Don't close browser here - reuse it for next profile # Call _stop_browser() explicitly when done with all profiles return downloaded_files def download_tagged(self, username: str, days_back: int = 14, max_posts: int = 50, output_dir: Path = None, phrase_config: dict = None, defer_database: bool = False): """Download tagged posts from a user Args: username: Instagram username days_back: How many days back to search max_posts: Maximum posts to check output_dir: Output directory phrase_config: Optional phrase search configuration defer_database: If True, defer database recording to pending_downloads list for later recording after file move is complete """ # Rate limiting to avoid Cloudflare blocks self._enforce_rate_limit("tagged") profile_name = username.lower() if output_dir is None: output_dir = Path(f"/opt/media-downloader/downloads/{profile_name}") output_dir.mkdir(parents=True, exist_ok=True) # Check site status before doing anything else self.log("Checking ImgInn site status...", "debug") site_status, error_msg = self.cf_handler.check_site_status("https://imginn.com/", timeout=10) if self.cf_handler.should_skip_download(site_status): self.log(f"Skipping tagged download for @{profile_name} - ImgInn is unavailable: {error_msg}", "warning") return [] elif site_status == SiteStatus.CLOUDFLARE_CHALLENGE: self.log("Cloudflare challenge detected, will attempt bypass during download", "info") # Scan existing files self._scan_existing_files(output_dir, profile_name) # Get processed posts from database processed_posts = self._get_processed_posts(profile_name) self.log(f"Loaded {len(processed_posts)} processed tagged posts for {profile_name} from database", "info") downloaded_files = [] cutoff_date = datetime.now() - timedelta(days=days_back) # Start or reuse browser self._start_browser() page = self.page try: # Navigate to tagged page directly self.log(f"Navigating to @{username} tagged posts page", "info") page.goto(f"https://imginn.com/tagged/{username}/?ref=index", wait_until='domcontentloaded') # CRITICAL: Wait for Cloudflare background JS challenges import random wait_time = 5 + random.uniform(0, 2) self.log(f"Waiting {wait_time:.1f}s for Cloudflare background validation...", "debug") time.sleep(wait_time) # Wait for page to load if not self.wait_for_cloudflare(page): self._page_load_failures += 1 level = "error" if self._page_load_failures >= self._page_load_failure_threshold else "warning" self.log(f"Page didn't load properly ({self._page_load_failures}x this session)", level) return [] # Save cookies self.save_cookies(self.context) # Wait for JavaScript to load posts (ImgInn loads posts dynamically on tagged page) self.log("Waiting for tagged posts to load via JavaScript...", "info") try: # Wait for post links to appear (up to 10 seconds) page.wait_for_selector('a[href*="/p/"]', timeout=10000) self.log("Tagged posts loaded successfully", "info") except Exception: # Timeout - posts might not exist, or page structure changed self.log("Timeout waiting for tagged posts to appear", "warning") time.sleep(2) # Give it a bit more time anyway # Scroll to load more posts (ImgInn uses infinite scroll on tagged page) self.log("Scrolling to load more tagged posts...", "info") previous_count = 0 scroll_attempts = 0 max_scroll_attempts = 10 # Scroll up to 10 times to load posts while scroll_attempts < max_scroll_attempts: # Get current count of post links current_count = page.locator('a[href*="/p/"]').count() if current_count == previous_count and scroll_attempts > 0: # No new posts loaded after scroll, we've reached the end self.log(f"No more tagged posts to load (total: {current_count})", "debug") break if current_count >= max_posts: # We have enough posts self.log(f"Loaded {current_count} tagged posts (reached max_posts limit)", "debug") break previous_count = current_count # Scroll to bottom of page page.evaluate("window.scrollTo(0, document.body.scrollHeight)") time.sleep(1.5) # Wait for new posts to load scroll_attempts += 1 self.log(f"Scroll {scroll_attempts}: Found {current_count} tagged posts", "debug") # Find posts on tagged page self.log("Finding tagged posts...", "info") # Debug: Check what's actually on the page page_content = page.content() if 'no posts' in page_content.lower() or 'page not found' in page_content.lower(): self.log("Page shows 'no posts' or 'not found'", "warning") post_links = page.locator('a[href*="/p/"]').all() self.log(f"Found {len(post_links)} tagged posts", "info") if not post_links: # Debug: Save screenshot to see what's wrong try: screenshot_path = Path(f"/tmp/imginn_no_tagged_{username}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png") page.screenshot(path=str(screenshot_path)) self.log(f"No tagged posts found - screenshot saved to {screenshot_path}", "warning") except Exception: pass self.log("No tagged posts found", "warning") return [] # Extract all post URLs upfront to avoid stale element issues # (elements become stale after page.go_back()) post_urls = [] for idx, post_link in enumerate(post_links[:max_posts]): try: href = post_link.get_attribute('href', timeout=5000) if href: # Ensure full URL if not href.startswith('http'): href = f"https://imginn.com{href}" post_urls.append(href) except Exception as e: self.log(f"Tagged {idx+1}: Failed to get URL: {str(e)[:50]}", "debug") continue self.log(f"Processing {len(post_urls)} tagged posts (max {max_posts})", "info") # Track consecutive old posts to handle pinned posts consecutive_old_posts = 0 max_consecutive_old_posts = 5 # Allow up to 5 old posts (pinned) before stopping # Set initial progress so dashboard shows 0/N immediately self.activity_manager.update_status( "Downloading tagged", progress_current=0, progress_total=len(post_urls) ) for i, post_url in enumerate(post_urls): # Update progress at start of each iteration (fires even on skips) self.activity_manager.update_status( "Downloading tagged", progress_current=i + 1, progress_total=len(post_urls) ) try: # Extract media ID from URL media_id = self._extract_media_id_from_url(post_url) if not media_id: self.log(f"Could not extract media ID from {post_url}", "warning") continue self.log(f"[{i+1}/{len(post_urls)}] Checking tagged post {media_id}", "debug") # Check if already processed (either downloaded or checked for phrases/age) if media_id in processed_posts or post_url in processed_posts: self.log(f"Post {media_id} already processed, skipping", "debug") continue # Rate limiting between post downloads to avoid Cloudflare blocks if i > 0: post_delay = random.uniform(3, 8) self.log(f"Rate limit: waiting {post_delay:.1f}s before tagged post {i+1}", "debug") time.sleep(post_delay) # For tagged posts, ALWAYS navigate to post page for high-res download # (Never use profile download which gives low-res .webp) page.goto(post_url, wait_until='domcontentloaded') # Wait for page to load time.sleep(2) # Wait for navigation to complete try: page.wait_for_load_state('networkidle', timeout=5000) except Exception: # Continue even if network isn't idle - page might still be usable self.log("Network didn't idle, but continuing", "debug") # Check if on post page if "/p/" not in page.url: self.log(f"Not a downloadable post (URL: {page.url})", "warning") self._safe_go_back(page, username, tagged=True) continue # IMPORTANT: Wait for post page content to fully render # This ensures download buttons are from the POST PAGE, not tagged page preview try: # Wait for the post container to be visible page.wait_for_selector('div.main-content, div.post, div.content, div.single-post', timeout=3000) time.sleep(1) # Additional wait for download buttons to render except Exception: self.log("Post container not found, checking for Cloudflare...", "debug") # Check for Cloudflare challenge and handle it cloudflare_bypassed = False if self._is_cloudflare_challenge(page): self.log(f"Cloudflare challenge detected on tagged post {media_id}", "warning") if not self._handle_cloudflare_on_post(page, post_url): # Cloudflare bypass failed - skip this post WITHOUT marking as processed # so it can be retried on next run self.log(f"Skipping tagged post {media_id} due to Cloudflare block (will retry later)", "warning") try: page.goto(f"https://imginn.com/tagged/{username}/?ref=index") time.sleep(3) except Exception: pass continue cloudflare_bypassed = True self.log(f"Navigated to tagged post page: {page.url}", "debug") self._dismiss_consent_dialog(page) # Extract the actual poster's username (not the tagged user) # On tagged pages, posts are FROM other users who tagged this user poster_username = profile_name # Default to tagged user try: username_elem = page.locator('div.username a').first if username_elem.count() > 0: username_href = username_elem.get_attribute('href') if username_href: # Extract username from href like "/evalongoria.of/" -> "evalongoria.of" poster_username = username_href.strip('/').lower() self.log(f"Poster username: @{poster_username}", "debug") except Exception as e: self.log(f"Could not extract poster username, using default: {e}", "debug") # Extract post date - ALWAYS extract for proper file naming post_date = self._extract_post_date(page) # Use post date for filename, or current date if post_date: date_str = post_date.strftime('%Y%m%d_%H%M%S') self.log(f"Original post date: {post_date.strftime('%Y-%m-%d %H:%M:%S')}", "debug") else: date_str = datetime.now().strftime('%Y%m%d_%H%M%S') self.log(f"No original date found, using current time", "debug") # Check date filter if post_date and post_date < cutoff_date: consecutive_old_posts += 1 self.log(f"Tagged post too old ({post_date.strftime('%Y-%m-%d')}), skipping (consecutive old: {consecutive_old_posts}/{max_consecutive_old_posts})", "info") # Clean up temp file if exists if 'temp_download_path' in locals() and temp_download_path and temp_download_path.exists(): temp_download_path.unlink() self.log(f"Deleted temp file for old post", "debug") # Mark this old post as checked in database - use poster_username for tagged content if phrase_config and media_id: self._record_download( media_id=media_id, username=poster_username, filename=f"_old_post_{media_id}", url=post_url, post_date=post_date, content_type='tagged', metadata={'marker': True, 'reason': 'old_post'} ) self._safe_go_back(page, username, tagged=True) # Stop only after 5 consecutive old posts (handles pinned posts at top) if consecutive_old_posts >= max_consecutive_old_posts: self.log(f"Found {consecutive_old_posts} consecutive old tagged posts - stopping", "info") break else: continue # Skip this old post but keep checking (might be pinned) # Reset consecutive old posts counter - we found a post within date range consecutive_old_posts = 0 # Check for phrase matching if configured if phrase_config and phrase_config.get('enabled'): if not self._check_post_phrases(page, phrase_config): self.log(f"Tagged post does not match phrase criteria, skipping download", "info") # Clean up temp file if exists if 'temp_download_path' in locals() and temp_download_path and temp_download_path.exists(): temp_download_path.unlink() self.log(f"Deleted temp file for non-matching post", "debug") # Mark this post as checked (but not downloaded) in database - use poster_username if media_id: self._record_download( media_id=media_id, username=poster_username, filename=f"_phrase_checked_{media_id}", url=post_url, post_date=post_date, content_type='tagged', metadata={'marker': True, 'reason': 'phrase_checked'} ) self._safe_go_back(page, username, tagged=True) continue else: self.log(f"Tagged post matches phrase criteria, using high-res download", "info") # Check for carousel carousel_next = page.locator('div[role="button"][aria-label*="Next"], .swiper-button-next').first has_carousel = carousel_next.count() > 0 if has_carousel: self.log(f"Carousel detected in tagged post - will download all carousel images", "info") self._dismiss_consent_dialog(page) # CRITICAL: Wait for POST PAGE carousel download buttons to be ready # This prevents downloading from the tagged page preview try: # Wait for download buttons with POST PAGE URLs (have "scontent" or "post" in them) page.wait_for_selector('a.btn[href*="scontent"], a[download], a.download', timeout=3000) time.sleep(1.5) # Additional wait for all carousel images to load self.log("Carousel download buttons ready on post page", "debug") except Exception: self.log("Download buttons not found, but continuing", "debug") else: self.log("Single image tagged post", "debug") # Handle downloads - always use download buttons from post page image_count = 0 max_images = 10 # Download images (carousel or single) if has_carousel: all_slides = page.locator('.swiper-slide').all() self.log(f"Found {len(all_slides)} carousel slides in tagged post", "debug") # Download each slide's image for slide_index in range(min(len(all_slides), max_images)): self.log(f"Processing carousel slide {slide_index + 1}/{len(all_slides)}", "debug") # Get the current slide element to scope our searches current_slide = all_slides[slide_index] # Click next to navigate to this slide (except for first one) if slide_index > 0: next_btn = page.locator('div[role="button"][aria-label*="Next"], .swiper-button-next').first if next_btn.count() > 0 and next_btn.is_visible(): try: next_btn.click(force=True) except Exception: self.log(f"Carousel next button click timed out at slide {slide_index + 1}, stopping carousel", "warning") break time.sleep(2) # Wait for slide transition and image to load # Look for download button - prefer high-res, fallback to .webp # IMPORTANT: Search within CURRENT SLIDE only, not entire page download_url = None webp_fallback_url = None slide_downloaded = False # Track if this specific slide was downloaded download_selectors = [ 'a.btn[href*="scontent"][href*=".jpg"]', # High-res jpg 'a.btn[href*="scontent"][href*=".mp4"]', # Video 'a.btn[href*="scontent"]', # Any scontent 'a[download][href*=".jpg"]', 'a[download][href*=".mp4"]', 'a.download', 'a[href*="/post"]' ] # Search for download buttons - first try within slide, then try page-level # Imginn often has download buttons outside the .swiper-slide elements search_contexts = [current_slide, page] for search_context in search_contexts: if download_url: # Already found, skip other contexts break for selector in download_selectors: btn = search_context.locator(selector).first if btn.count() > 0: temp_url = btn.get_attribute('href') if temp_url and temp_url != '#' and temp_url != 'javascript:void(0)': if not temp_url.startswith('http'): temp_url = f"https://imginn.com{temp_url}" # Store .webp as fallback, but keep looking for better if '.webp' in temp_url.lower(): if not webp_fallback_url: webp_fallback_url = temp_url self.log(f"Found .webp link (fallback): {temp_url[:80]}...", "debug") continue # Found non-.webp link, use it download_url = temp_url self.log(f"Found high-res download for carousel slide {slide_index + 1}: {download_url[:80]}...", "debug") break # Use .webp fallback if no high-res found used_webp_fallback = False if not download_url and webp_fallback_url: download_url = webp_fallback_url used_webp_fallback = True self.log(f"Using .webp fallback for carousel slide {slide_index + 1}", "info") # If we found a download button, use it for high-res if download_url: try: import requests from urllib.parse import urlparse, unquote response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename and media ID from the actual file url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] # The media ID is the filename without extension actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename for carousel image using actual media ID (use poster's username) filename = f"{poster_username}_{date_str}_{actual_media_id}_{slide_index + 1}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) # Log with appropriate quality label quality_label = "fallback" if used_webp_fallback else "high-res" self.log(f"Downloaded tagged ({quality_label}): {filename} from @{poster_username} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib # Check for duplicate hash (hash blacklist persists even if original deleted) file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): # Duplicate hash found - content was already downloaded (prevents redownload of deleted content) self.log(f"āš  Duplicate content detected (hash match): {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") # Delete the duplicate regardless of whether original file still exists try: filepath.unlink() self.log(f"Deleted duplicate (hash blacklist): {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) image_count += 1 slide_downloaded = True # Mark this slide as successfully downloaded # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) - use poster_username for tagged content unique_url = f"{post_url}#{filename}" self._record_download( media_id=actual_media_id, username=poster_username, filename=filename, url=unique_url, post_date=post_date, file_path=str(filepath), content_type='tagged', deferred=defer_database ) except Exception as e: self.log(f"Failed to download carousel image {slide_index + 1}: {e}", "error") # Don't continue - try fallback method below # Fallback: Download from current slide's img/video src if no download button worked if not slide_downloaded: self.log(f"Trying fallback: downloading from slide {slide_index + 1} media src", "debug") # current_slide already defined at top of loop # Try img first, then video media_src = None slide_img = current_slide.locator('img').first if slide_img.count() > 0: media_src = slide_img.get_attribute('src') else: # Check for video tag slide_video = current_slide.locator('video source, video').first if slide_video.count() > 0: media_src = slide_video.get_attribute('src') self.log(f"Found video for slide {slide_index + 1}", "debug") if media_src: # Skip lazy placeholders if 'lazy.jpg' not in media_src and '483011604' not in media_src: try: import requests from urllib.parse import urlparse, unquote if not media_src.startswith('http'): media_src = f"https:{media_src}" if media_src.startswith('//') else f"https://imginn.com{media_src}" response = requests.get(media_src, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename url_path = urlparse(media_src).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename filename = f"{poster_username}_{date_str}_{actual_media_id}_{slide_index + 1}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded tagged (fallback): {filename} from @{poster_username} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): existing_path = PathLib(existing['file_path']) if existing_path.exists(): self.log(f"āš  Duplicate file detected: {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") try: filepath.unlink() self.log(f"Deleted duplicate: {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) image_count += 1 # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) - use poster_username for tagged content unique_url = f"{post_url}#{filename}" self._record_download( media_id=actual_media_id, username=poster_username, filename=filename, url=unique_url, post_date=post_date, file_path=str(filepath), content_type='tagged', deferred=defer_database ) except Exception as e: self.log(f"Failed to download from media src for slide {slide_index + 1}: {e}", "error") else: self.log(f"No media (img/video) found for carousel slide {slide_index + 1}", "warning") else: # Single image - download from post page using download button download_url = None webp_fallback_url = None download_selectors = [ 'a.btn[href*="scontent"][href*=".jpg"]', # High-res jpg 'a.btn[href*="scontent"][href*=".mp4"]', # Video 'a.btn[href*="scontent"]', # Any scontent 'a[download][href*=".jpg"]', 'a[download][href*=".mp4"]', 'a.download', 'a[href*="/post"]' ] for selector in download_selectors: btn = page.locator(selector).first if btn.count() > 0: temp_url = btn.get_attribute('href') if temp_url and temp_url != '#' and temp_url != 'javascript:void(0)': if not temp_url.startswith('http'): temp_url = f"https://imginn.com{temp_url}" # Store .webp as fallback, but keep looking for better if '.webp' in temp_url.lower(): if not webp_fallback_url: webp_fallback_url = temp_url self.log(f"Found .webp link (fallback): {temp_url[:80]}...", "debug") continue # Found non-.webp link, use it download_url = temp_url self.log(f"Found high-res download for single image: {download_url[:80]}...", "debug") break # Use .webp fallback if no high-res found if not download_url and webp_fallback_url: download_url = webp_fallback_url self.log(f"Using .webp fallback for single image", "info") if download_url: try: import requests from urllib.parse import urlparse, unquote response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Extract filename and media ID from the actual file url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] # The media ID is the filename without extension actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.jpg' # Build filename using poster's username filename = f"{poster_username}_{date_str}_{actual_media_id}{ext}" filepath = output_dir / filename # Save file with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded tagged (high-res): {filename} from @{poster_username} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): existing_path = PathLib(existing['file_path']) if existing_path.exists(): self.log(f"āš  Duplicate file detected: {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") try: filepath.unlink() self.log(f"Deleted duplicate: {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps if post_date: self._update_file_timestamps(filepath, post_date) image_count = 1 # Add to tracking self.downloaded_files.add(actual_media_id) # Mark in database (or defer for later) - use poster_username for tagged content self._record_download( media_id=actual_media_id, username=poster_username, filename=filename, url=post_url, post_date=post_date, file_path=str(filepath), content_type='tagged', deferred=defer_database ) except Exception as e: self.log(f"Failed to download single image: {e}", "warning") else: # No download button found - try video/image src as fallback self.log("No download button found, trying video/image src fallback", "debug") media_src = None # Try video first - multiple selectors for different page structures video_selectors = [ 'video source[src]', 'video[src]', 'video source[type*="mp4"]', '.video-container video', '.post-video video', 'div[class*="video"] video', 'video' ] for v_selector in video_selectors: video_elem = page.locator(v_selector).first if video_elem.count() > 0: # Try src attribute first, then check source child media_src = video_elem.get_attribute('src') if not media_src: source_elem = video_elem.locator('source').first if source_elem.count() > 0: media_src = source_elem.get_attribute('src') if media_src and media_src != '#': self.log(f"Found video src via '{v_selector}': {media_src[:80]}...", "debug") break # If no video found, wait a bit and try again (videos may lazy-load) if not media_src: time.sleep(2) for v_selector in video_selectors: video_elem = page.locator(v_selector).first if video_elem.count() > 0: media_src = video_elem.get_attribute('src') if not media_src: source_elem = video_elem.locator('source').first if source_elem.count() > 0: media_src = source_elem.get_attribute('src') if media_src and media_src != '#': self.log(f"Found video src after wait via '{v_selector}': {media_src[:80]}...", "debug") break # Try image if no video if not media_src: img_elem = page.locator('img[src*="scontent"]:not([src*="profile"]), img[src*="post"]').first if img_elem.count() > 0: media_src = img_elem.get_attribute('src') if media_src and 'lazy.jpg' not in media_src: self.log(f"Found image src: {media_src[:80]}...", "debug") else: media_src = None if media_src: try: import requests from urllib.parse import urlparse, unquote if not media_src.startswith('http'): media_src = f"https://imginn.com{media_src}" response = requests.get(media_src, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() url_path = urlparse(media_src).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) if original_name.startswith('post'): original_name = original_name[4:] actual_media_id = Path(original_name).stem ext = Path(original_name).suffix or '.mp4' filename = f"{poster_username}_{date_str}_{actual_media_id}{ext}" filepath = output_dir / filename with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded (fallback): {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) if post_date: self._update_file_timestamps(filepath, post_date) image_count = 1 self.downloaded_files.add(actual_media_id) self._record_download( media_id=actual_media_id, username=poster_username, filename=filename, url=post_url, post_date=post_date, file_path=str(filepath), content_type='tagged', deferred=defer_database ) except Exception as e: self.log(f"Failed to download via fallback: {e}", "error") else: self.log("No download button or media src found for single post", "warning") # Debug: capture screenshot and page content when download fails try: debug_dir = Path("debug") debug_dir.mkdir(exist_ok=True) timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') screenshot_path = debug_dir / f"no_media_tagged_{media_id}_{timestamp}.png" page.screenshot(path=str(screenshot_path)) self.log(f"Debug screenshot saved: {screenshot_path}", "debug") # Also log page title title = page.title() self.log(f"Page title: {title}", "debug") # Check if this is a Cloudflare block - don't mark as processed if so if self._is_cloudflare_challenge(page): self.log(f"Cloudflare block detected - NOT marking tagged post {media_id} as processed (will retry later)", "warning") # Skip to next post without marking as processed try: page.goto(f"https://imginn.com/tagged/{username}/?ref=index") time.sleep(3) except Exception: pass continue except Exception as e: self.log(f"Failed to capture debug screenshot: {e}", "debug") # Navigate back to tagged page if image_count > 0: self.log(f"Successfully downloaded {image_count} image(s) from tagged post {media_id}", "info") self._safe_go_back(page, username, tagged=True) # If we just bypassed Cloudflare, wait longer to let session stabilize if cloudflare_bypassed: cooldown = random.uniform(15, 25) self.log(f"Post-bypass cooldown: waiting {cooldown:.1f}s to stabilize session", "info") time.sleep(cooldown) else: time.sleep(1) except KeyboardInterrupt: self.log("Download interrupted by user", "warning") break except Exception as e: self.log(f"Error processing tagged post: {e}", "error") self._safe_go_back(page, username, tagged=True) self.log(f"Downloaded {len(downloaded_files)} tagged files", "info") except Exception as e: self.log(f"Error: {e}", "error") # Don't close browser here - reuse it for next profile return downloaded_files def download_stories(self, username: str, days_back: int = 1, max_stories: int = 50, output_dir: Path = None, skip_database: bool = False, defer_database: bool = False): """Download stories from a user with FastDL naming Args: username: Instagram username days_back: How many days back to search (stories expire after 24h) max_stories: Maximum stories to download output_dir: Output directory skip_database: If True, don't record downloads in database (for temporary processing) defer_database: If True, defer database recording to pending_downloads list for later recording after file move is complete """ profile_name = username.lower() if output_dir is None: output_dir = Path(f"/opt/media-downloader/downloads/{profile_name}") output_dir.mkdir(parents=True, exist_ok=True) # Check site status before doing anything else self.log("Checking ImgInn site status...", "debug") site_status, error_msg = self.cf_handler.check_site_status("https://imginn.com/", timeout=10) if self.cf_handler.should_skip_download(site_status): self.log(f"Skipping stories download for @{profile_name} - ImgInn is unavailable: {error_msg}", "warning") return [] elif site_status == SiteStatus.CLOUDFLARE_CHALLENGE: self.log("Cloudflare challenge detected, will attempt bypass during download", "info") # Scan existing files self._scan_existing_files(output_dir, profile_name) # Get processed stories from database processed_stories = self._get_processed_posts(profile_name) self.log(f"Loaded {len(processed_stories)} processed stories for {profile_name} from database", "info") downloaded_files = [] cutoff_date = datetime.now() - timedelta(days=days_back) # Start or reuse browser self._start_browser() page = self.page try: # Navigate to stories page self.log(f"Navigating to @{username} stories page", "info") page.goto(f"https://imginn.com/stories/{username}/?ref=index", wait_until='domcontentloaded') # CRITICAL: Wait for Cloudflare background JS challenges import random wait_time = 5 + random.uniform(0, 2) self.log(f"Waiting {wait_time:.1f}s for Cloudflare background validation...", "debug") time.sleep(wait_time) # Wait for page to load if not self.wait_for_cloudflare(page): self.log("Stories page didn't load properly", "error") return [] # Save cookies self.save_cookies(self.context) # Wait for stories container to load self.log("Waiting for stories to load...", "info") try: page.wait_for_selector('.swiper-container.reels', timeout=10000) self.log("Stories container loaded", "info") except Exception: self.log("No stories found - may have expired or page structure changed", "warning") return [] # Find the Stories reel (first li.reel with data-uid and title "Stories") self.log("Looking for Stories reel...", "info") stories_reel = None reels = page.locator('li.reel[data-uid]').all() for reel in reels: try: # Check if this is the "Stories" reel title = reel.locator('.title').first.text_content() if title and title.strip().lower() == "stories": stories_reel = reel self.log(f"Found Stories reel", "info") break except Exception: continue if not stories_reel: self.log("No active Stories found for this user", "warning") return [] # Click the Stories reel to open viewer self.log("Opening Stories viewer...", "info") stories_reel.click() time.sleep(2) # Wait for viewer to open # Find all download buttons in the story viewer self.log("Finding story download links...", "info") download_links = page.locator('div.action a.download').all() if not download_links: self.log("No story download links found", "warning") return [] self.log(f"Found {len(download_links)} stories", "info") # Set initial progress so dashboard shows 0/N immediately stories_to_download = min(len(download_links), max_stories) self.activity_manager.update_status( "Downloading stories", progress_current=0, progress_total=stories_to_download ) # Download each story story_index = 1 for i, download_link in enumerate(download_links[:max_stories]): # Update progress at start of each iteration (fires even on skips) self.activity_manager.update_status( "Downloading stories", progress_current=i + 1, progress_total=stories_to_download ) try: # Get download URL download_url = download_link.get_attribute('href') if not download_url or download_url == '#': self.log(f"Story {story_index}: Invalid download URL", "warning") continue self.log(f"Story {story_index}: {download_url[:80]}...", "debug") # Extract media ID from URL or generate unique ID from urllib.parse import urlparse, unquote url_path = urlparse(download_url).path original_name = unquote(url_path.split('/')[-1].split('?')[0]) media_id_full = Path(original_name).stem # Full filename stem for unique naming ext = Path(original_name).suffix or '.jpg' # Extract real Instagram media ID (18-digit number) for duplicate checking media_id_for_tracking = extract_instagram_media_id(media_id_full) self.log(f"Story {story_index}: Full ID: {media_id_full[:40]}..., Tracking ID: {media_id_for_tracking}", "debug") # Check if already downloaded using the normalized media ID if media_id_for_tracking in self.downloaded_files or media_id_for_tracking in processed_stories: self.log(f"Story {story_index}: Already downloaded (tracking ID: {media_id_for_tracking}), skipping", "debug") story_index += 1 continue # Also check with full ID for backwards compatibility if media_id_full in self.downloaded_files or media_id_full in processed_stories: self.log(f"Story {story_index}: Already downloaded (full ID: {media_id_full[:30]}...), skipping", "debug") story_index += 1 continue # Use current date for stories (they expire after 24h) story_date = datetime.now() date_str = story_date.strftime('%Y%m%d_%H%M%S') # Build filename: {profile}_{date}_{media_id}_story{index}{ext} # Use full media ID in filename for uniqueness filename = f"{profile_name}_{date_str}_{media_id_full}_story{story_index}{ext}" filepath = output_dir / filename # Download the story try: import requests response = requests.get(download_url, timeout=30, headers={ 'User-Agent': self.user_agent, 'Referer': 'https://imginn.com/' }, cookies=self._get_cookies_for_requests()) response.raise_for_status() # Save file with open(filepath, 'wb') as f: f.write(response.content) self.log(f"Downloaded story: {filename} ({len(response.content)} bytes)", "info") downloaded_files.append(str(filepath)) # Check for duplicate hash before recording if self.unified_db: from pathlib import Path as PathLib file_hash = self.unified_db.get_file_hash(str(filepath)) if file_hash: existing = self.unified_db.get_download_by_file_hash(file_hash) if existing and existing.get('file_path') and str(filepath) != existing.get('file_path'): existing_path = PathLib(existing['file_path']) if existing_path.exists(): self.log(f"āš  Duplicate file detected: {filename} matches {existing['filename']} from {existing['platform']}/{existing['source']}", "warning") try: filepath.unlink() self.log(f"Deleted duplicate: {filename}", "debug") continue except Exception as e: self.log(f"Failed to delete duplicate {filename}: {e}", "warning") # Update timestamps self._update_file_timestamps(filepath, story_date) # Add both tracking ID and full ID to tracking set for comprehensive duplicate prevention self.downloaded_files.add(media_id_for_tracking) self.downloaded_files.add(media_id_full) # Mark in database with media_id in metadata (or defer for later) # Use the normalized media ID for database tracking to prevent future duplicates if not skip_database or defer_database: self._record_download( media_id=media_id_for_tracking, username=profile_name, filename=filename, url=download_url, post_date=story_date, file_path=str(filepath), content_type='stories', metadata={'media_id_full': media_id_full}, deferred=defer_database ) story_index += 1 except Exception as e: self.log(f"Failed to download story {story_index}: {e}", "error") story_index += 1 continue except Exception as e: self.log(f"Error processing story {story_index}: {e}", "error") story_index += 1 continue self.log(f"Downloaded {len(downloaded_files)} story files", "info") except Exception as e: self.log(f"Error downloading stories: {e}", "error") # Don't close browser here - reuse it for next profile return downloaded_files def main(): """Test the downloader with FastDL naming""" import sys print("=" * 60) print("ImgInn Downloader - FastDL Compatible Naming") print("=" * 60) print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print("=" * 60) downloader = ImgInnDownloader( api_key=os.environ.get("IMGINN_API_KEY", ""), headless=False # Use with xvfb ) # Check for specific post URL in arguments if len(sys.argv) > 1 and 'imginn.com/p/' in sys.argv[-1]: # Download specific post without date filter files = downloader.download_posts( username="evalongoria", days_back=365, # Use large value to bypass date filter max_posts=5, specific_post_url=sys.argv[-1] ) else: # Download evalongoria posts from last 2 weeks files = downloader.download_posts( username="evalongoria", days_back=14, max_posts=50 ) print("\n" + "=" * 60) print("RESULTS") print("=" * 60) if files: print(f"Successfully downloaded {len(files)} files!") print("\nšŸ“ Downloaded files (FastDL naming format):") for f in files: name = Path(f).name size = Path(f).stat().st_size / 1024 # Show the naming format parts = name.split('_', 3) if len(parts) >= 4: print(f" - {name}") print(f" Profile: {parts[0]}") print(f" Date: {parts[1]}_{parts[2]}") print(f" Media ID: {parts[3].split('.')[0]}") print(f" Size: {size:.1f} KB") else: print("No files downloaded") # Check total in folder download_dir = Path("/opt/media-downloader/downloads/evalongoria") if download_dir.exists(): all_files = list(download_dir.glob("*")) total_size = sum(f.stat().st_size for f in all_files) / 1024 print(f"\nšŸ“Š Total in folder: {len(all_files)} files ({total_size:.1f} KB)") if __name__ == "__main__": main()