Free Tool: Backlink Indexability Cleaner (Save Indexing Money)

"""
Cloudflare-Resistant Indexability Checker
Uses curl_cffi to bypass TLS fingerprinting and Cloudflare protection
"""

import os
import time
import csv
import json
import argparse
from urllib.parse import urlparse, urljoin, parse_qs, urlunparse
import re
from bs4 import BeautifulSoup
import robotexclusionrulesparser
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from rich.live import Live
from rich.layout import Layout
from rich import box
from rich.text import Text
from threading import Lock
from dataclasses import dataclass, field
from typing import Optional, Dict, Tuple, List
from collections import Counter
import threading
import warnings
import random

# CRITICAL: Use curl_cffi instead of requests to bypass TLS fingerprinting
try:
    from curl_cffi import requests as cf_requests
    from curl_cffi.requests import Session as CurlSession
    CURL_CFFI_AVAILABLE = True
except ImportError:
    print("WARNING: curl_cffi not available. Install with: pip install curl-cffi")
    print("Falling back to requests library (will NOT bypass Cloudflare)")
    import requests
    CURL_CFFI_AVAILABLE = False

warnings.filterwarnings("ignore")

console = Console()

# ======================= CONFIGURATION =======================

TARGET_URLS_PER_HOUR = 240
MIN_REQUEST_SPACING = 9.0
MAX_REQUEST_SPACING = 20.0
BURST_COOLDOWN = 50.0
BURST_THRESHOLD = 6

DEFAULT_TIMEOUT = 35
DEFAULT_MAX_RETRIES = 3
DEFAULT_DOMAIN_DELAY = 7.0

PARAMS_TO_STRIP = {
    "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
    "gclid", "fbclid", "mc_cid", "mc_eid", "_ga", "campaignid",
    "ref", "source", "campaign", "ad_id",
}

BROWSER_PROFILES = [
    "chrome120", "chrome119", "chrome116",
    "edge99", "safari15_5", "safari15_3",
]

# Global state
robots_cache: Dict[str, robotexclusionrulesparser.RobotExclusionRulesParser] = {}
robots_lock = Lock()

domain_throttle: Dict[str, float] = {}
domain_throttle_lock = Lock()

crawl_delay_cache: Dict[str, float] = {}
crawl_delay_lock = Lock()

domain_failure_counts: Dict[str, int] = {}
domain_cooldowns: Dict[str, float] = {}
domain_failure_lock = Lock()

request_counter = 0
request_counter_lock = Lock()

cloudflare_detected: Dict[str, bool] = {}
cloudflare_lock = Lock()

_session_local = threading.local()

# ======================= DATA CLASSES =======================


@dataclass
class IndexabilityResult:
    url: str
    indexable: bool
    http_status: Optional[int] = None
    final_url: Optional[str] = None
    reason: str = ""
    content_type: Optional[str] = None
    content_length: Optional[int] = None
    robots_allowed: Optional[bool] = None
    canonical_url: Optional[str] = None
    canonical_matches: Optional[bool] = None
    meta_noindex: Optional[bool] = None
    x_robots_noindex: Optional[bool] = None
    redirect_chain: Optional[int] = None
    response_time_ms: Optional[int] = None
    warning_flags: Optional[str] = None
    cloudflare_detected: Optional[bool] = None
    tls_fingerprint: Optional[str] = None


@dataclass
class Stats:
    total: int = 0
    processed: int = 0
    indexable: int = 0
    blocked: int = 0
    warnings: int = 0
    cloudflare_sites: int = 0
    current_url: str = ""
    current_action: str = ""
    recent_events: list = field(default_factory=list)
    start_time: float = field(default_factory=time.time)
    failure_reasons: Counter = field(default_factory=Counter)

    def urls_per_hour(self) -> float:
        elapsed = time.time() - self.start_time
        hours = elapsed / 3600.0
        return self.processed / hours if hours > 0 else 0

    def eta_seconds(self) -> int:
        elapsed = time.time() - self.start_time
        if self.processed == 0:
            return 0
        rate = self.processed / elapsed
        remaining = self.total - self.processed
        return int(remaining / rate) if rate > 0 else 0


# ======================= CLOUDFLARE DETECTION =======================


def detect_cloudflare(response_headers: dict, html: str = "") -> Tuple[bool, str]:
    """Detect if site is using Cloudflare protection."""
    cf_headers = [
        "cf-ray", "cf-request-id", "cf-cache-status",
        "__cf_bm", "cf-mitigated",
    ]

    headers_lower = {k.lower(): v for k, v in response_headers.items()}

    for header in cf_headers:
        if header in headers_lower:
            if "cf-mitigated" in headers_lower:
                return True, "Cloudflare WAF Challenge"
            return True, "Cloudflare (passive)"

    server = headers_lower.get("server", "").lower()
    if "cloudflare" in server:
        return True, "Cloudflare Server"

    if html:
        cf_signatures = [
            "challenge-platform", "cf-challenge", "cf_clearance",
            "Checking your browser", "cf-browser-verification",
            "ray id", "__cf_chl_jschl_tk__",
        ]

        html_lower = html.lower()
        for sig in cf_signatures:
            if sig.lower() in html_lower:
                return True, "Cloudflare JS Challenge"

    return False, ""


# ======================= SESSION MANAGEMENT =======================


def get_session(impersonate: str = None):
    """Get thread-local curl_cffi session with browser impersonation."""
    if not CURL_CFFI_AVAILABLE:
        sess = getattr(_session_local, "session", None)
        if sess is None:
            sess = requests.Session()
            _session_local.session = sess
        return sess

    sess = getattr(_session_local, "cf_session", None)
    if sess is None or (impersonate and getattr(sess, "_impersonate", None) != impersonate):
        if impersonate is None:
            impersonate = random.choice(BROWSER_PROFILES)

        sess = CurlSession(impersonate=impersonate)
        sess._impersonate = impersonate
        _session_local.cf_session = sess

    return sess


# ======================= ENHANCED HEADERS =======================


def build_cloudflare_resistant_headers(referer: Optional[str] = None) -> dict:
    """Build headers that pass Cloudflare's checks."""
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Cache-Control": "max-age=0",
    }

    if referer and random.random() < 0.35:
        headers["Referer"] = referer
        headers["Sec-Fetch-Site"] = "same-origin"

    return headers


# ======================= RATE LIMITING =======================


def global_rate_limit():
    """Advanced global rate limiting with human-like patterns."""
    global request_counter

    with request_counter_lock:
        request_counter += 1
        current_count = request_counter

    # First URL: no delay
    if current_count == 1:
        return

    base_delay = random.uniform(MIN_REQUEST_SPACING, MAX_REQUEST_SPACING)

    # Burst cooldown every N requests
    if current_count > 0 and current_count % BURST_THRESHOLD == 0:
        burst_delay = random.uniform(BURST_COOLDOWN * 0.85, BURST_COOLDOWN * 1.15)
        base_delay += burst_delay

    # Random extended pauses (30% chance)
    if random.random() < 0.3:
        base_delay += random.uniform(3.0, 10.0)

    time.sleep(base_delay)


def throttle_domain(domain: str, delay: float):
    """Per-domain throttling with jitter."""
    with crawl_delay_lock:
        cd = crawl_delay_cache.get(domain, 0)

    effective_delay = max(delay, cd) if cd else delay
    jitter = random.uniform(-0.5, 3.0)
    effective_delay = max(4.0, effective_delay + jitter)

    with domain_throttle_lock:
        last_request = domain_throttle.get(domain, 0)
        now = time.time()
        elapsed = now - last_request

        if elapsed < effective_delay:
            sleep_time = effective_delay - elapsed
            time.sleep(sleep_time)

        domain_throttle[domain] = time.time()


def wait_for_domain_cooldown(domain: str):
    """Wait if domain is in cooldown."""
    with domain_failure_lock:
        until = domain_cooldowns.get(domain)
    if not until:
        return

    now = time.time()
    if now >= until:
        with domain_failure_lock:
            domain_cooldowns.pop(domain, None)
        return

    sleep_for = until - now + random.uniform(5.0, 10.0)
    if sleep_for > 0:
        time.sleep(sleep_for)


def record_domain_failure(domain: str, code: str):
    """Track failures and implement progressive cooldown."""
    with domain_failure_lock:
        count = domain_failure_counts.get(domain, 0) + 1
        domain_failure_counts[domain] = count

        if code in ("403", "429", "503", "cf_challenge") and count >= 2:
            cooldown = random.uniform(2400, 4200)  # 40-70 minutes
            domain_cooldowns[domain] = time.time() + cooldown


def reset_domain_failure(domain: str):
    """Reset failure count on success."""
    with domain_failure_lock:
        domain_failure_counts.pop(domain, None)


# ======================= URL UTILITIES =======================


def normalize_url(url: str) -> str:
    """Normalize URL for deduplication."""
    try:
        parsed = urlparse(url)
        parsed = parsed._replace(fragment="")

        if parsed.query:
            query_params = parse_qs(parsed.query, keep_blank_values=True)
            sorted_params = sorted(query_params.items())
            new_query = "&".join(f"{k}={v[0]}" for k, v in sorted_params)
            parsed = parsed._replace(query=new_query)

        return urlunparse(parsed)
    except Exception:
        return url


def strip_tracking_params(url: str) -> str:
    """Remove tracking parameters."""
    try:
        parsed = urlparse(url)
        if not parsed.query:
            return url

        query_params = parse_qs(parsed.query, keep_blank_values=True)
        filtered = {k: v for k, v in query_params.items() if k.lower() not in PARAMS_TO_STRIP}
        new_query = "&".join(f"{k}={v[0]}" for k, v in filtered.items())
        parsed = parsed._replace(query=new_query)
        return urlunparse(parsed)
    except Exception:
        return url


def validate_url(raw_url: str, allow_http: bool = False) -> Tuple[bool, str]:
    """Validate and clean URL."""
    url = raw_url.strip()
    if not url or url.startswith("#"):
        return False, url

    if url.startswith("//"):
        url = "https:" + url
    elif not url.startswith(("http://", "https://")):
        url = "https://" + url

    if not allow_http and url.startswith("http://"):
        url = url.replace("http://", "https://", 1)

    try:
        parsed = urlparse(url)
        if not parsed.scheme or not parsed.netloc:
            return False, url
        return True, url
    except Exception:
        return False, url


# ======================= ROBOTS.TXT =======================


def _parse_crawl_delay(host: str, robots_text: str):
    """Extract crawl-delay from robots.txt."""
    for line in robots_text.splitlines():
        low = line.strip().lower()
        if "crawl-delay" in low:
            m = re.search(r"crawl-delay\s*:\s*([0-9.]+)", low, re.I)
            if m:
                try:
                    delay = float(m.group(1))
                    with crawl_delay_lock:
                        crawl_delay_cache[host] = delay
                    return
                except Exception:
                    continue


def is_allowed_by_robots(url: str, session, user_agent: str = "Mozilla/5.0") -> Tuple[bool, str]:
    """Check robots.txt compliance."""
    try:
        parsed = urlparse(url)
        host = parsed.netloc
        robots_url = f"{parsed.scheme}://{host}/robots.txt"

        with robots_lock:
            parser = robots_cache.get(robots_url)

        if parser is None and robots_url not in robots_cache:
            parser_obj = robotexclusionrulesparser.RobotExclusionRulesParser()
            parser_obj.user_agent = user_agent

            try:
                headers = build_cloudflare_resistant_headers()
                r = session.get(robots_url, timeout=15, headers=headers)
                if r.status_code == 200:
                    text = r.text or ""
                    parser_obj.parse(text)
                    parser = parser_obj
                    _parse_crawl_delay(host, text)
                else:
                    parser = None
            except Exception:
                parser = None

            with robots_lock:
                robots_cache[robots_url] = parser

        if parser is None:
            return True, "No robots.txt"

        allowed = parser.is_allowed(user_agent, url)
        return allowed, ("Allowed" if allowed else "Blocked by robots.txt")
    except Exception:
        return True, "Robots check failed"


# ======================= META CHECKS =======================


def check_meta_robots(html: str) -> Tuple[bool, list]:
    """Check meta robots tags."""
    issues = []
    try:
        soup = BeautifulSoup(html, "html.parser")
        metas = soup.find_all("meta", attrs={"name": re.compile(r"robots|googlebot", re.I)})

        for meta in metas:
            content = (meta.get("content", "") or "").lower()
            if "noindex" in content or "none" in content:
                issues.append("Meta noindex detected")
                return True, issues

        return False, issues
    except Exception:
        return False, []


def check_canonical(html: str, final_url: str) -> Tuple[bool, Optional[str]]:
    """Check canonical tag."""
    try:
        soup = BeautifulSoup(html, "html.parser")
        canonical = soup.find("link", rel=re.compile(r"\bcanonical\b", re.I))

        if canonical and canonical.get("href"):
            canonical_url = urljoin(final_url, canonical["href"])
            normalized_final = normalize_url(final_url)
            normalized_canonical = normalize_url(canonical_url)
            return normalized_final == normalized_canonical, canonical_url

        return True, None
    except Exception:
        return True, None


# ======================= REQUEST WITH CLOUDFLARE BYPASS =======================


def get_with_retries(session, url: str, timeout: int, max_retries: int, impersonate: str = None):
    """Request with Cloudflare bypass via curl_cffi browser impersonation."""
    delay = 3.0
    last_exception = None

    for attempt in range(max_retries):
        try:
            start = time.time()
            eff_timeout = timeout * random.uniform(0.95, 1.05)

            parsed = urlparse(url)
            referer = None
            if parsed.path.count("/") >= 2:
                referer = f"{parsed.scheme}://{parsed.netloc}/"

            headers = build_cloudflare_resistant_headers(referer=referer)

            if CURL_CFFI_AVAILABLE:
                r = session.get(
                    url,
                    timeout=eff_timeout,
                    headers=headers,
                    allow_redirects=True,
                    verify=False,
                )
            else:
                r = session.get(
                    url,
                    timeout=eff_timeout,
                    headers=headers,
                    allow_redirects=True,
                    verify=False,
                )

            response_time = int((time.time() - start) * 1000)
            return r, response_time

        except Exception as e:
            last_exception = e
            if attempt < max_retries - 1:
                time.sleep(delay)
                delay = min(delay * 2, 45)
                continue

    raise last_exception


# ======================= INDEXABILITY CHECK =======================


def check_indexability(
    url: str,
    timeout: int,
    max_retries: int,
    no_robots: bool,
    args,
    stats: Stats,
    stats_lock: Lock,
) -> IndexabilityResult:
    """Check URL indexability with Cloudflare resistance."""

    result = IndexabilityResult(url=url, indexable=False)
    warnings_list = []

    impersonate = random.choice(BROWSER_PROFILES) if CURL_CFFI_AVAILABLE else None
    session = get_session(impersonate)

    if impersonate:
        result.tls_fingerprint = impersonate

    if args.strip_params:
        url = strip_tracking_params(url)

    parsed = urlparse(url)
    domain = parsed.netloc

    wait_for_domain_cooldown(domain)
    throttle_domain(domain, args.domain_delay)

    with stats_lock:
        stats.current_action = "Fetching..."

    try:
        r, response_time = get_with_retries(session, url, timeout, max_retries, impersonate)
        result.response_time_ms = response_time
    except Exception as e:
        err_str = str(e).lower()
        if "timeout" in err_str or "timed out" in err_str:
            record_domain_failure(domain, "timeout")
            result.reason = f"Timeout after {timeout}s"
        elif "connection" in err_str or "connect" in err_str:
            record_domain_failure(domain, "connection")
            result.reason = "Connection failed"
        else:
            result.reason = f"Request error: {e.__class__.__name__}"
        return result

    result.http_status = r.status_code
    result.final_url = r.url
    result.content_type = r.headers.get("Content-Type", "")
    result.redirect_chain = len(r.history) if hasattr(r, "history") else 0

    with stats_lock:
        stats.current_action = "Analyzing..."

    try:
        html_text = r.text or ""
        result.content_length = len(html_text)

        is_cf, cf_type = detect_cloudflare(dict(r.headers), html_text[:5000])
        result.cloudflare_detected = is_cf

        if is_cf:
            with stats_lock:
                stats.cloudflare_sites += 1

            with cloudflare_lock:
                cloudflare_detected[domain] = True

            if "challenge" in cf_type.lower() or "js challenge" in cf_type.lower():
                warnings_list.append(f"CF: {cf_type}")

                if r.status_code == 200:
                    warnings_list.append("CF challenge bypassed")
                elif r.status_code == 403:
                    record_domain_failure(domain, "cf_challenge")
                    result.indexable = True
                    result.reason = "CF 403 (treated as indexable)"
                    result.warning_flags = "; ".join(warnings_list)
                    return result
    except Exception:
        html_text = ""
        result.content_length = 0

    if r.status_code in (403, 429, 503):
        record_domain_failure(domain, str(r.status_code))
    elif 200 <= r.status_code < 300:
        reset_domain_failure(domain)

    # Treat 403 as indexable (likely just blocking crawlers, but page exists)
    if r.status_code == 403:
        result.indexable = True
        result.reason = "HTTP 403 (treated as indexable)"
        warnings_list.append("403 - possible WAF block")
        result.warning_flags = "; ".join(warnings_list)
        return result

    if r.status_code >= 400:
        result.reason = f"HTTP {r.status_code}"
        return result

    if not (200 <= r.status_code < 300):
        result.reason = f"HTTP {r.status_code}"
        return result

    xrobots = (r.headers.get("X-Robots-Tag", "") or "").lower()
    if "noindex" in xrobots or "none" in xrobots:
        result.x_robots_noindex = True
        result.reason = f"X-Robots-Tag: {xrobots}"
        return result

    if html_text and "text/html" in result.content_type.lower():
        has_noindex, meta_issues = check_meta_robots(html_text)
        result.meta_noindex = has_noindex

        if has_noindex:
            result.reason = "; ".join(meta_issues)
            return result

        is_self_canonical, canonical_url = check_canonical(html_text, r.url)
        result.canonical_url = canonical_url
        result.canonical_matches = is_self_canonical

        if canonical_url and not is_self_canonical:
            warnings_list.append("Non-self-canonical")
    else:
        if result.content_type and "text/html" not in result.content_type.lower():
            warnings_list.append(f"Non-HTML: {result.content_type.split(';')[0]}")

    if not no_robots:
        allowed, robots_note = is_allowed_by_robots(r.url, session)
        result.robots_allowed = allowed

        if not allowed:
            result.reason = "Blocked by robots.txt"
            return result

    result.indexable = True
    result.reason = "✓ Indexable"

    if warnings_list:
        result.warning_flags = "; ".join(warnings_list)

    return result


# ======================= ARGUMENT PARSING =======================


def parse_args():
    parser = argparse.ArgumentParser(
        description="Cloudflare-Resistant Indexability Checker"
    )
    parser.add_argument("--urls", default="urls.txt", help="Input file with URLs")
    parser.add_argument("--out", help="Output indexable URLs (TXT)")
    parser.add_argument("--csv", help="Output CSV with all results")
    parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Request timeout")
    parser.add_argument("--retries", type=int, default=DEFAULT_MAX_RETRIES, help="Max retries")
    parser.add_argument("--domain-delay", type=float, default=DEFAULT_DOMAIN_DELAY, help="Per-domain delay")
    parser.add_argument("--no-robots", action="store_true", help="Skip robots.txt check")
    parser.add_argument("--allow-http", action="store_true", help="Allow HTTP URLs")
    parser.add_argument("--strip-params", action="store_true", help="Strip tracking parameters")

    return parser.parse_args()


# ======================= UI HELPERS =======================


def _progress_bar(pct: float, width: int = 30) -> str:
    """Generate progress bar."""
    pct = max(0.0, min(100.0, pct))
    filled = int((pct / 100.0) * width)
    return "█" * filled + "░" * (width - filled)


def _format_time(seconds: int) -> str:
    """Format seconds as human-readable time."""
    h = seconds // 3600
    m = (seconds % 3600) // 60
    s = seconds % 60
    if h > 0:
        return f"{h}h {m}m {s}s"
    elif m > 0:
        return f"{m}m {s}s"
    return f"{s}s"


def render_ui(stats: Stats, args) -> Layout:
    """Main Rich layout for interactive terminals."""
    layout = Layout()
    layout.split_column(
        Layout(name="header", size=4),
        Layout(name="body", ratio=1),
        Layout(name="footer", size=1),
    )

    # ----- Header ----- #
    header_text = Text(justify="center")
    header_text.append("INDEXABILITY SCANNER BHW\n", style="bold cyan")
    header_text.append("Cloudflare Resistant Edition", style="dim cyan")

    layout["header"].update(
        Panel(
            header_text,
            border_style="cyan",
            box=box.DOUBLE,
            padding=(0, 2),
        )
    )

    # ----- Body ----- #
    layout["body"].split_row(
        Layout(name="main", ratio=3),
        Layout(name="side", ratio=2),
    )

    # Stats panel (left)
    progress_pct = (stats.processed / stats.total * 100) if stats.total > 0 else 0
    stats_table = Table(box=box.MINIMAL, show_header=False, pad_edge=False, padding=0)
    stats_table.add_column(style="cyan bold", justify="right", width=12, no_wrap=True)
    stats_table.add_column(style="white", justify="left", no_wrap=True)

    bar = _progress_bar(progress_pct, width=30)
    rate = stats.urls_per_hour()
 
    if rate == 0:
        rate_color = "dim"
        rate_text = "starting..."
    elif 200 <= rate <= 280:
        rate_color = "green"
        rate_text = f"{rate:.1f} URLs/hr"
    else:
        rate_color = "yellow"
        rate_text = f"{rate:.1f} URLs/hr"

    elapsed = time.time() - stats.start_time
    elapsed_str = _format_time(int(elapsed))

    stats_table.add_row("Runtime", elapsed_str)
    stats_table.add_row("Progress", f"{stats.processed}/{stats.total}")
    stats_table.add_row("", f"{bar} {progress_pct:.1f}%")
    stats_table.add_row("Indexable", f"[green]{stats.indexable}[/green]")
    stats_table.add_row("Blocked", f"[red]{stats.blocked}[/red]")
    stats_table.add_row("Warnings", f"[yellow]{stats.warnings}[/yellow]")
    stats_table.add_row("Cloudflare", f"[cyan]{stats.cloudflare_sites}[/cyan]")
    stats_table.add_row("Rate", f"[{rate_color}]{rate_text}[/{rate_color}]")
    eta = stats.eta_seconds()
    stats_table.add_row("ETA", _format_time(eta) if eta > 0 else "calculating...")

    layout["body"]["main"].update(
        Panel(
            stats_table,
            title="Statistics",
            border_style="green",
            box=box.ROUNDED,
            padding=(1, 2),
        )
    )

    # Activity panel (right)
    current_url_display = (
        stats.current_url[:50] + "..." if len(stats.current_url) > 50 else stats.current_url
    )

    activity = Text()
    activity.append("Current URL\n", style="bold cyan")
    if current_url_display:
        activity.append(current_url_display + "\n", style="yellow")
    else:
        activity.append("—\n", style="dim")
    activity.append(f"{stats.current_action or 'Idle'}\n\n", style="dim")

    activity.append("Recent Activity\n", style="bold cyan")
    if stats.recent_events:
        for event in stats.recent_events[-6:]:
            activity.append_text(Text.from_markup(event + "\n"))
    else:
        activity.append("No activity yet\n", style="dim")

    layout["body"]["side"].update(
        Panel(
            activity,
            title="Live Activity",
            border_style="yellow",
            box=box.ROUNDED,
            padding=(1, 2),
        )
    )

    # ----- Footer ----- #
    footer_text = Text("", justify="center", style="dim")
    if CURL_CFFI_AVAILABLE:
        footer_text.append("TLS Bypass: Active  |  ", style="dim green")
    else:
        footer_text.append("TLS Bypass: Disabled  |  ", style="dim yellow")
    footer_text.append(f"Target: {TARGET_URLS_PER_HOUR} URLs/hr", style="dim")

    layout["footer"].update(footer_text)

    return layout


# ======================= MAIN =======================


def main():
    args = parse_args()

    input_base = os.path.splitext(args.urls)[0]
    if not args.out:
        args.out = f"{input_base}-indexable.txt"
    if not args.csv:
        args.csv = f"{input_base}-results.csv"

    console.clear()

    use_live = console.is_terminal

    if not CURL_CFFI_AVAILABLE:
        console.print()
        console.print(
            Panel(
                "[bold red]⚠️  WARNING: curl_cffi NOT INSTALLED[/bold red]\n\n"
                "Install with: pip install curl-cffi\n\n"
                "Continuing with limited effectiveness...",
                border_style="red",
                box=box.ROUNDED,
            )
        )
        time.sleep(2)

    if not use_live:
        console.print()
        console.print(
            Panel.fit(
                "[bold cyan]️  INDEXABILITY SCANNER  ️[/bold cyan]\n"
                "[dim cyan]BHW Edition - Cloudflare Resistant[/dim cyan]\n\n"
                f"TLS Bypass: {'✓ ACTIVE' if CURL_CFFI_AVAILABLE else '⚠ DISABLED'}\n"
                f"Stealth mode: {TARGET_URLS_PER_HOUR} URLs/hour target\n\n"
                "[dim]Slow, human-like crawling to avoid WAF / bot detection[/dim]",
                border_style="cyan",
                box=box.ROUNDED,
            )
        )

    if not os.path.exists(args.urls):
        console.print(f"[red]❌ File not found:[/red] {args.urls}")
        return

    with open(args.urls, "r", encoding="utf-8") as f:
        raw_urls = [l.strip() for l in f if l.strip() and not l.startswith("#")]

    urls = []
    for raw in raw_urls:
        valid, clean = validate_url(raw, args.allow_http)
        if valid:
            urls.append(clean)

    # Deduplicate normalized URLs
    seen = set()
    unique_urls = []
    for url in urls:
        normalized = normalize_url(url)
        if normalized not in seen:
            seen.add(normalized)
            unique_urls.append(url)
    urls = unique_urls

    random.shuffle(urls)

    console.print(f"[green]✓[/green] Loaded {len(urls)} unique URLs")
    console.print(f"[cyan]ETA:[/cyan] ~{len(urls) / TARGET_URLS_PER_HOUR:.1f} hours\n")
    time.sleep(1)

    stats = Stats(total=len(urls))
    results: List[IndexabilityResult] = []
    stats_lock = Lock()

    def process_url(url: str) -> IndexabilityResult:
        global_rate_limit()

        with stats_lock:
            stats.current_url = url

        result = check_indexability(
            url, args.timeout, args.retries, args.no_robots, args, stats, stats_lock
        )

        with stats_lock:
            stats.processed += 1

            short_url = url[:40] + "..." if len(url) > 40 else url

            if result.indexable:
                stats.indexable += 1
                if result.warning_flags:
                    stats.warnings += 1
                    stats.recent_events.append(f"[yellow]⚠[/yellow] {short_url}")
                else:
                    stats.recent_events.append(f"[green]✓[/green] {short_url}")
            else:
                stats.blocked += 1
                stats.failure_reasons[result.reason] += 1
                stats.recent_events.append(f"[red]✗[/red] {short_url}")

            if len(stats.recent_events) > 14:
                stats.recent_events.pop(0)

        return result

    if use_live:
        with Live(
            render_ui(stats, args),
            refresh_per_second=1,
            console=console,
            screen=True,
            transient=True,
        ) as live:
            for url in urls:
                result = process_url(url)
                results.append(result)
                live.update(render_ui(stats, args))
    else:
        console.print(
            "[yellow]ℹ Non-interactive output detected. Using simple progress log.[/yellow]\n"
        )
        for url in urls:
            result = process_url(url)
            results.append(result)
            if stats.processed % 25 == 0 or stats.processed == stats.total:
                pct = (stats.processed / stats.total * 100) if stats.total else 0
                console.print(
                    f"[cyan]{stats.processed}/{stats.total} ({pct:.1f}%)[/cyan] "
                    f"[{'green' if result.indexable else 'red'}]{'✓' if result.indexable else '✗'}[/] "
                    f"{result.reason}"
                )

    # ======================= OUTPUT FILES =======================

    indexable_urls = [r.url for r in results if r.indexable]

    with open(args.out, "w", encoding="utf-8") as f:
        for url in indexable_urls:
            f.write(url + "\n")

    csv_rows = []
    for r in results:
        csv_rows.append(
            {
                "url": r.url,
                "indexable": "YES" if r.indexable else "NO",
                "status": r.http_status or "",
                "reason": r.reason,
                "final_url": r.final_url or "",
                "cloudflare": "YES" if r.cloudflare_detected else "NO",
                "tls_fingerprint": r.tls_fingerprint or "",
                "content_type": r.content_type or "",
                "robots_allowed": r.robots_allowed if r.robots_allowed is not None else "",
                "canonical_url": r.canonical_url or "",
                "meta_noindex": r.meta_noindex if r.meta_noindex is not None else "",
                "response_time_ms": r.response_time_ms or "",
                "warnings": r.warning_flags or "",
            }
        )

    with open(args.csv, "w", encoding="utf-8", newline="") as f:
        if csv_rows:
            writer = csv.DictWriter(f, fieldnames=csv_rows[0].keys())
            writer.writeheader()
            writer.writerows(csv_rows)

    # ======================= SUMMARY PANEL =======================

    warning_count = sum(1 for r in results if r.indexable and r.warning_flags)
    cf_count = sum(1 for r in results if r.cloudflare_detected)

    summary = Table(show_header=False, box=box.ROUNDED, padding=(0, 2))
    summary.add_column(style="cyan bold", justify="right")
    summary.add_column(style="white bold")

    summary.add_row("✅ Indexable", f"[green]{len(indexable_urls)}[/green]")
    summary.add_row("⚠️  Warnings", f"[yellow]{warning_count}[/yellow]")
    summary.add_row("☁️  Cloudflare", f"[cyan]{cf_count}[/cyan]")
    summary.add_row("❌ Blocked", f"[red]{len(results) - len(indexable_urls)}[/red]")
    summary.add_row(" Total", f"[white]{len(results)}[/white]")

    elapsed = time.time() - stats.start_time
    rate = stats.urls_per_hour()

    summary.add_row("⏱️  Time", f"[cyan]{_format_time(int(elapsed))}[/cyan]")
    summary.add_row("⚡ Rate", f"[cyan]{rate:.1f}[/cyan] URLs/hour")
    summary.add_row(" TXT", f"[dim]{args.out}[/dim]")
    summary.add_row(" CSV", f"[dim]{args.csv}[/dim]")

    console.print("\n" + "=" * 70)
    console.print(
        Panel(
            summary,
            title="[bold green]✅ COMPLETED[/bold green]",
            border_style="green",
            box=box.DOUBLE,
        )
    )

    if cf_count > 0:
        console.print(f"\n[cyan]ℹ️  Cloudflare detected on {cf_count} sites[/cyan]")
        if CURL_CFFI_AVAILABLE:
            console.print("[green]✓[/green] TLS bypass was active")
        else:
            console.print("[yellow]⚠[/yellow] Install curl-cffi for better results")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        console.print("\n[red]⚠️  Interrupted[/red]")
    except Exception as e:
        console.print(f"\n[red]❌ Error: {e}[/red]")
        raise