#!/usr/bin/env python3
"""
ffuf_automation.py — Professional ffuf Web Directory Discovery Automation Tool
Author: Hackers
Purpose: Automate ffuf scanning across multiple authorized domains with structured reporting.

⚠️  LEGAL WARNING — READ BEFORE USE ⚠️
This tool is designed exclusively for:
  - Domains you own and control
  - Domains for which you have written/explicit authorization to test
Unauthorized use against third-party systems is illegal under the Computer Fraud
and Abuse Act (CFAA), the Computer Misuse Act (UK), and equivalent laws worldwide.
The author assumes zero liability for misuse. Use responsibly and ethically.
"""

import argparse
import csv
import json
import logging
import os
import pty
import re
import select
import signal
import subprocess
import sys
import threading
import time
from datetime import datetime
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse

try:
    import tty
    import termios
    HAS_TTY = True
except ImportError:
    HAS_TTY = False

# ─────────────────────────────────────────────
# Optional third-party imports (report extras)
# ─────────────────────────────────────────────
try:
    from jinja2 import Environment, BaseLoader
    HAS_JINJA2 = True
except ImportError:
    HAS_JINJA2 = False

# ─────────────────────────────────────────────
# Constants
# ─────────────────────────────────────────────
INTERESTING_PATHS = [
    "/api", "/admin", "/login", "/dashboard", "/backup",
    "/config", "/core", "/uploads", "/dev", "/test",
    "/.env", "/.git", "/wp-admin", "/phpmyadmin",
    "/swagger", "/graphql", "/internal", "/debug", "/secret",
]

HIGHLIGHT_CODES = {200, 301, 302, 401, 403, 500}

VERSION = "1.0.0"
TOOL_NAME = "ffuf-automation"

# Parses ffuf progress line: ":: Progress: [...] :: 245 req/sec :: ... :: Errors: 3 ::"
_STATS_RE = re.compile(r'(\d+)\s+req/sec.*?Errors:\s*(\d+)', re.IGNORECASE | re.DOTALL)

# ─────────────────────────────────────────────
# Stats-aware console handler
# ─────────────────────────────────────────────
class StatsAwareHandler(logging.StreamHandler):
    """Clears the live stats line before writing a log record."""
    def __init__(self, stream=None, verbose: bool = False):
        super().__init__(stream)
        self.verbose = verbose

    def emit(self, record):
        if not self.verbose:
            sys.stdout.write('\r\033[2K')   # erase the \r stats line
            sys.stdout.flush()
        super().emit(record)


# ─────────────────────────────────────────────
# Logging Setup
# ─────────────────────────────────────────────
def setup_logging(output_dir: str, verbose: bool = False) -> logging.Logger:
    """Configure dual-output logger (stdout + file)."""
    log_file = os.path.join(output_dir, "logs", f"scan_{datetime.now():%Y%m%d_%H%M%S}.log")
    os.makedirs(os.path.dirname(log_file), exist_ok=True)

    logger = logging.getLogger(TOOL_NAME)
    logger.setLevel(logging.DEBUG)

    fmt = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S")

    # Console handler — clears stats line before each message (skipped in verbose mode)
    ch = StatsAwareHandler(sys.stdout, verbose=verbose)
    ch.setLevel(logging.INFO)
    ch.setFormatter(fmt)

    # File handler — DEBUG and above
    fh = logging.FileHandler(log_file)
    fh.setLevel(logging.DEBUG)
    fh.setFormatter(fmt)

    logger.addHandler(ch)
    logger.addHandler(fh)
    return logger


# ─────────────────────────────────────────────
# Pre-flight Checks
# ─────────────────────────────────────────────
def check_ffuf_installed() -> str:
    """Verify ffuf is installed and return its path."""
    try:
        result = subprocess.run(["which", "ffuf"], capture_output=True, text=True)
        if result.returncode == 0:
            path = result.stdout.strip()
            # Also get version
            ver = subprocess.run(["ffuf", "-V"], capture_output=True, text=True)
            version_str = ver.stdout.strip() or ver.stderr.strip()
            return path, version_str
        else:
            return None, None
    except FileNotFoundError:
        return None, None


def print_banner():
    """Print the tool banner."""
    banner = f"""
\033[1;36m
╔══════════════════════════════════════════════════════════════════════╗
║          ffuf-automation v{VERSION} — Web Discovery Toolkit           ║                  ║
╚══════════════════════════════════════════════════════════════════════╝
\033[0m
\033[1;33m⚠️  AUTHORIZED USE ONLY — You must own or have written permission    ⚠️
⚠️  to test every domain in your input file. Unauthorized scanning    ⚠️
⚠️  is illegal. The author bears no responsibility for misuse.        ⚠️\033[0m
"""
    print(banner)


def confirm_scan(domains: list[str]) -> bool:
    """Ask the user to explicitly confirm before scanning."""
    print(f"\n\033[1;34m[SCOPE] The following {len(domains)} domain(s) will be scanned:\033[0m")
    for d in domains:
        print(f"  → {d}")
    print()
    answer = input("\033[1;31mDo you confirm you own or have explicit permission to test ALL listed domains? [yes/no]: \033[0m").strip().lower()
    return answer == "yes"


# ─────────────────────────────────────────────
# Domain Normalization
# ─────────────────────────────────────────────
def normalize_domain(raw: str, scheme: str = "https") -> Optional[str]:
    """
    Normalize a raw domain string:
    - Strip whitespace / comments
    - Apply chosen scheme: http, https, or both
    - Remove trailing slashes
    """
    raw = raw.strip()
    if not raw or raw.startswith("#"):
        return None
    # Strip any existing scheme so we can re-apply the user choice
    if raw.startswith("http://"):
        raw = raw[7:]
    elif raw.startswith("https://"):
        raw = raw[8:]
    return f"{scheme}://{raw.rstrip('/')}"


def load_domains(filepath: str, scheme: str = "https") -> list[str]:
    """Read and normalize domains from a file, applying chosen scheme."""
    if not os.path.isfile(filepath):
        raise FileNotFoundError(f"Domains file not found: {filepath}")
    domains = []
    seen = set()
    with open(filepath) as f:
        for line in f:
            if scheme == "both":
                # Generate http:// AND https:// variant for every domain
                for s in ("http", "https"):
                    domain = normalize_domain(line, s)
                    if domain and domain not in seen:
                        domains.append(domain)
                        seen.add(domain)
            else:
                domain = normalize_domain(line, scheme)
                if domain and domain not in seen:
                    domains.append(domain)
                    seen.add(domain)
    return domains


def safe_filename(url: str) -> str:
    """Convert a URL into a filesystem-safe name."""
    parsed = urlparse(url)
    name = parsed.netloc + parsed.path
    return re.sub(r"[^\w\-.]", "_", name).strip("_")


# ─────────────────────────────────────────────
# ffuf Command Builder
# ─────────────────────────────────────────────
def build_ffuf_command(
    domain: str,
    wordlist: str,
    output_file: str,
    args: argparse.Namespace,
    use_ffuf_recursion: bool = True,
) -> list[str]:
    """Construct the ffuf command from user arguments."""
    cmd = [
        "ffuf",
        "-w", f"{wordlist}:FUZZ",
        "-u", f"{domain}/FUZZ",
        "-of", "json",
        "-o", output_file,
        "-t", str(args.threads),
        "-timeout", str(args.timeout),
        "-noninteractive",   # suppress interactive prompts
    ]

    # Recursion — disabled when Python-level force recursion handles it
    if use_ffuf_recursion and args.depth > 0:
        cmd += ["-recursion", "-recursion-depth", str(args.depth)]

    # Extensions
    if args.extensions:
        cmd += ["-e", args.extensions]

    # Status code match
    if args.match_codes:
        cmd += ["-mc", args.match_codes]

    # Status code filter
    if args.filter_codes:
        cmd += ["-fc", args.filter_codes]

    # Word count filter
    if args.filter_words:
        cmd += ["-fw", args.filter_words]

    # Size filter
    if args.filter_sizes:
        cmd += ["-fs", args.filter_sizes]

    # Line filter
    if args.filter_lines:
        cmd += ["-fl", args.filter_lines]

    # Rate limit / delay
    if args.rate:
        cmd += ["-rate", str(args.rate)]

    # Max results per domain
    if args.max_results:
        cmd += ["-maxtime-job", "0"]  # unlimited time but we'll cap in parsing

    return cmd


# ─────────────────────────────────────────────
# ffuf Runner
# ─────────────────────────────────────────────
# ─────────────────────────────────────────────
# Pause / Resume Controller
# ─────────────────────────────────────────────
class PauseController:
    """
    Sends SIGSTOP / SIGCONT to the ffuf subprocess.
    A background thread reads raw keypresses:
      p  →  pause       r  →  resume
      q  →  stop + save report    Ctrl+C  →  same as q
    """

    _CONTROLS = (
        "\033[2m  Controls: \033[0m"
        "\033[1;36m[p]\033[0m pause  "
        "\033[1;32m[r]\033[0m resume  "
        "\033[1;31m[q]\033[0m stop & save"
    )

    def __init__(self, proc: subprocess.Popen, logger: logging.Logger,
                 verbose: bool = False, total_requests: int = 0):
        self.proc           = proc
        self.logger         = logger
        self.verbose        = verbose
        self.total_requests = total_requests
        self.paused         = False
        self.done           = threading.Event()
        self._old_tc        = None
        self.req_sec        = 0
        self.errors         = 0
        self.done_count     = 0
        self._stats_lock    = threading.Lock()
        # Anchored on the known total so it matches regardless of surrounding
        # format (":: 12345/87000 ::" or "Progress: [12345/87000]" etc.)
        self._done_re = (
            re.compile(rf'(\d+)\s*/\s*{total_requests}(?!\d)')
            if total_requests > 0 else None
        )

    # ── public controls ──────────────────────────────────
    def pause(self):
        if self.paused or self.proc.poll() is not None:
            return
        try:
            self.proc.send_signal(signal.SIGSTOP)
            self.paused = True
        except OSError:
            return
        print(
            "\n\033[1;33m"
            "  ┌──────────────────────────────────────┐\n"
            "  │  ⏸  SCAN PAUSED                       │\n"
            "  │  [r] resume  │  [q] stop & save report│\n"
            "  └──────────────────────────────────────┘"
            "\033[0m", flush=True
        )

    def resume(self):
        if not self.paused or self.proc.poll() is not None:
            return
        try:
            self.proc.send_signal(signal.SIGCONT)
            self.paused = False
        except OSError:
            return
        print(
            "\n\033[1;32m  ▶  RESUMED — scan continuing...\033[0m",
            flush=True
        )
        print(self._CONTROLS, flush=True)

    def stop(self):
        """Resume if paused, then interrupt ffuf."""
        self.done.set()
        if self.paused:
            try:
                self.proc.send_signal(signal.SIGCONT)
            except OSError:
                pass
        try:
            self.proc.send_signal(signal.SIGINT)
        except OSError:
            pass

    # ── ffuf stdout stats parser (background thread) ─────
    def read_stats(self, fd: int):
        """Read from PTY master fd, parse req/sec + errors, and forward to stdout in verbose mode."""
        _ansi = re.compile(rb'\x1b\[[0-9;]*[a-zA-Z]|\r')
        buf = b""
        while not self.done.is_set():
            try:
                r, _, _ = select.select([fd], [], [], 0.2)
                if not r:
                    continue
                chunk = os.read(fd, 512)
                if not chunk:
                    break
                # In verbose mode, stream ffuf's raw output straight to the terminal.
                # flush the text-mode buffer first so bytes aren't reordered.
                if self.verbose:
                    sys.stdout.flush()
                    sys.stdout.buffer.write(chunk)
                    sys.stdout.buffer.flush()
                buf = (buf + chunk)[-4096:]
                clean = _ansi.sub(b" ", buf)
                text  = clean.decode("utf-8", errors="replace")
                m = _STATS_RE.search(text)
                if m:
                    with self._stats_lock:
                        self.req_sec = int(m.group(1))
                        self.errors  = int(m.group(2))
                # Take the rightmost (most recent) progress value from the buffer.
                if self._done_re:
                    done_matches = self._done_re.findall(text)
                    if done_matches:
                        with self._stats_lock:
                            self.done_count = int(done_matches[-1])
            except OSError:
                break
            except Exception:
                break

    def _print_stats(self):
        """Write live stats line using \\r (overwrites itself each tick)."""
        with self._stats_lock:
            rps   = self.req_sec
            err   = self.errors
            done  = self.done_count
            total = self.total_requests
        err_col = "31" if err else "32"
        if total > 0:
            left    = max(0, total - done)
            prog_str = (
                f"  \033[2m│  progress:\033[0m "
                f"\033[1;35m{done}/{total}\033[0m "
                f"\033[2m({left} left)\033[0m"
            )
        else:
            prog_str = ""
        sys.stdout.write(
            f"\r  \033[2m↳\033[0m  "
            f"\033[2mreq/sec:\033[0m \033[1;36m{rps:<6}\033[0m  "
            f"\033[2m│  errors:\033[0m \033[1;{err_col}m{err}\033[0m"
            f"{prog_str}"
            f"          "   # trailing spaces erase previous longer content
        )
        sys.stdout.flush()

    # ── raw keyboard listener (background thread) ────────
    def listen(self):
        if not HAS_TTY or not sys.stdin.isatty():
            return

        fd = sys.stdin.fileno()
        try:
            self._old_tc = termios.tcgetattr(fd)
            tty.setraw(fd)
        except termios.error:
            return

        try:
            while not self.done.is_set():
                r, _, _ = select.select([sys.stdin], [], [], 0.15)
                # In non-verbose mode, overwrite the current line with live stats.
                # In verbose mode ffuf's own output fills the terminal, so skip it.
                if not self.paused and not self.verbose:
                    self._print_stats()
                if not r:
                    continue
                key = sys.stdin.read(1)
                if key in ('p', 'P'):
                    self.pause()
                elif key in ('r', 'R'):
                    self.resume()
                elif key in ('q', 'Q', '\x03'):      # q or Ctrl+C
                    self.stop()
                    # Raise KeyboardInterrupt in the main thread
                    os.kill(os.getpid(), signal.SIGINT)
                    break
        except Exception:
            pass
        finally:
            self._restore_terminal(fd)

    def _restore_terminal(self, fd: int):
        if self._old_tc is not None:
            try:
                termios.tcsetattr(fd, termios.TCSADRAIN, self._old_tc)
            except Exception:
                pass
            self._old_tc = None


# ─────────────────────────────────────────────
# ffuf Runner
# ─────────────────────────────────────────────
def run_ffuf(
    cmd: list[str],
    domain: str,
    logger: logging.Logger,
    timeout_limit: int = 3600,
    verbose: bool = False,
) -> bool:
    """
    Execute ffuf as a subprocess with full pause/resume/stop support.
    Keyboard controls are active while ffuf runs:
      p = pause   r = resume   q / Ctrl+C = stop & save partial report
    When verbose=True, ffuf's full output is streamed to the terminal instead of
    the condensed req/sec stats line.
    Returns True on success, False on failure, raises KeyboardInterrupt on stop.
    """
    logger.info(f"[SCAN] Starting: {domain}")
    logger.debug(f"[CMD] {' '.join(cmd)}")

    # Pre-compute total expected requests for the progress counter.
    # Extract wordlist path from "-w /path/to/list.txt:FUZZ" and extensions from "-e .php,.txt".
    _wl_path, _ext_str = "", ""
    for _i, _part in enumerate(cmd):
        if _part == '-w' and _i + 1 < len(cmd):
            _wl_path = cmd[_i + 1].split(':')[0]
        elif _part == '-e' and _i + 1 < len(cmd):
            _ext_str = cmd[_i + 1]
    _word_count = 0
    if _wl_path:
        try:
            with open(_wl_path) as _wl:
                _word_count = sum(
                    1 for _ln in _wl
                    if _ln.strip() and not _ln.strip().startswith('#')
                )
        except OSError:
            pass
    _ext_count    = len([_e for _e in _ext_str.split(',') if _e.strip()]) if _ext_str else 0
    total_requests = _word_count * (1 + _ext_count)

    try:
        # Open a PTY so ffuf thinks stdout is a real terminal and
        # outputs its live progress line (req/sec, errors, etc.)
        pty_master, pty_slave = pty.openpty()

        proc = subprocess.Popen(
            cmd,
            stdout=pty_slave,         # ffuf writes to PTY slave (thinks it's a TTY)
            stderr=subprocess.PIPE,   # binary — checked for error messages
        )
        os.close(pty_slave)           # only the child needs the slave end

        ctrl = PauseController(proc, logger, verbose=verbose, total_requests=total_requests)

        # Background thread: read PTY master for req/sec + errors
        stats_thread = threading.Thread(
            target=ctrl.read_stats, args=(pty_master,), daemon=True
        )
        stats_thread.start()

        # Background thread: drain stderr and parse stats.
        # ffuf v2 writes its progress bar (\r-overwritten line) to stderr,
        # not stdout, so req/sec + done/total must be read from here.
        stderr_buf: list[bytes] = []
        def _read_stderr():
            _se_ansi = re.compile(rb'\x1b\[[0-9;]*[a-zA-Z]|\r')
            buf = b""
            se_fd = proc.stderr.fileno()
            while not ctrl.done.is_set():
                try:
                    r, _, _ = select.select([se_fd], [], [], 0.2)
                    if not r:
                        continue
                    chunk = os.read(se_fd, 512)
                    if not chunk:
                        break
                    stderr_buf.append(chunk)
                    buf = (buf + chunk)[-4096:]
                    clean = _se_ansi.sub(b" ", buf)
                    text  = clean.decode("utf-8", errors="replace")
                    m = _STATS_RE.search(text)
                    if m:
                        with ctrl._stats_lock:
                            ctrl.req_sec = int(m.group(1))
                            ctrl.errors  = int(m.group(2))
                    if ctrl._done_re:
                        done_matches = ctrl._done_re.findall(text)
                        if done_matches:
                            with ctrl._stats_lock:
                                ctrl.done_count = int(done_matches[-1])
                except OSError:
                    break
        stderr_thread = threading.Thread(target=_read_stderr, daemon=True)
        stderr_thread.start()

        # Background thread: keyboard listener
        key_thread = threading.Thread(target=ctrl.listen, daemon=True)
        key_thread.start()

        print(PauseController._CONTROLS, flush=True)

        try:
            proc.wait(timeout=timeout_limit)
        except KeyboardInterrupt:
            ctrl.done.set()
            key_thread.join(timeout=2)          # restore terminal first
            logger.warning(f"[INTERRUPT] Waiting for ffuf to save results: {domain}")
            try:
                proc.wait(timeout=15)
            except subprocess.TimeoutExpired:
                proc.kill()
                proc.wait()
            raise
        except subprocess.TimeoutExpired:
            ctrl.done.set()
            key_thread.join(timeout=2)
            proc.kill()
            proc.wait()
            logger.error(f"[TIMEOUT] Scan timed out for {domain}")
            return False
        finally:
            ctrl.done.set()
            key_thread.join(timeout=2)
            # Close PTY master — unblocks read_stats thread
            try:
                os.close(pty_master)
            except OSError:
                pass
            # Clear the stats line so the next log message starts clean
            sys.stdout.write('\r\033[2K')
            sys.stdout.flush()

        stderr_thread.join(timeout=2)
        stats_thread.join(timeout=2)
        stderr = b"".join(stderr_buf).decode("utf-8", errors="replace")

        if proc.returncode == 0:
            logger.info(f"[DONE] Completed: {domain}")
            return True
        else:
            if "no results" in stderr.lower() or proc.returncode == 1:
                logger.warning(f"[WARN] ffuf exited {proc.returncode} for {domain}: {stderr.strip()[:200]}")
                return True
            logger.error(f"[FAIL] ffuf error for {domain}: {stderr.strip()[:300]}")
            return False

    except KeyboardInterrupt:
        raise
    except Exception as e:
        logger.error(f"[ERROR] Unexpected error scanning {domain}: {e}")
        return False


# ─────────────────────────────────────────────
# Force Recursion (custom Python-level)
# ─────────────────────────────────────────────
def load_wordlist_words(wordlist_path: str) -> list[str]:
    """Read words from a wordlist file, skipping blank lines and comments."""
    words = []
    try:
        with open(wordlist_path) as f:
            for line in f:
                w = line.strip()
                if w and not w.startswith("#"):
                    words.append(w)
    except IOError:
        pass
    return words


def run_force_recursive_scan(
    domain: str,
    args: argparse.Namespace,
    logger: logging.Logger,
    raw_dir: str,
) -> tuple:
    """
    Python-level recursion regardless of HTTP response codes (even 404).

    When --force-recursion-paths is given:
      depth 0 : domain/FUZZ           (full wordlist at root)
      depth 1 : domain/forcepath/FUZZ (full wordlist inside each forced path)
      depth 2+: domain/forcepath/word/FUZZ ... (full wordlist at every level)

    When no --force-recursion-paths:
      every depth uses all wordlist words as sub-path candidates.

    Returns (all_results, base_cmd_str, failed).
    """
    wordlist_words = load_wordlist_words(args.wordlist)

    recursion_paths_file = getattr(args, "force_recursion_paths", None)
    if recursion_paths_file:
        forced_paths = [w.lstrip("/") for w in load_wordlist_words(recursion_paths_file)]
        logger.info(f"[FORCE-RECURSION] {len(forced_paths)} forced entry path(s) from {recursion_paths_file}, "
                    f"then full wordlist ({len(wordlist_words)} words) at each deeper level")
    else:
        forced_paths = None
        logger.info(f"[FORCE-RECURSION] Using all {len(wordlist_words)} wordlist word(s) at every depth")

    all_results = []
    seen_urls: set = set()
    base_cmd_str = ""
    current_bases = [domain]
    interrupted = False

    for depth_level in range(args.depth + 1):
        if interrupted:
            break
        next_bases = []

        for base in current_bases:
            fname = safe_filename(base)
            json_out = os.path.join(raw_dir, f"{fname}.json")
            cmd = build_ffuf_command(base, args.wordlist, json_out, args, use_ffuf_recursion=False)

            if not base_cmd_str:
                base_cmd_str = " ".join(cmd)

            if os.path.isfile(json_out) and os.path.getsize(json_out) > 10:
                logger.info(f"[SKIP] Already scanned (depth={depth_level}): {base}")
            else:
                try:
                    success = run_ffuf(cmd, base, logger, verbose=getattr(args, 'verbose', False))
                except KeyboardInterrupt:
                    # Parse whatever ffuf wrote before it exited
                    for r in parse_ffuf_results(json_out, args.max_results):
                        if r["url"] not in seen_urls:
                            seen_urls.add(r["url"])
                            all_results.append(r)
                    logger.info(f"  → {len(all_results)} partial path(s) saved (cancelled)")
                    interrupted = True
                    break
                if not success:
                    if base == domain:
                        return all_results, base_cmd_str, True, False
                    logger.warning(f"[WARN] Sub-scan failed (depth={depth_level}): {base}")
                    continue

            for r in parse_ffuf_results(json_out, args.max_results):
                if r["url"] not in seen_urls:
                    seen_urls.add(r["url"])
                    all_results.append(r)

        if not interrupted and depth_level < args.depth:
            if forced_paths is not None and depth_level == 0:
                expansion = forced_paths
            else:
                expansion = wordlist_words

            for base in current_bases:
                for word in expansion:
                    next_bases.append(f"{base}/{word}")
            current_bases = next_bases
            logger.info(f"[FORCE-RECURSION] Depth {depth_level + 1}: {len(current_bases)} sub-path(s) queued")

    return all_results, base_cmd_str, False, interrupted


# ─────────────────────────────────────────────
# Result Parser
# ─────────────────────────────────────────────
def parse_ffuf_results(json_file: str, max_results: Optional[int] = None) -> list[dict]:
    """
    Parse ffuf JSON output into a list of result dicts.
    Returns empty list if file missing or malformed.
    """
    if not os.path.isfile(json_file):
        return []

    try:
        with open(json_file) as f:
            data = json.load(f)
    except (json.JSONDecodeError, IOError):
        return []

    results_raw = data.get("results", [])
    if max_results:
        results_raw = results_raw[:max_results]

    parsed = []
    seen_urls = set()

    for r in results_raw:
        url = r.get("url", "")
        if url in seen_urls:
            continue
        seen_urls.add(url)

        # Extract redirect location from headers if present
        redirect = None
        headers = r.get("headers", {}) or {}
        if isinstance(headers, dict):
            redirect = headers.get("location") or headers.get("Location")

        parsed.append({
            "url":        url,
            "path":       urlparse(url).path,
            "status":     r.get("status", 0),
            "length":     r.get("length", 0),
            "words":      r.get("words", 0),
            "lines":      r.get("lines", 0),
            "redirect":   redirect or "",
            "input":      r.get("input", {}).get("FUZZ", ""),
        })

    return parsed


def find_interesting(results: list[dict]) -> list[dict]:
    """Flag results whose paths match known interesting patterns."""
    interesting = []
    for r in results:
        path_lower = r["path"].lower()
        for pattern in INTERESTING_PATHS:
            if pattern in path_lower:
                r_copy = dict(r)
                r_copy["match_reason"] = pattern
                interesting.append(r_copy)
                break
    return interesting


def build_path_tree(results: list[dict]) -> dict:
    """Build a nested path tree from flat results for the HTML tree view."""
    tree = {}
    for r in sorted(results, key=lambda x: x["path"]):
        parts = [p for p in r["path"].split("/") if p]
        node = tree
        for i, part in enumerate(parts):
            if part not in node:
                node[part] = {"_item": None, "_children": {}}
            if i == len(parts) - 1:
                node[part]["_item"] = r
            node = node[part]["_children"]
    return tree


# ─────────────────────────────────────────────
# Report Generators
# ─────────────────────────────────────────────
def write_json_report(scan_data: dict, output_path: str):
    """Write a full JSON report."""
    with open(output_path, "w") as f:
        json.dump(scan_data, f, indent=2, default=str)


def write_csv_report(all_results: list[dict], output_path: str):
    """Write a flat CSV report of all discovered paths."""
    fieldnames = ["domain", "url", "path", "status", "length", "words", "lines", "redirect", "interesting"]
    with open(output_path, "w", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for row in all_results:
            writer.writerow({k: row.get(k, "") for k in fieldnames})


HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>ffuf-automation Report — {{ scan_date }}</title>
<style>
  :root {
    --bg: #0d1117; --surface: #161b22; --border: #30363d;
    --text: #e6edf3; --muted: #8b949e; --accent: #58a6ff;
    --green: #3fb950; --yellow: #d29922; --red: #f85149;
    --orange: #d18616; --purple: #bc8cff;
  }
  * { box-sizing: border-box; margin: 0; padding: 0; }
  body { font-family: 'Segoe UI', system-ui, sans-serif; background: var(--bg); color: var(--text); font-size: 14px; line-height: 1.6; }
  header { background: var(--surface); border-bottom: 1px solid var(--border); padding: 24px 40px; }
  header h1 { font-size: 22px; font-weight: 700; color: var(--accent); }
  header .meta { color: var(--muted); font-size: 12px; margin-top: 4px; }
  .warning { background: #2d1f00; border: 1px solid var(--yellow); border-radius: 6px; padding: 12px 16px; margin: 20px 40px; color: var(--yellow); font-size: 13px; }
  .container { padding: 20px 40px; }
  .section { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; margin-bottom: 24px; overflow: hidden; }
  .section-header { padding: 14px 20px; border-bottom: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; }
  .section-header h2 { font-size: 15px; font-weight: 600; }
  .badge { font-size: 11px; padding: 2px 10px; border-radius: 20px; font-weight: 600; }
  .badge-blue { background: #1f3050; color: var(--accent); }
  .badge-green { background: #1a3328; color: var(--green); }
  .badge-red { background: #3d1c1c; color: var(--red); }
  .badge-yellow { background: #2d2208; color: var(--yellow); }
  .badge-muted { background: #1c2128; color: var(--muted); }
  .meta-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); gap: 12px; padding: 16px 20px; }
  .meta-item label { display: block; font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 4px; }
  .meta-item span { font-size: 13px; color: var(--text); font-family: monospace; }
  table { width: 100%; border-collapse: collapse; font-size: 13px; }
  th { background: #1c2128; color: var(--muted); text-align: left; padding: 10px 14px; font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; border-bottom: 1px solid var(--border); }
  td { padding: 9px 14px; border-bottom: 1px solid #1c2128; vertical-align: middle; }
  tr:last-child td { border-bottom: none; }
  tr:hover td { background: #1c2128; }
  .url-cell { font-family: monospace; max-width: 400px; word-break: break-all; }
  .url-cell a { color: var(--accent); text-decoration: none; }
  .url-cell a:hover { text-decoration: underline; }
  .status { display: inline-block; padding: 2px 8px; border-radius: 4px; font-family: monospace; font-weight: 700; font-size: 12px; }
  .s200 { background: #1a3328; color: var(--green); }
  .s301,.s302 { background: #1f3050; color: var(--accent); }
  .s401 { background: #2d2208; color: var(--yellow); }
  .s403 { background: #2d1f00; color: var(--orange); }
  .s500 { background: #3d1c1c; color: var(--red); }
  .s-other { background: #1c2128; color: var(--muted); }
  .interesting-row td { background: #1e1a0e !important; }
  .interesting-tag { font-size: 10px; background: #2d2208; color: var(--yellow); padding: 1px 6px; border-radius: 4px; margin-left: 6px; }
  .summary-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); gap: 16px; padding: 20px; }
  .stat-card { background: #1c2128; border-radius: 8px; padding: 16px; text-align: center; border: 1px solid var(--border); }
  .stat-card .num { font-size: 28px; font-weight: 700; color: var(--accent); }
  .stat-card .lbl { font-size: 11px; color: var(--muted); margin-top: 4px; }
  .cmd-block { font-family: monospace; font-size: 12px; background: #1c2128; border-radius: 6px; padding: 14px 16px; margin: 16px 20px; color: #7ee787; border: 1px solid var(--border); white-space: pre-wrap; word-break: break-all; }
  .no-results { padding: 24px; text-align: center; color: var(--muted); font-size: 13px; }
  footer { text-align: center; padding: 24px; color: var(--muted); font-size: 12px; border-top: 1px solid var(--border); margin-top: 24px; }
  .domain-anchor { color: var(--accent); font-size: 12px; text-decoration: none; }
  .toc { padding: 16px 20px; }
  .toc ul { list-style: none; padding: 0; }
  .toc li { padding: 4px 0; }
  .toc a { color: var(--accent); text-decoration: none; font-size: 13px; }
  .toc a:hover { text-decoration: underline; }
  /* ── tree view ── */
  .view-toggle { display: flex; gap: 6px; }
  .view-btn { background: transparent; border: 1px solid var(--border); color: var(--muted); padding: 3px 10px; border-radius: 4px; cursor: pointer; font-size: 11px; }
  .view-btn.active { background: #1f3050; color: var(--accent); border-color: var(--accent); }
  details.tree-node { margin: 1px 0; }
  details.tree-node > summary { padding: 5px 10px; cursor: pointer; list-style: none; display: flex; align-items: center; gap: 8px; border-radius: 4px; font-family: monospace; font-size: 13px; }
  details.tree-node > summary::-webkit-details-marker { display: none; }
  details.tree-node > summary:hover { background: #1c2128; }
  details.tree-node > summary .caret { font-size: 9px; color: var(--muted); display: inline-block; transition: transform 0.15s; min-width: 10px; }
  details.tree-node[open] > summary .caret { transform: rotate(90deg); }
  .tree-children { padding-left: 18px; border-left: 1px solid var(--border); margin-left: 10px; }
  .tree-folder-name { color: var(--text); font-weight: 600; }
  .tree-leaf-row { padding: 4px 10px; display: flex; align-items: center; gap: 8px; border-radius: 4px; font-family: monospace; font-size: 13px; }
  .tree-leaf-row:hover { background: #1c2128; }
  .tree-link { color: var(--accent); text-decoration: none; }
  .tree-link:hover { text-decoration: underline; }
  .tree-meta { color: var(--muted); font-size: 11px; }
  .tree-wrap { padding: 14px 20px; display: none; }
</style>
</head>
<body>
<header>
  <h1>🔍 ffuf-automation — Web Discovery Report</h1>
  <div class="meta">Generated: {{ scan_date }} &nbsp;|&nbsp; Total domains: {{ domains|length }} &nbsp;|&nbsp; Total paths found: {{ total_paths }} &nbsp;|&nbsp; Interesting findings: {{ total_interesting }}</div>
</header>

<div class="warning">
  ⚠️ <strong>AUTHORIZED USE ONLY</strong> — This report was generated by ffuf-automation. Scanning must only be performed against domains you own or have explicit written authorization to test.
</div>

<div class="container">

  <!-- Summary Stats -->
  <div class="section">
    <div class="section-header"><h2>📊 Scan Summary</h2></div>
    <div class="summary-grid">
      <div class="stat-card"><div class="num">{{ domains|length }}</div><div class="lbl">Domains Scanned</div></div>
      <div class="stat-card"><div class="num">{{ total_paths }}</div><div class="lbl">Paths Discovered</div></div>
      <div class="stat-card"><div class="num">{{ total_interesting }}</div><div class="lbl">Interesting Paths</div></div>
      <div class="stat-card"><div class="num">{{ failed_domains|length }}</div><div class="lbl">Failed Domains</div></div>
      <div class="stat-card"><div class="num">{{ scan_config.depth }}</div><div class="lbl">Recursion Depth</div></div>
      <div class="stat-card"><div class="num">{{ scan_config.threads }}</div><div class="lbl">Threads</div></div>
    </div>
  </div>

  <!-- Scan Config -->
  <div class="section">
    <div class="section-header"><h2>⚙️ Scan Configuration</h2></div>
    <div class="meta-grid">
      <div class="meta-item"><label>Wordlist</label><span>{{ scan_config.wordlist }}</span></div>
      <div class="meta-item"><label>Extensions</label><span>{{ scan_config.extensions or "None" }}</span></div>
      <div class="meta-item"><label>Match Codes</label><span>{{ scan_config.match_codes or "Default" }}</span></div>
      <div class="meta-item"><label>Filter Codes</label><span>{{ scan_config.filter_codes or "None" }}</span></div>
      <div class="meta-item"><label>Filter Words</label><span>{{ scan_config.filter_words or "None" }}</span></div>
      <div class="meta-item"><label>Filter Sizes</label><span>{{ scan_config.filter_sizes or "None" }}</span></div>
      <div class="meta-item"><label>Threads</label><span>{{ scan_config.threads }}</span></div>
      <div class="meta-item"><label>Timeout</label><span>{{ scan_config.timeout }}s</span></div>
      <div class="meta-item"><label>Recursion Depth</label><span>{{ scan_config.depth }}</span></div>
    </div>
  </div>

  <!-- Table of Contents -->
  <div class="section">
    <div class="section-header"><h2>📋 Domains Index</h2></div>
    <div class="toc">
      <ul>
        {% for d in domains %}
        <li><a href="#{{ d.domain | replace('https://', '') | replace('http://', '') | replace('/', '_') }}">{{ d.domain }}</a>
        &nbsp;
        {% if d.pending %}<span class="badge badge-muted">NOT SCANNED</span>
        {% else %}<span class="badge badge-blue">{{ d.results|length }} paths</span>
        {% if d.interesting %}<span class="badge badge-yellow">{{ d.interesting|length }} interesting</span>{% endif %}
        {% if d.failed %}<span class="badge badge-red">FAILED</span>{% endif %}
        {% endif %}
        </li>
        {% endfor %}
      </ul>
    </div>
  </div>

  <!-- Per-Domain Results -->
  {% macro render_node(node_name, node, path) %}
  {% set full_path = path + "/" + node_name %}
  {% if node._children %}
  <details class="tree-node">
    <summary>
      <span class="caret">▶</span>
      <span class="tree-folder-name">{{ node_name }}/</span>
      {% if node._item %}
      <a href="{{ node._item.url }}" class="tree-link" target="_blank" rel="noopener">{{ full_path }}</a>
      {% set sc = node._item.status | string %}
      <span class="status {% if sc == '200' %}s200{% elif sc in ['301','302'] %}s301{% elif sc == '401' %}s401{% elif sc == '403' %}s403{% elif sc == '500' %}s500{% else %}s-other{% endif %}">{{ node._item.status }}</span>
      <span class="tree-meta">{{ node._item.length }}B</span>
      {% endif %}
    </summary>
    <div class="tree-children">
      {% for child_name, child_node in node._children | dictsort %}
        {{ render_node(child_name, child_node, full_path) }}
      {% endfor %}
    </div>
  </details>
  {% elif node._item %}
  <div class="tree-leaf-row">
    <span class="caret" style="visibility:hidden">▶</span>
    <span>📄</span>
    <a href="{{ node._item.url }}" class="tree-link" target="_blank" rel="noopener">{{ full_path }}</a>
    {% set sc = node._item.status | string %}
    <span class="status {% if sc == '200' %}s200{% elif sc in ['301','302'] %}s301{% elif sc == '401' %}s401{% elif sc == '403' %}s403{% elif sc == '500' %}s500{% else %}s-other{% endif %}">{{ node._item.status }}</span>
    <span class="tree-meta">{{ node._item.length }}B · {{ node._item.words }}w</span>
    {% if node._item.redirect %}<span class="tree-meta">→ {{ node._item.redirect }}</span>{% endif %}
  </div>
  {% endif %}
  {% endmacro %}

  {% for d in domains %}
  {% set anchor = d.domain | replace('https://', '') | replace('http://', '') | replace('/', '_') %}
  <div class="section" id="{{ anchor }}">
    <div class="section-header">
      <h2>🌐 {{ d.domain }}</h2>
      <div style="display:flex;align-items:center;gap:10px;">
        {% if d.pending %}<span class="badge badge-muted">NOT SCANNED</span>
        {% else %}
        <span class="badge badge-blue">{{ d.results|length }} paths</span>
        {% if d.interesting %}<span class="badge badge-yellow">{{ d.interesting|length }} interesting</span>{% endif %}
        {% if d.failed %}<span class="badge badge-red">SCAN FAILED</span>{% endif %}
        {% if d.results and not d.failed %}
        <div class="view-toggle">
          <button class="view-btn active" id="btntbl-{{ anchor }}" onclick="toggleView('{{ anchor }}','table')">Table</button>
          <button class="view-btn"        id="btntree-{{ anchor }}" onclick="toggleView('{{ anchor }}','tree')">Tree</button>
        </div>
        {% endif %}
        {% endif %}
      </div>
    </div>

    {% if d.pending %}
    <div class="no-results" style="color:var(--muted)">⏸ Scan was cancelled before this domain was reached.</div>
    {% elif d.command %}
    <div class="cmd-block">$ {{ d.command }}</div>
    {% endif %}

    {% if d.results and not d.pending %}
    <div id="tbl-{{ anchor }}">
    <table>
      <thead>
        <tr>
          <th>URL</th>
          <th>Status</th>
          <th>Size</th>
          <th>Words</th>
          <th>Lines</th>
          <th>Redirect</th>
        </tr>
      </thead>
      <tbody>
        {% for r in d.results %}
        {% set is_interesting = r.url in (d.interesting | map(attribute='url') | list) %}
        <tr {% if is_interesting %}class="interesting-row"{% endif %}>
          <td class="url-cell">
            <a href="{{ r.url }}" target="_blank" rel="noopener">{{ r.url }}</a>
            {% if is_interesting %}<span class="interesting-tag">⚠ interesting</span>{% endif %}
          </td>
          <td>
            {% set sc = r.status | string %}
            <span class="status
              {% if sc == '200' %}s200
              {% elif sc in ['301','302'] %}s301
              {% elif sc == '401' %}s401
              {% elif sc == '403' %}s403
              {% elif sc == '500' %}s500
              {% else %}s-other{% endif %}">
              {{ r.status }}
            </span>
          </td>
          <td>{{ r.length }}</td>
          <td>{{ r.words }}</td>
          <td>{{ r.lines }}</td>
          <td style="font-family:monospace;font-size:11px;color:var(--muted)">{{ r.redirect or "—" }}</td>
        </tr>
        {% endfor %}
      </tbody>
    </table>
    </div>
    <div id="tree-{{ anchor }}" class="tree-wrap">
      {% if d.path_tree %}
        {% for node_name, node in d.path_tree | dictsort %}
          {{ render_node(node_name, node, "") }}
        {% endfor %}
      {% else %}
        <div class="no-results">No paths to display.</div>
      {% endif %}
    </div>
    {% elif d.failed and not d.pending %}
    <div class="no-results">❌ Scan failed for this domain. Check logs for details.</div>
    {% elif not d.pending and not d.results %}
    <div class="no-results">No results found for this domain.</div>
    {% endif %}

    {% if d.interesting %}
    <div style="border-top:1px solid var(--border);padding:14px 20px;">
      <strong style="color:var(--yellow);font-size:13px;">⚠ Interesting Findings ({{ d.interesting|length }})</strong>
      <table style="margin-top:10px;">
        <thead><tr><th>URL</th><th>Match Reason</th><th>Status</th></tr></thead>
        <tbody>
          {% for i in d.interesting %}
          <tr>
            <td class="url-cell"><a href="{{ i.url }}" target="_blank">{{ i.url }}</a></td>
            <td style="font-family:monospace;color:var(--yellow)">{{ i.match_reason }}</td>
            <td><span class="status s-other">{{ i.status }}</span></td>
          </tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
    {% endif %}
  </div>
  {% endfor %}

  {% if failed_domains %}
  <div class="section">
    <div class="section-header"><h2>❌ Failed Domains</h2><span class="badge badge-red">{{ failed_domains|length }}</span></div>
    <table>
      <thead><tr><th>Domain</th><th>Reason</th></tr></thead>
      <tbody>
        {% for fd in failed_domains %}
        <tr><td>{{ fd.domain }}</td><td style="color:var(--muted)">{{ fd.reason }}</td></tr>
        {% endfor %}
      </tbody>
    </table>
  </div>
  {% endif %}

</div>
<script>
function toggleView(id, view) {
  var tbl  = document.getElementById('tbl-'  + id);
  var tree = document.getElementById('tree-' + id);
  var btnT = document.getElementById('btntbl-'  + id);
  var btnR = document.getElementById('btntree-' + id);
  if (!tbl || !tree) return;
  tbl.style.display  = view === 'table' ? '' : 'none';
  tree.style.display = view === 'tree'  ? 'block' : 'none';
  btnT.classList.toggle('active', view === 'table');
  btnR.classList.toggle('active', view === 'tree');
}
</script>
<footer>
  ffuf-automation v1.0.0 —  &nbsp;|&nbsp; Report generated {{ scan_date }} &nbsp;|&nbsp; Authorized use only
</footer>
</body>
</html>
"""


def write_html_report(scan_data: dict, output_path: str):
    """Render the HTML report using either Jinja2 or a simple string format."""
    if HAS_JINJA2:
        env = Environment(loader=BaseLoader())
        tmpl = env.from_string(HTML_TEMPLATE)
        html = tmpl.render(**scan_data)
    else:
        # Fallback: basic substitution (Jinja2 not installed)
        html = _render_html_fallback(scan_data)

    with open(output_path, "w") as f:
        f.write(html)


def _render_html_fallback(scan_data: dict) -> str:
    """Very basic HTML fallback if Jinja2 is missing."""
    rows = ""
    for d in scan_data.get("domains", []):
        rows += f"<h2>{d['domain']}</h2>"
        for r in d.get("results", []):
            rows += f"<p>{r['url']} — {r['status']} — {r['length']}B</p>"
    return f"""<!DOCTYPE html><html><head><title>ffuf Report</title></head>
<body style="font-family:monospace;background:#0d1117;color:#e6edf3;padding:24px">
<h1>ffuf-automation Report — {scan_data.get('scan_date','')}</h1>
<p>Install Jinja2 for the full styled report: pip install jinja2</p>
{rows}
</body></html>"""


# ─────────────────────────────────────────────
# Scope Validation
# ─────────────────────────────────────────────
def load_scope(scope_file: Optional[str]) -> Optional[set]:
    """Load a scope whitelist file (one domain per line)."""
    if not scope_file:
        return None
    if not os.path.isfile(scope_file):
        raise FileNotFoundError(f"Scope file not found: {scope_file}")
    scope = set()
    with open(scope_file) as f:
        for line in f:
            d = normalize_domain(line)
            if d:
                scope.add(d)
    return scope


def in_scope(domain: str, scope: Optional[set]) -> bool:
    """Return True if domain is in scope (or scope is unrestricted)."""
    if scope is None:
        return True
    return domain in scope


# ─────────────────────────────────────────────
# Main Scan Orchestrator
# ─────────────────────────────────────────────
def run_scan(args: argparse.Namespace, logger: logging.Logger):
    """Main orchestration loop — scan each domain and collect results."""

    # 1. Load & validate domains
    domains = load_domains(args.domains, scheme=args.scheme)
    if not domains:
        logger.error("No valid domains found in input file.")
        sys.exit(1)

    # 2. Scope check
    scope = load_scope(getattr(args, "scope", None))
    domains_in_scope = [d for d in domains if in_scope(d, scope)]
    out_of_scope = [d for d in domains if not in_scope(d, scope)]
    if out_of_scope:
        logger.warning(f"[SCOPE] Skipping {len(out_of_scope)} domain(s) not in scope file.")
    domains = domains_in_scope

    if not domains:
        logger.error("All domains are out of scope. Exiting.")
        sys.exit(1)

    # 3. Confirmation prompt
    if not confirm_scan(domains):
        logger.warning("Scan cancelled by user.")
        sys.exit(0)

    # 4. Setup output dirs
    raw_dir = os.path.join(args.output, "raw")
    reports_dir = os.path.join(args.output, "reports")
    os.makedirs(raw_dir, exist_ok=True)
    os.makedirs(reports_dir, exist_ok=True)
    os.makedirs(os.path.join(args.output, "logs"), exist_ok=True)

    scan_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    all_domain_data = []
    all_flat_results = []
    failed_domains = []
    total_paths = 0
    total_interesting = 0

    scan_config = {
        "wordlist":     args.wordlist,
        "extensions":   args.extensions,
        "match_codes":  args.match_codes,
        "filter_codes": args.filter_codes,
        "filter_words": args.filter_words,
        "filter_sizes": args.filter_sizes,
        "filter_lines": getattr(args, "filter_lines", None),
        "threads":      args.threads,
        "timeout":      args.timeout,
        "depth":        args.depth,
        "scheme":       args.scheme,
    }

    # 5. Per-domain scan loop
    cancelled = False
    try:
        for i, domain in enumerate(domains, 1):
            logger.info(f"[{i}/{len(domains)}] Scanning: {domain}")

            if getattr(args, "force_recursion", False):
                results, cmd_str, scan_failed, was_interrupted = run_force_recursive_scan(domain, args, logger, raw_dir)
                if scan_failed:
                    failed_domains.append({"domain": domain, "reason": "ffuf exited with error"})
                    all_domain_data.append({
                        "domain": domain, "failed": True,
                        "results": [], "interesting": [], "command": cmd_str,
                    })
                    continue
                # Always add domain (even partial/empty) then re-raise if interrupted
                interesting = find_interesting(results)
                total_paths += len(results)
                total_interesting += len(interesting)
                for r in results:
                    is_int = r["url"] in {x["url"] for x in interesting}
                    all_flat_results.append({**r, "domain": domain, "interesting": "YES" if is_int else ""})
                all_domain_data.append({
                    "domain": domain, "failed": False,
                    "results": results, "interesting": interesting, "command": cmd_str,
                    "path_tree": build_path_tree(results),
                })
                logger.info(f"  → {len(results)} paths found, {len(interesting)} interesting")
                if was_interrupted:
                    raise KeyboardInterrupt
                continue
            else:
                fname = safe_filename(domain)
                json_out = os.path.join(raw_dir, f"{fname}.json")
                cmd = build_ffuf_command(domain, args.wordlist, json_out, args)
                cmd_str = " ".join(cmd)

                if os.path.isfile(json_out) and os.path.getsize(json_out) > 10:
                    logger.info(f"[SKIP] Already scanned (raw file exists): {domain}")
                else:
                    try:
                        success = run_ffuf(cmd, domain, logger, verbose=args.verbose)
                    except KeyboardInterrupt:
                        # ffuf wrote whatever it had — always save the domain
                        # even if 0 results so it shows as scanned, not pending
                        partial = parse_ffuf_results(json_out, args.max_results)
                        interesting_p = find_interesting(partial)
                        total_paths += len(partial)
                        total_interesting += len(interesting_p)
                        for r in partial:
                            is_int = r["url"] in {x["url"] for x in interesting_p}
                            all_flat_results.append({**r, "domain": domain, "interesting": "YES" if is_int else ""})
                        all_domain_data.append({
                            "domain": domain, "failed": False,
                            "results": partial, "interesting": interesting_p, "command": cmd_str,
                            "path_tree": build_path_tree(partial),
                        })
                        logger.info(f"  → {len(partial)} partial path(s) saved for {domain} (cancelled)")
                        raise
                    if not success:
                        failed_domains.append({"domain": domain, "reason": "ffuf exited with error"})
                        all_domain_data.append({
                            "domain": domain, "failed": True,
                            "results": [], "interesting": [], "command": cmd_str,
                        })
                        continue

                results = parse_ffuf_results(json_out, args.max_results)

            interesting = find_interesting(results)
            total_paths += len(results)
            total_interesting += len(interesting)
            logger.info(f"  → {len(results)} paths found, {len(interesting)} interesting")

            for r in results:
                is_int = r["url"] in {x["url"] for x in interesting}
                all_flat_results.append({**r, "domain": domain, "interesting": "YES" if is_int else ""})

            all_domain_data.append({
                "domain":      domain,
                "failed":      False,
                "results":     results,
                "interesting": interesting,
                "command":     cmd_str,
                "path_tree":   build_path_tree(results),
            })

            if args.delay and i < len(domains):
                logger.info(f"[DELAY] Sleeping {args.delay}s before next domain...")
                time.sleep(args.delay)

    except KeyboardInterrupt:
        cancelled = True
        logger.warning("\n[INTERRUPTED] Scan cancelled — generating partial report...")
        # Add every unscanned domain as pending so they appear in the report
        scanned = {d["domain"] for d in all_domain_data} | {d["domain"] for d in failed_domains}
        for domain in domains:
            if domain not in scanned:
                all_domain_data.append({
                    "domain":    domain,
                    "failed":    False,
                    "pending":   True,
                    "results":   [],
                    "interesting": [],
                    "command":   "",
                })

    # Skip report if nothing was scanned at all
    if not all_domain_data:
        logger.warning("[SKIPPED] No domains completed — no report generated.")
        sys.exit(0)

    # 7. Build report payload
    scan_data = {
        "scan_date":        scan_date,
        "tool":             f"ffuf-automation v{VERSION}",
        "scan_config":      scan_config,
        "domains":          all_domain_data,
        "failed_domains":   failed_domains,
        "total_paths":      total_paths,
        "total_interesting": total_interesting,
    }

    # 8. Write reports
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    suffix = "_partial" if cancelled else ""

    json_report = os.path.join(reports_dir, f"report_{ts}{suffix}.json")
    csv_report  = os.path.join(reports_dir, f"report_{ts}{suffix}.csv")
    html_report = os.path.join(reports_dir, f"report_{ts}{suffix}.html")

    write_json_report(scan_data, json_report)
    logger.info(f"[REPORT] JSON → {json_report}")

    write_csv_report(all_flat_results, csv_report)
    logger.info(f"[REPORT] CSV  → {csv_report}")

    write_html_report(scan_data, html_report)
    logger.info(f"[REPORT] HTML → {html_report}")

    # 9. Print summary
    status_label = "PARTIAL REPORT (cancelled)" if cancelled else "SCAN COMPLETE"
    print(f"""
\033[1;36m{'─'*60}
 {status_label}
{'─'*60}\033[0m
 Domains scanned : {len(all_domain_data)}
 Paths found     : {total_paths}
 Interesting     : {total_interesting}
 Failed          : {len(failed_domains)}
 Reports         : {reports_dir}
\033[1;36m{'─'*60}\033[0m
""")


# ─────────────────────────────────────────────
# CLI Argument Parser
# ─────────────────────────────────────────────
def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        prog="ffuf_automation.py",
        description="Professional ffuf web directory discovery automation — AUTHORIZED DOMAINS ONLY",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python3 ffuf_automation.py --domains domains.txt --wordlist wordlist.txt --output results
  python3 ffuf_automation.py --domains domains.txt --wordlist /path/to/wl.txt \\
      --depth 2 --extensions .php,.txt,.bak \\
      --match-codes 200,204,301,302,403 --filter-codes 404 \\
      --threads 50 --timeout 10 --output results

LEGAL: Only scan domains you own or have explicit written authorization to test.
        """,
    )

    parser.add_argument("--domains",       required=True,  help="Path to domains file (one domain per line)")
    parser.add_argument("--wordlist",      required=True,  help="Path to wordlist for ffuf")
    parser.add_argument("--output",        default="results", help="Output directory (default: results)")
    parser.add_argument("--depth",         type=int, default=1, help="Recursion depth (default: 1, 0=disabled)")
    parser.add_argument("--force-recursion", dest="force_recursion", action="store_true", default=False,
                        help="Custom Python-level recursion: recurse into paths regardless of HTTP response (even 404)")
    parser.add_argument("--force-recursion-paths", dest="force_recursion_paths", default="",
                        help="File with specific paths to force-recurse into (one per line). If omitted, uses all wordlist words.")
    parser.add_argument("--extensions",    default="",     help="Extensions to append e.g. .php,.txt,.bak")
    parser.add_argument("--extensions-file", dest="extensions_file", default="",
                        help="File with extensions to append, one per line (e.g. extensions.txt)")
    parser.add_argument("--match-codes",   dest="match_codes",  default="200,204,301,302,403,401", help="Match HTTP status codes")
    parser.add_argument("--filter-codes",  dest="filter_codes", default="404", help="Filter out HTTP status codes")
    parser.add_argument("--filter-words",  dest="filter_words", default="", help="Filter by word count e.g. 10,20")
    parser.add_argument("--filter-sizes",  dest="filter_sizes", default="", help="Filter by response size e.g. 1234,5678")
    parser.add_argument("--filter-lines",  dest="filter_lines", default="", help="Filter by line count e.g. 0,1")
    parser.add_argument("--threads",       type=int, default=40, help="Number of ffuf threads (default: 40)")
    parser.add_argument("--timeout",       type=int, default=10, help="Per-request timeout in seconds (default: 10)")
    parser.add_argument("--rate",          type=int, default=0,  help="Max requests per second (0=unlimited)")
    parser.add_argument("--delay",         type=float, default=0, help="Delay in seconds between domains (default: 0)")
    parser.add_argument("--max-results",   dest="max_results", type=int, default=0, help="Max results per domain (0=unlimited)")
    parser.add_argument("--scope",         default="",    help="Optional scope whitelist file")
    parser.add_argument("--interesting-file", dest="interesting_file", default="",
                        help="File with interesting path patterns, one per line")
    parser.add_argument("--scheme",        default="https", choices=["http", "https", "both"],
                        help="Protocol to use: http, https, or both (default: https)")
    parser.add_argument("--verbose",       action="store_true", default=False,
                        help="Stream ffuf's full live output to the terminal instead of the condensed req/sec stats line")
    parser.add_argument("--version",       action="version", version=f"ffuf-automation {VERSION}")

    return parser


# ─────────────────────────────────────────────
# Entry Point
# ─────────────────────────────────────────────
def main():
    print_banner()

    parser = build_parser()
    args = parser.parse_args()

    # Normalize optional empty strings to None
    if not args.scope:
        args.scope = None
    if not args.filter_words:
        args.filter_words = None
    if not args.filter_sizes:
        args.filter_sizes = None
    if not args.filter_lines:
        args.filter_lines = None
    if args.max_results == 0:
        args.max_results = None

    # Check ffuf
    ffuf_path, ffuf_ver = check_ffuf_installed()
    if not ffuf_path:
        print("\033[1;31m[ERROR] ffuf is not installed or not in PATH.\033[0m")
        print("Install it: go install github.com/ffuf/ffuf/v2@latest")
        print("Or: apt install ffuf  (Kali Linux)")
        sys.exit(1)
    print(f"\033[1;32m[OK] ffuf found: {ffuf_path}  {ffuf_ver}\033[0m\n")

    # Load interesting paths from file if provided
    if args.interesting_file:
        if not os.path.isfile(args.interesting_file):
            print(f"\033[1;31m[ERROR] Interesting paths file not found: {args.interesting_file}\033[0m")
            sys.exit(1)
        with open(args.interesting_file) as f:
            custom = [
                line.strip() for line in f
                if line.strip() and not line.strip().startswith("#")
            ]
        custom = [p if p.startswith("/") else f"/{p}" for p in custom]
        added = [p for p in custom if p not in INTERESTING_PATHS]
        INTERESTING_PATHS.extend(added)
        print(f"\033[1;32m[OK] Loaded {len(custom)} interesting path pattern(s) from {args.interesting_file}\033[0m\n")

    # Load extensions from file if provided
    if args.extensions_file:
        if not os.path.isfile(args.extensions_file):
            print(f"\033[1;31m[ERROR] Extensions file not found: {args.extensions_file}\033[0m")
            sys.exit(1)
        with open(args.extensions_file) as f:
            file_exts = [
                line.strip() for line in f
                if line.strip() and not line.strip().startswith("#")
            ]
        # Ensure each extension starts with a dot
        file_exts = [e if e.startswith(".") else f".{e}" for e in file_exts]
        # Merge with any --extensions value
        existing = [e for e in args.extensions.split(",") if e] if args.extensions else []
        merged = existing + [e for e in file_exts if e not in existing]
        args.extensions = ",".join(merged)
        print(f"\033[1;32m[OK] Loaded {len(file_exts)} extension(s) from {args.extensions_file}: {args.extensions}\033[0m\n")

    # Validate --force-recursion-paths if provided
    if args.force_recursion_paths:
        if not os.path.isfile(args.force_recursion_paths):
            print(f"\033[1;31m[ERROR] Force-recursion paths file not found: {args.force_recursion_paths}\033[0m")
            sys.exit(1)
    else:
        args.force_recursion_paths = None

    # Check wordlist
    if not os.path.isfile(args.wordlist):
        print(f"\033[1;31m[ERROR] Wordlist not found: {args.wordlist}\033[0m")
        sys.exit(1)

    # Setup logging
    logger = setup_logging(args.output, verbose=args.verbose)

    # Run
    try:
        run_scan(args, logger)
    except KeyboardInterrupt:
        logger.warning("\n[INTERRUPTED] Scan interrupted by user.")
        sys.exit(0)
    except Exception as e:
        logger.error(f"[FATAL] Unexpected error: {e}", exc_info=True)
        sys.exit(1)


if __name__ == "__main__":
    main()