import { load } from "cheerio/slim";
import type { CheerioAPI } from "cheerio/slim";
import type { Element } from "domhandler";
import type { SelectorRules } from "./rules";

export interface SearchResult {
  title: string;
  url: string;
  snippet: string;
}

export interface PageData {
  url: string;
  title: string;
  html: string;
  text: string;
}

export type PageKind = "ok" | "captcha" | "blocked" | "no_results" | "parser_failed" | "empty";

const clean = (s: string | undefined | null): string => (s ?? "").replace(/\s+/g, " ").trim();

export function hostMatches(host: string, domains: string[]): boolean {
  const h = host.toLowerCase();
  return domains.some((d) => h === d || h.endsWith(`.${d}`));
}

// Turns a result link into the real destination URL, unwrapping engine redirect links.
// Returns null for links that stay on the engine itself or are not http(s).
export function resolveResultUrl(
  raw: string | undefined,
  base: string,
  domains: string[],
  preferredParam?: string
): string | null {
  if (!raw) return null;
  let u: URL;
  try {
    u = new URL(raw, base);
  } catch {
    return null;
  }
  if (hostMatches(u.hostname, domains)) {
    const params = [preferredParam, "uddg", "u", "url", "target"].filter((p): p is string => !!p);
    for (const name of params) {
      const value = u.searchParams.get(name);
      if (value && /^https?:\/\//i.test(value)) {
        try {
          u = new URL(value);
        } catch {
          return null;
        }
        break;
      }
    }
  }
  if (u.protocol !== "http:" && u.protocol !== "https:") return null;
  if (hostMatches(u.hostname, domains)) return null;
  u.hash = "";
  return u.toString();
}

function safeSelect($: CheerioAPI, selector: string) {
  try {
    return $(selector);
  } catch {
    return null;
  }
}

export interface RuleStats {
  containers: number;
  withTitle: number;
  withLink: number;
  withSnippet: number;
}

export function applyRules(
  html: string,
  baseUrl: string,
  rules: SelectorRules,
  domains: string[],
  stats?: RuleStats
): SearchResult[] {
  const $ = load(html);
  const out: SearchResult[] = [];
  const seen = new Set<string>();
  const containers = safeSelect($, rules.container);
  if (!containers) return out;
  if (stats) stats.containers = containers.length;
  containers.each((_, el) => {
    const $el = $(el);
    if (rules.exclude) {
      try {
        if ($el.is(rules.exclude)) return;
      } catch {
        // invalid exclude selectors are ignored
      }
    }
    let title = "";
    let href: string | undefined;
    let snippet = "";
    try {
      title = clean($el.find(rules.title).first().text());
      href = $el.find(rules.link).first().attr("href");
      if (rules.snippet) snippet = clean($el.find(rules.snippet).first().text());
    } catch {
      return;
    }
    const url = resolveResultUrl(href, baseUrl, domains, rules.urlParam);
    if (stats) {
      if (title) stats.withTitle++;
      if (url) stats.withLink++;
      if (snippet.length >= 20) stats.withSnippet++;
    }
    if (!title || !url || seen.has(url)) return;
    seen.add(url);
    out.push({ title: title.slice(0, 200), url, snippet: snippet.slice(0, 400) });
  });
  return out.slice(0, 15);
}

interface HeuristicLink extends SearchResult {
  el: Element;
}

const URL_LIKE = /^(https?:\/\/)?(www\.)?[\w-]+(\.[\w-]+)+(\s*[\u203a>\/].*)?$/i;

function looksLikeUrlText(text: string): boolean {
  return URL_LIKE.test(text) && !/\s\w+\s\w+\s/.test(text);
}

// Layout-independent fallback: outbound links with real link text, skipping page chrome.
// When several links share a destination (title, displayed URL, description), the one
// that reads most like a title wins: inside a heading, not a bare URL, reasonably short.
function collectOutboundLinks($: CheerioAPI, baseUrl: string, domains: string[], limit: number): HeuristicLink[] {
  $("script, style, noscript, template").remove();
  const byUrl = new Map<string, { order: number; best: { el: Element; title: string; score: number } }>();
  let order = 0;
  $("a[href]").each((_, a) => {
    const $a = $(a);
    if ($a.closest("header, footer, nav, [role=navigation], form").length) return;
    const title = clean($a.text());
    if (title.length < 12) return;
    const url = resolveResultUrl($a.attr("href"), baseUrl, domains);
    if (!url) return;
    let score = 0;
    if ($a.closest("h1, h2, h3, h4").length || $a.find("h1, h2, h3, h4").length) score += 4;
    if (looksLikeUrlText(title)) score -= 4;
    if (title.length > 160) score -= 2;
    const entry = byUrl.get(url);
    if (!entry) byUrl.set(url, { order: order++, best: { el: a as Element, title, score } });
    else if (score > entry.best.score) entry.best = { el: a as Element, title, score };
  });
  const picked = [...byUrl.entries()].sort((x, y) => x[1].order - y[1].order).slice(0, limit);
  return picked.map(([url, { best }]) => {
    let snippet = "";
    let parent = $(best.el).parent();
    for (let i = 0; i < 4 && parent.length; i++) {
      const text = clean(parent.text());
      if (text.length >= best.title.length + 40) {
        snippet = clean(text.replace(best.title, "")).slice(0, 300);
        break;
      }
      parent = parent.parent();
    }
    return { title: best.title.slice(0, 200), url, snippet, el: best.el };
  });
}

export function heuristicResults(html: string, baseUrl: string, domains: string[], limit = 10): SearchResult[] {
  return collectOutboundLinks(load(html), baseUrl, domains, limit).map(({ title, url, snippet }) => ({
    title,
    url,
    snippet,
  }));
}

const CAPTCHA_SELECTORS = [
  "#anomaly-modal",
  ".anomaly-modal",
  ".g-recaptcha",
  ".h-captcha",
  ".cf-turnstile",
  "#challenge-form",
  "#challenge-stage",
  "iframe[src*='captcha']",
  "iframe[src*='challenges.cloudflare.com']",
  "form[action*='captcha']",
].join(", ");

const CAPTCHA_TEXT =
  /verify (that )?you(?:'re|’re| are) (a )?human|are you a robot|not a robot|unusual traffic from your|bots use duckduckgo too|complete the following challenge|select all (the )?(images|squares)|checking (if the site connection is secure|your browser)|just a moment\.\.\./i;

const BLOCKED_TEXT =
  /automated queries|access denied|too many requests|temporarily (blocked|unavailable)|request (has been )?blocked|403 forbidden/i;

const NO_RESULTS_TEXT = /no results (found|for)|did not match any|no web results|there are no results/i;

export function looksLikeCaptcha(page: PageData): boolean {
  if (/captcha|\/challenge/i.test(page.url)) return true;
  if (page.html) {
    try {
      if (load(page.html)(CAPTCHA_SELECTORS).length) return true;
    } catch {
      // fall through to the text check
    }
  }
  return CAPTCHA_TEXT.test(page.text.slice(0, 6000));
}

export interface Classification {
  kind: PageKind;
  results: SearchResult[];
}

export function classifySearchPage(
  page: PageData,
  rules: SelectorRules,
  domains: string[],
  noResultsSelector?: string
): Classification {
  if (!page.html && !page.text) return { kind: "empty", results: [] };
  const results = page.html ? applyRules(page.html, page.url, rules, domains) : [];
  if (results.length) return { kind: "ok", results };
  if (looksLikeCaptcha(page)) return { kind: "captcha", results: [] };
  let noResults = NO_RESULTS_TEXT.test(page.text.slice(0, 6000));
  if (!noResults && noResultsSelector && page.html) {
    try {
      noResults = load(page.html)(noResultsSelector).length > 0;
    } catch {
      // ignore
    }
  }
  if (noResults) return { kind: "no_results", results: [] };
  if (BLOCKED_TEXT.test(page.text.slice(0, 6000))) return { kind: "blocked", results: [] };
  return { kind: "parser_failed", results: page.html ? heuristicResults(page.html, page.url, domains) : [] };
}

function describe($: CheerioAPI, el: Element): string {
  const $el = $(el);
  const id = $el.attr("id");
  const classes = ($el.attr("class") ?? "").split(/\s+/).filter(Boolean).slice(0, 4);
  return `${el.tagName}${id ? `#${id}` : ""}${classes.map((c) => `.${c}`).join("")}`;
}

// A compact picture of how result links sit in the page, for writing new selectors.
export function layoutSample(html: string, baseUrl: string, domains: string[]): string {
  const $ = load(html);
  const links = collectOutboundLinks($, baseUrl, domains, 8);
  if (!links.length) return "No outbound result links were found on the page, so there is nothing to build selectors from.";
  const sample = links.slice(0, 6);
  const lines: string[] = [];
  const shared = new Map<string, { level: number; desc: string; elements: Set<Element>; holdsDescription: number }>();
  sample.forEach((link, i) => {
    const block = $(link.el).parents().eq(2);
    const candidate = block
      .find("p, span, div, a")
      .filter((_, e) => {
        if (e === link.el || $(e).find(link.el).length || $(e).closest("h1, h2, h3, h4").length || $(e).find("h1, h2, h3, h4").length) return false;
        const t = clean($(e).text());
        return t.length >= 40 && !t.includes(link.title) && !looksLikeUrlText(t) && $(e).children().length <= 3;
      })
      .first();
    const descEl = candidate.length ? (candidate[0] as Element) : null;

    const chain: string[] = [describe($, link.el)];
    let parent = $(link.el).parent();
    for (let level = 1; level <= 5 && parent.length; level++) {
      const node = parent[0] as Element;
      if (node.type !== "tag") break;
      const desc = describe($, node);
      chain.push(desc);
      const key = `${level}|${desc}`;
      const entry = shared.get(key) ?? { level, desc, elements: new Set<Element>(), holdsDescription: 0 };
      entry.elements.add(node);
      if (!descEl || $(node).find(descEl).length) entry.holdsDescription++;
      shared.set(key, entry);
      parent = parent.parent();
    }
    lines.push(`${i + 1}. title link "${link.title.slice(0, 70)}" -> ${link.url.slice(0, 90)}`);
    lines.push(`   title link and its parents, innermost first: ${chain.join(" < ")}`);
    if (descEl) lines.push(`   description candidate: ${describe($, descEl)} "${clean(candidate.text()).slice(0, 80)}"`);
  });
  // A good container appears once per result: the same descriptor, but a different
  // element around each sample link. One element around all of them is the list itself.
  const perResult = [...shared.values()]
    .filter((e) => e.elements.size >= Math.min(3, sample.length) && e.holdsDescription >= e.elements.size / 2)
    .sort((a, b) => a.level - b.level);
  const lists = [...shared.values()].filter((e) => e.elements.size === 1 && sample.length > 1).sort((a, b) => a.level - b.level);
  if (perResult.length) {
    lines.push("");
    lines.push("Elements that wrap each result separately (good container candidates, closest to the link first):");
    for (const e of perResult.slice(0, 4)) lines.push(`- ${e.desc} (${e.level} level(s) above the title link, one per result)`);
  }
  if (lists.length) {
    lines.push(`Element that wraps the whole list (not a container): ${lists[0].desc}`);
  }
  return lines.join("\n").slice(0, 4000);
}

import { load } from "cheerio/slim";
import type { CheerioAPI } from "cheerio/slim";
import type { Element } from "domhandler";
import type { SelectorRules } from "./rules";

export interface SearchResult {
  title: string;
  url: string;
  snippet: string;
}

export interface PageData {
  url: string;
  title: string;
  html: string;
  text: string;
}

export type PageKind = "ok" | "captcha" | "blocked" | "no_results" | "parser_failed" | "empty";

const clean = (s: string | undefined | null): string => (s ?? "").replace(/\s+/g, " ").trim();

export function hostMatches(host: string, domains: string[]): boolean {
  const h = host.toLowerCase();
  return domains.some((d) => h === d || h.endsWith(`.${d}`));
}

// Turns a result link into the real destination URL, unwrapping engine redirect links.
// Returns null for links that stay on the engine itself or are not http(s).
export function resolveResultUrl(
  raw: string | undefined,
  base: string,
  domains: string[],
  preferredParam?: string
): string | null {
  if (!raw) return null;
  let u: URL;
  try {
    u = new URL(raw, base);
  } catch {
    return null;
  }
  if (hostMatches(u.hostname, domains)) {
    const params = [preferredParam, "uddg", "u", "url", "target"].filter((p): p is string => !!p);
    for (const name of params) {
      const value = u.searchParams.get(name);
      if (value && /^https?:\/\//i.test(value)) {
        try {
          u = new URL(value);
        } catch {
          return null;
        }
        break;
      }
    }
  }
  if (u.protocol !== "http:" && u.protocol !== "https:") return null;
  if (hostMatches(u.hostname, domains)) return null;
  u.hash = "";
  return u.toString();
}

function safeSelect($: CheerioAPI, selector: string) {
  try {
    return $(selector);
  } catch {
    return null;
  }
}

export interface RuleStats {
  containers: number;
  withTitle: number;
  withLink: number;
  withSnippet: number;
}

export function applyRules(
  html: string,
  baseUrl: string,
  rules: SelectorRules,
  domains: string[],
  stats?: RuleStats
): SearchResult[] {
  const $ = load(html);
  const out: SearchResult[] = [];
  const seen = new Set<string>();
  const containers = safeSelect($, rules.container);
  if (!containers) return out;
  if (stats) stats.containers = containers.length;
  containers.each((_, el) => {
    const $el = $(el);
    if (rules.exclude) {
      try {
        if ($el.is(rules.exclude)) return;
      } catch {
        // invalid exclude selectors are ignored
      }
    }
    let title = "";
    let href: string | undefined;
    let snippet = "";
    try {
      title = clean($el.find(rules.title).first().text());
      href = $el.find(rules.link).first().attr("href");
      if (rules.snippet) snippet = clean($el.find(rules.snippet).first().text());
    } catch {
      return;
    }
    const url = resolveResultUrl(href, baseUrl, domains, rules.urlParam);
    if (stats) {
      if (title) stats.withTitle++;
      if (url) stats.withLink++;
      if (snippet.length >= 20) stats.withSnippet++;
    }
    if (!title || !url || seen.has(url)) return;
    seen.add(url);
    out.push({ title: title.slice(0, 200), url, snippet: snippet.slice(0, 400) });
  });
  return out.slice(0, 15);
}

interface HeuristicLink extends SearchResult {
  el: Element;
}

const URL_LIKE = /^(https?:\/\/)?(www\.)?[\w-]+(\.[\w-]+)+(\s*[\u203a>\/].*)?$/i;

function looksLikeUrlText(text: string): boolean {
  return URL_LIKE.test(text) && !/\s\w+\s\w+\s/.test(text);
}

// Layout-independent fallback: outbound links with real link text, skipping page chrome.
// When several links share a destination (title, displayed URL, description), the one
// that reads most like a title wins: inside a heading, not a bare URL, reasonably short.
function collectOutboundLinks($: CheerioAPI, baseUrl: string, domains: string[], limit: number): HeuristicLink[] {
  $("script, style, noscript, template").remove();
  const byUrl = new Map<string, { order: number; best: { el: Element; title: string; score: number } }>();
  let order = 0;
  $("a[href]").each((_, a) => {
    const $a = $(a);
    if ($a.closest("header, footer, nav, [role=navigation], form").length) return;
    const title = clean($a.text());
    if (title.length < 12) return;
    const url = resolveResultUrl($a.attr("href"), baseUrl, domains);
    if (!url) return;
    let score = 0;
    if ($a.closest("h1, h2, h3, h4").length || $a.find("h1, h2, h3, h4").length) score += 4;
    if (looksLikeUrlText(title)) score -= 4;
    if (title.length > 160) score -= 2;
    const entry = byUrl.get(url);
    if (!entry) byUrl.set(url, { order: order++, best: { el: a as Element, title, score } });
    else if (score > entry.best.score) entry.best = { el: a as Element, title, score };
  });
  const picked = [...byUrl.entries()].sort((x, y) => x[1].order - y[1].order).slice(0, limit);
  return picked.map(([url, { best }]) => {
    let snippet = "";
    let parent = $(best.el).parent();
    for (let i = 0; i < 4 && parent.length; i++) {
      const text = clean(parent.text());
      if (text.length >= best.title.length + 40) {
        snippet = clean(text.replace(best.title, "")).slice(0, 300);
        break;
      }
      parent = parent.parent();
    }
    return { title: best.title.slice(0, 200), url, snippet, el: best.el };
  });
}

export function heuristicResults(html: string, baseUrl: string, domains: string[], limit = 10): SearchResult[] {
  return collectOutboundLinks(load(html), baseUrl, domains, limit).map(({ title, url, snippet }) => ({
    title,
    url,
    snippet,
  }));
}

const CAPTCHA_SELECTORS = [
  "#anomaly-modal",
  ".anomaly-modal",
  ".g-recaptcha",
  ".h-captcha",
  ".cf-turnstile",
  "#challenge-form",
  "#challenge-stage",
  "iframe[src*='captcha']",
  "iframe[src*='challenges.cloudflare.com']",
  "form[action*='captcha']",
].join(", ");

const CAPTCHA_TEXT =
  /verify (that )?you(?:'re|’re| are) (a )?human|are you a robot|not a robot|unusual traffic from your|bots use duckduckgo too|complete the following challenge|select all (the )?(images|squares)|checking (if the site connection is secure|your browser)|just a moment\.\.\./i;

const BLOCKED_TEXT =
  /automated queries|access denied|too many requests|temporarily (blocked|unavailable)|request (has been )?blocked|403 forbidden/i;

const NO_RESULTS_TEXT = /no results (found|for)|did not match any|no web results|there are no results/i;

export function looksLikeCaptcha(page: PageData): boolean {
  if (/captcha|\/challenge/i.test(page.url)) return true;
  if (page.html) {
    try {
      if (load(page.html)(CAPTCHA_SELECTORS).length) return true;
    } catch {
      // fall through to the text check
    }
  }
  return CAPTCHA_TEXT.test(page.text.slice(0, 6000));
}

export interface Classification {
  kind: PageKind;
  results: SearchResult[];
}

export function classifySearchPage(
  page: PageData,
  rules: SelectorRules,
  domains: string[],
  noResultsSelector?: string
): Classification {
  if (!page.html && !page.text) return { kind: "empty", results: [] };
  const results = page.html ? applyRules(page.html, page.url, rules, domains) : [];
  if (results.length) return { kind: "ok", results };
  if (looksLikeCaptcha(page)) return { kind: "captcha", results: [] };
  let noResults = NO_RESULTS_TEXT.test(page.text.slice(0, 6000));
  if (!noResults && noResultsSelector && page.html) {
    try {
      noResults = load(page.html)(noResultsSelector).length > 0;
    } catch {
      // ignore
    }
  }
  if (noResults) return { kind: "no_results", results: [] };
  if (BLOCKED_TEXT.test(page.text.slice(0, 6000))) return { kind: "blocked", results: [] };
  return { kind: "parser_failed", results: page.html ? heuristicResults(page.html, page.url, domains) : [] };
}

function describe($: CheerioAPI, el: Element): string {
  const $el = $(el);
  const id = $el.attr("id");
  const classes = ($el.attr("class") ?? "").split(/\s+/).filter(Boolean).slice(0, 4);
  return `${el.tagName}${id ? `#${id}` : ""}${classes.map((c) => `.${c}`).join("")}`;
}

// A compact picture of how result links sit in the page, for writing new selectors.
export function layoutSample(html: string, baseUrl: string, domains: string[]): string {
  const $ = load(html);
  const links = collectOutboundLinks($, baseUrl, domains, 8);
  if (!links.length) return "No outbound result links were found on the page, so there is nothing to build selectors from.";
  const sample = links.slice(0, 6);
  const lines: string[] = [];
  const shared = new Map<string, { level: number; desc: string; elements: Set<Element>; holdsDescription: number }>();
  sample.forEach((link, i) => {
    const block = $(link.el).parents().eq(2);
    const candidate = block
      .find("p, span, div, a")
      .filter((_, e) => {
        if (e === link.el || $(e).find(link.el).length || $(e).closest("h1, h2, h3, h4").length || $(e).find("h1, h2, h3, h4").length) return false;
        const t = clean($(e).text());
        return t.length >= 40 && !t.includes(link.title) && !looksLikeUrlText(t) && $(e).children().length <= 3;
      })
      .first();
    const descEl = candidate.length ? (candidate[0] as Element) : null;

    const chain: string[] = [describe($, link.el)];
    let parent = $(link.el).parent();
    for (let level = 1; level <= 5 && parent.length; level++) {
      const node = parent[0] as Element;
      if (node.type !== "tag") break;
      const desc = describe($, node);
      chain.push(desc);
      const key = `${level}|${desc}`;
      const entry = shared.get(key) ?? { level, desc, elements: new Set<Element>(), holdsDescription: 0 };
      entry.elements.add(node);
      if (!descEl || $(node).find(descEl).length) entry.holdsDescription++;
      shared.set(key, entry);
      parent = parent.parent();
    }
    lines.push(`${i + 1}. title link "${link.title.slice(0, 70)}" -> ${link.url.slice(0, 90)}`);
    lines.push(`   title link and its parents, innermost first: ${chain.join(" < ")}`);
    if (descEl) lines.push(`   description candidate: ${describe($, descEl)} "${clean(candidate.text()).slice(0, 80)}"`);
  });
  // A good container appears once per result: the same descriptor, but a different
  // element around each sample link. One element around all of them is the list itself.
  const perResult = [...shared.values()]
    .filter((e) => e.elements.size >= Math.min(3, sample.length) && e.holdsDescription >= e.elements.size / 2)
    .sort((a, b) => a.level - b.level);
  const lists = [...shared.values()].filter((e) => e.elements.size === 1 && sample.length > 1).sort((a, b) => a.level - b.level);
  if (perResult.length) {
    lines.push("");
    lines.push("Elements that wrap each result separately (good container candidates, closest to the link first):");
    for (const e of perResult.slice(0, 4)) lines.push(`- ${e.desc} (${e.level} level(s) above the title link, one per result)`);
  }
  if (lists.length) {
    lines.push(`Element that wraps the whole list (not a container): ${lists[0].desc}`);
  }
  return lines.join("\n").slice(0, 4000);
}