import { Readability } from "@mozilla/readability";
import { JSDOM, VirtualConsole } from "jsdom";
import TurndownService from "turndown";
import { ExtractedWebpage } from "../cache/WebpageCache";
import { extractJsonLd } from "./jsonLdExtractor";
import { extractOpenGraph } from "./openGraphExtractor";
import { detectRssFeedUrl } from "./rssExtractor";
import { extractMainContentImageMarkers } from "./imageExtractor";

export const MAX_RAW_HTML_BYTES = 10 * 1024 * 1024; // 10 MB safeguard limit

/**
 * Creates a configured VirtualConsole for JSDOM that suppresses noisy stylesheet parsing
 * errors (which are irrelevant for content extraction) while forwarding all other JSDOM diagnostics.
 */
export function createExtractionVirtualConsole(): VirtualConsole {
  const virtualConsole = new VirtualConsole();
  virtualConsole.sendTo(console, { omitJSDOMErrors: true });

  virtualConsole.on("jsdomError", (error: any) => {
    if (error?.type === "css parsing" || error?.type === "css @import URL parsing") {
      // Suppress CSSOM parser failures — stylesheets are never used for content extraction
      return;
    }
    // Forward all other real JSDOM errors/diagnostics to console.error
    console.error(error?.stack || error?.message || error, error?.detail);
  });

  return virtualConsole;
}

/**
 * Resolves all relative image src and anchor href attributes in the DOM document to absolute URLs against baseUrl.
 */
export function resolveRelativeUrls(doc: Document, baseUrl: string): void {
  try {
    const images = doc.querySelectorAll("img");
    images.forEach((img) => {
      const src = img.getAttribute("src");
      if (src && !src.startsWith("data:") && !src.startsWith("blob:")) {
        try {
          const absoluteUrl = new URL(src, baseUrl).href;
          img.setAttribute("src", absoluteUrl);
        } catch {
          // Ignore invalid URL resolution
        }
      }
    });

    const links = doc.querySelectorAll("a");
    links.forEach((a) => {
      const href = a.getAttribute("href");
      if (
        href &&
        !href.startsWith("#") &&
        !href.startsWith("javascript:") &&
        !href.startsWith("mailto:")
      ) {
        try {
          const absoluteUrl = new URL(href, baseUrl).href;
          a.setAttribute("href", absoluteUrl);
        } catch {
          // Ignore invalid URL resolution
        }
      }
    });
  } catch {
    // Fail-safe URL resolution
  }
}

/**
 * GFM Table conversion rules for Turndown to preserve structured tables (tables, tr, th, td).
 */
export function addGfmTableRules(turndownService: TurndownService): void {
  turndownService.addRule("tableCell", {
    filter: ["th", "td"],
    replacement: (content, node) => {
      const cleanText = content.trim().replace(/\s*\n\s*/g, " ");
      return ` ${cleanText} |`;
    },
  });

  turndownService.addRule("tableRow", {
    filter: "tr",
    replacement: (content, node) => {
      const parent = node.parentNode;
      const isHeader =
        (parent && parent.nodeName === "THEAD") ||
        (node as Element).querySelector("th") !== null;

      let row = `|${content}\n`;

      if (isHeader) {
        const cells = (node as Element).querySelectorAll("th, td");
        const divider = "|" + Array.from(cells).map(() => " --- |").join("") + "\n";
        row += divider;
      }

      return row;
    },
  });

  turndownService.addRule("tableSection", {
    filter: ["thead", "tbody", "tfoot"],
    replacement: (content) => content,
  });

  turndownService.addRule("table", {
    filter: "table",
    replacement: (content) => {
      const cleanTable = content.trim();
      return cleanTable ? `\n\n${cleanTable}\n\n` : "";
    },
  });
}

/**
 * Creates and configures a TurndownService instance optimized for LLM token efficiency.
 * Emits clean visible text for links and image descriptions without redundant Markdown syntax bloat.
 */
export function createTurndownService(): TurndownService {
  const turndownService = new TurndownService({
    headingStyle: "atx",
    codeBlockStyle: "fenced",
    bulletListMarker: "-",
    hr: "---",
  });

  // Ensure script, style, noscript, iframe, and svg tags are removed completely
  turndownService.remove((node) => {
    const tag = node.nodeName.toLowerCase();
    return ["script", "style", "noscript", "iframe", "svg"].includes(tag);
  });

  // Add GFM table support to preserve structured table data
  addGfmTableRules(turndownService);

  // Image Serialization Rule:
  // Decorative or empty images -> strip completely. Meaningful images -> preserve clean description block.
  turndownService.addRule("cleanImages", {
    filter: "img",
    replacement: (content, node) => {
      const element = node as Element;
      const alt = element.getAttribute("alt")?.trim();
      const title = element.getAttribute("title")?.trim();
      const figure = element.closest("figure");
      const captionNode = figure?.querySelector("figcaption");
      const caption = captionNode?.textContent?.trim();

      const cleanAlt = alt && !["image", "photo", "picture", "icon"].includes(alt.toLowerCase()) ? alt : undefined;
      const cleanCaption = caption || title || undefined;
      const description = cleanAlt || cleanCaption;

      if (!description || description.toLowerCase() === "image" || description.toLowerCase() === "icon") {
        return "";
      }
      const cleanDesc = description.replace(/\s+/g, " ").trim();
      if (!cleanDesc) {
        return "";
      }
      return `\n\n> **Image:** ${cleanDesc}\n\n`;
    },
  });

  // Link Serialization Rule:
  // Browse feeds Markdown directly to an LLM. Hyperlink destinations are stripped
  // to output only the visible anchor text (e.g. "python.org", "Guido van Rossum", "[1]"),
  // maximizing token information density without redundant [text](url) syntax.
  turndownService.addRule("cleanLinks", {
    filter: "a",
    replacement: (content) => {
      return content.trim();
    },
  });

  // Turndown's default escaping is optimized for producing conservative,
  // human-facing Markdown. Browse feeds Markdown directly to an
  // LLM, so we intentionally reduce unnecessary escaping to maximize
  // information density while preserving valid Markdown.
  turndownService.escape = function (string: string): string {
    // Only escape backslashes and characters that would form ambiguous markdown headings/lists at line starts
    return string
      .replace(/\\/g, "\\\\")
      .replace(/^(\s*)([#\-+*])/gm, "$1\\$2");
  };

  return turndownService;
}

/**
 * Normalizes Markdown whitespace to maximize LLM token efficiency.
 * Collapses 3 or more consecutive newlines to 2, strips trailing whitespace, and normalizes line endings.
 */
export function normalizeWhitespace(markdown: string): string {
  if (!markdown) return "";
  return markdown
    .replace(/\r\n/g, "\n")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
}

/**
 * Checks if markdown body content is sparse (< 600 chars or < 3 paragraphs).
 */
export function isContentSparse(markdown: string): boolean {
  const clean = normalizeWhitespace(markdown);
  if (clean.length < 600) return true;
  const paragraphs = clean.split(/\n\s*\n/).filter((p) => p.trim().length > 0);
  return paragraphs.length < 3;
}

/**
 * Formats a clean metadata header block at top of body markdown,
 * excluding metadata fields already present in top 500 chars of body.
 */
export function formatDocumentMarkdown(
  metadata: { title?: string; author?: string; publishedDate?: string; site?: string },
  bodyMarkdown: string
): string {
  const cleanBody = normalizeWhitespace(bodyMarkdown);
  const topPrefix = cleanBody.substring(0, 500).toLowerCase();
  const headerLines: string[] = [];

  const title = metadata.title?.trim();
  if (title && !topPrefix.includes(title.toLowerCase())) {
    headerLines.push(`# ${title}`);
  }

  const metaDetails: string[] = [];

  const author = metadata.author?.trim();
  if (author && !topPrefix.includes(author.toLowerCase())) {
    metaDetails.push(`**Author:** ${author}`);
  }

  const publishedDate = metadata.publishedDate?.trim();
  if (publishedDate && !topPrefix.includes(publishedDate.toLowerCase())) {
    metaDetails.push(`**Published:** ${publishedDate}`);
  }

  const site = metadata.site?.trim();
  if (site && !topPrefix.includes(site.toLowerCase())) {
    metaDetails.push(`**Site:** ${site}`);
  }

  if (metaDetails.length > 0) {
    headerLines.push(metaDetails.join(" | "));
  }

  if (headerLines.length > 0) {
    return normalizeWhitespace(`${headerLines.join("\n\n")}\n\n${cleanBody}`);
  }

  return cleanBody;
}

/**
 * Synchronous HTML content extractor.
 * Parses HTML, runs Readability extraction, extracts JSON-LD & Open Graph metadata,
 * detects RSS feed URLs, and evaluates content sparsity.
 */
export function extractHtmlContent(html: string, baseUrl: string): ExtractedWebpage {
  const rawSizeBytes = Buffer.byteLength(html || "", "utf-8");
  if (rawSizeBytes > MAX_RAW_HTML_BYTES) {
    const sizeMb = (rawSizeBytes / (1024 * 1024)).toFixed(2);
    throw new Error(
      `Webpage raw HTML size (${sizeMb} MB) exceeds the maximum supported size (10 MB).`
    );
  }

  const dom = new JSDOM(html, {
    url: baseUrl,
    virtualConsole: createExtractionVirtualConsole(),
  });
  const doc = dom.window.document;

  // Detect HTML lang attribute
  const lang =
    doc.documentElement.getAttribute("lang") ||
    doc.documentElement.getAttribute("xml:lang") ||
    undefined;

  // Resolve relative URLs in DOM
  resolveRelativeUrls(doc, baseUrl);

  // Extract structured JSON-LD metadata & Open Graph metadata
  const jsonLd = extractJsonLd(doc);
  const og = extractOpenGraph(doc);
  const rssFeedUrl = detectRssFeedUrl(doc, baseUrl);

  // Primary HTML Readability extraction
  const reader = new Readability(doc);
  const article = reader.parse();

  let articleDoc: Document | null = null;
  let primaryBodyMarkdown = "";

  if (article && article.content) {
    const articleDom = new JSDOM(article.content, {
      url: baseUrl,
      virtualConsole: createExtractionVirtualConsole(),
    });
    articleDoc = articleDom.window.document;

    const turndownService = createTurndownService();
    primaryBodyMarkdown = turndownService.turndown(article.content);
  }

  primaryBodyMarkdown = normalizeWhitespace(primaryBodyMarkdown);

  // Evaluate primary content sparsity
  let bodyMarkdown = primaryBodyMarkdown;
  const isSparseInitially = isContentSparse(bodyMarkdown);

  // Fallback 1: JSON-LD articleBody if primary extraction is sparse and articleBody is substantial
  if (isSparseInitially && jsonLd.articleBody && jsonLd.articleBody.length >= 100) {
    bodyMarkdown = normalizeWhitespace(jsonLd.articleBody);
  }

  if (!bodyMarkdown) {
    throw new Error(
      "No readable content could be extracted from this webpage. (Possible reasons: JavaScript-rendered page, authentication/paywall login required, or non-standard DOM structure.)"
    );
  }

  // Consolidated metadata merging (priority: JSON-LD > OG > Readability)
  const metadata = {
    site: jsonLd.publisher || og.siteName || article?.siteName?.trim() || undefined,
    title: jsonLd.headline || og.title || article?.title?.trim() || doc.title?.trim() || undefined,
    author: jsonLd.author || og.author || article?.byline?.trim() || undefined,
    publishedDate: jsonLd.datePublished || og.publishedDate || article?.publishedTime?.trim() || undefined,
    language: lang?.trim() || undefined,
    excerpt: jsonLd.description || og.description || article?.excerpt?.trim() || undefined,
  };

  const finalIsSparse = isContentSparse(bodyMarkdown);
  const fullMarkdown = formatDocumentMarkdown(metadata, bodyMarkdown);

  const hasJsonLd = Boolean(
    jsonLd.headline || jsonLd.articleBody || jsonLd.author || jsonLd.datePublished || jsonLd.description
  );
  const hasOg = Boolean(
    og.title || og.description || og.publishedDate || og.author || og.siteName
  );
  const hasTables = fullMarkdown.includes("| ---") || fullMarkdown.includes("| --- |");

  // Extract structured main content image markers strictly from articleDoc (Single Source of Truth)
  const imageExtractRes = articleDoc
    ? extractMainContentImageMarkers(articleDoc, baseUrl)
    : { markers: [], foundCount: 0, filteredCount: 0 };

  return {
    url: baseUrl,
    metadata,
    fullMarkdown,
    totalChars: fullMarkdown.length,
    isSparse: finalIsSparse,
    rssFeedUrl,
    rawHtmlLength: rawSizeBytes,
    readabilityLength: primaryBodyMarkdown.length,
    usedJsonLd: hasJsonLd,
    usedOg: hasOg,
    tablesDetected: hasTables,
    imageMarkers: imageExtractRes.markers,
    imagesFoundCount: imageExtractRes.foundCount,
    imagesFilteredCount: imageExtractRes.filteredCount,
  };
}

/**
 * Evaluates whether a destination URL is visibly presented to the reader as part of the webpage's content.
 */
export function isSemanticUrl(content: string, href: string): boolean {
  if (!href || !href.trim()) return false;
  const cleanHref = href.trim().toLowerCase();
  const cleanContent = content.trim().toLowerCase();

  return (
    cleanContent.startsWith("http://") ||
    cleanContent.startsWith("https://") ||
    cleanContent.startsWith("ftp://") ||
    cleanContent === cleanHref ||
    /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\/[^\s]*)?$/i.test(cleanContent)
  );
}

import { Readability } from "@mozilla/readability";
import { JSDOM, VirtualConsole } from "jsdom";
import TurndownService from "turndown";
import { ExtractedWebpage } from "../cache/WebpageCache";
import { extractJsonLd } from "./jsonLdExtractor";
import { extractOpenGraph } from "./openGraphExtractor";
import { detectRssFeedUrl } from "./rssExtractor";
import { extractMainContentImageMarkers } from "./imageExtractor";

export const MAX_RAW_HTML_BYTES = 10 * 1024 * 1024; // 10 MB safeguard limit

/**
 * Creates a configured VirtualConsole for JSDOM that suppresses noisy stylesheet parsing
 * errors (which are irrelevant for content extraction) while forwarding all other JSDOM diagnostics.
 */
export function createExtractionVirtualConsole(): VirtualConsole {
  const virtualConsole = new VirtualConsole();
  virtualConsole.sendTo(console, { omitJSDOMErrors: true });

  virtualConsole.on("jsdomError", (error: any) => {
    if (error?.type === "css parsing" || error?.type === "css @import URL parsing") {
      // Suppress CSSOM parser failures — stylesheets are never used for content extraction
      return;
    }
    // Forward all other real JSDOM errors/diagnostics to console.error
    console.error(error?.stack || error?.message || error, error?.detail);
  });

  return virtualConsole;
}

/**
 * Resolves all relative image src and anchor href attributes in the DOM document to absolute URLs against baseUrl.
 */
export function resolveRelativeUrls(doc: Document, baseUrl: string): void {
  try {
    const images = doc.querySelectorAll("img");
    images.forEach((img) => {
      const src = img.getAttribute("src");
      if (src && !src.startsWith("data:") && !src.startsWith("blob:")) {
        try {
          const absoluteUrl = new URL(src, baseUrl).href;
          img.setAttribute("src", absoluteUrl);
        } catch {
          // Ignore invalid URL resolution
        }
      }
    });

    const links = doc.querySelectorAll("a");
    links.forEach((a) => {
      const href = a.getAttribute("href");
      if (
        href &&
        !href.startsWith("#") &&
        !href.startsWith("javascript:") &&
        !href.startsWith("mailto:")
      ) {
        try {
          const absoluteUrl = new URL(href, baseUrl).href;
          a.setAttribute("href", absoluteUrl);
        } catch {
          // Ignore invalid URL resolution
        }
      }
    });
  } catch {
    // Fail-safe URL resolution
  }
}

/**
 * GFM Table conversion rules for Turndown to preserve structured tables (tables, tr, th, td).
 */
export function addGfmTableRules(turndownService: TurndownService): void {
  turndownService.addRule("tableCell", {
    filter: ["th", "td"],
    replacement: (content, node) => {
      const cleanText = content.trim().replace(/\s*\n\s*/g, " ");
      return ` ${cleanText} |`;
    },
  });

  turndownService.addRule("tableRow", {
    filter: "tr",
    replacement: (content, node) => {
      const parent = node.parentNode;
      const isHeader =
        (parent && parent.nodeName === "THEAD") ||
        (node as Element).querySelector("th") !== null;

      let row = `|${content}\n`;

      if (isHeader) {
        const cells = (node as Element).querySelectorAll("th, td");
        const divider = "|" + Array.from(cells).map(() => " --- |").join("") + "\n";
        row += divider;
      }

      return row;
    },
  });

  turndownService.addRule("tableSection", {
    filter: ["thead", "tbody", "tfoot"],
    replacement: (content) => content,
  });

  turndownService.addRule("table", {
    filter: "table",
    replacement: (content) => {
      const cleanTable = content.trim();
      return cleanTable ? `\n\n${cleanTable}\n\n` : "";
    },
  });
}

/**
 * Creates and configures a TurndownService instance optimized for LLM token efficiency.
 * Emits clean visible text for links and image descriptions without redundant Markdown syntax bloat.
 */
export function createTurndownService(): TurndownService {
  const turndownService = new TurndownService({
    headingStyle: "atx",
    codeBlockStyle: "fenced",
    bulletListMarker: "-",
    hr: "---",
  });

  // Ensure script, style, noscript, iframe, and svg tags are removed completely
  turndownService.remove((node) => {
    const tag = node.nodeName.toLowerCase();
    return ["script", "style", "noscript", "iframe", "svg"].includes(tag);
  });

  // Add GFM table support to preserve structured table data
  addGfmTableRules(turndownService);

  // Image Serialization Rule:
  // Decorative or empty images -> strip completely. Meaningful images -> preserve clean description block.
  turndownService.addRule("cleanImages", {
    filter: "img",
    replacement: (content, node) => {
      const element = node as Element;
      const alt = element.getAttribute("alt")?.trim();
      const title = element.getAttribute("title")?.trim();
      const figure = element.closest("figure");
      const captionNode = figure?.querySelector("figcaption");
      const caption = captionNode?.textContent?.trim();

      const cleanAlt = alt && !["image", "photo", "picture", "icon"].includes(alt.toLowerCase()) ? alt : undefined;
      const cleanCaption = caption || title || undefined;
      const description = cleanAlt || cleanCaption;

      if (!description || description.toLowerCase() === "image" || description.toLowerCase() === "icon") {
        return "";
      }
      const cleanDesc = description.replace(/\s+/g, " ").trim();
      if (!cleanDesc) {
        return "";
      }
      return `\n\n> **Image:** ${cleanDesc}\n\n`;
    },
  });

  // Link Serialization Rule:
  // Browse feeds Markdown directly to an LLM. Hyperlink destinations are stripped
  // to output only the visible anchor text (e.g. "python.org", "Guido van Rossum", "[1]"),
  // maximizing token information density without redundant [text](url) syntax.
  turndownService.addRule("cleanLinks", {
    filter: "a",
    replacement: (content) => {
      return content.trim();
    },
  });

  // Turndown's default escaping is optimized for producing conservative,
  // human-facing Markdown. Browse feeds Markdown directly to an
  // LLM, so we intentionally reduce unnecessary escaping to maximize
  // information density while preserving valid Markdown.
  turndownService.escape = function (string: string): string {
    // Only escape backslashes and characters that would form ambiguous markdown headings/lists at line starts
    return string
      .replace(/\\/g, "\\\\")
      .replace(/^(\s*)([#\-+*])/gm, "$1\\$2");
  };

  return turndownService;
}

/**
 * Normalizes Markdown whitespace to maximize LLM token efficiency.
 * Collapses 3 or more consecutive newlines to 2, strips trailing whitespace, and normalizes line endings.
 */
export function normalizeWhitespace(markdown: string): string {
  if (!markdown) return "";
  return markdown
    .replace(/\r\n/g, "\n")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
}

/**
 * Checks if markdown body content is sparse (< 600 chars or < 3 paragraphs).
 */
export function isContentSparse(markdown: string): boolean {
  const clean = normalizeWhitespace(markdown);
  if (clean.length < 600) return true;
  const paragraphs = clean.split(/\n\s*\n/).filter((p) => p.trim().length > 0);
  return paragraphs.length < 3;
}

/**
 * Formats a clean metadata header block at top of body markdown,
 * excluding metadata fields already present in top 500 chars of body.
 */
export function formatDocumentMarkdown(
  metadata: { title?: string; author?: string; publishedDate?: string; site?: string },
  bodyMarkdown: string
): string {
  const cleanBody = normalizeWhitespace(bodyMarkdown);
  const topPrefix = cleanBody.substring(0, 500).toLowerCase();
  const headerLines: string[] = [];

  const title = metadata.title?.trim();
  if (title && !topPrefix.includes(title.toLowerCase())) {
    headerLines.push(`# ${title}`);
  }

  const metaDetails: string[] = [];

  const author = metadata.author?.trim();
  if (author && !topPrefix.includes(author.toLowerCase())) {
    metaDetails.push(`**Author:** ${author}`);
  }

  const publishedDate = metadata.publishedDate?.trim();
  if (publishedDate && !topPrefix.includes(publishedDate.toLowerCase())) {
    metaDetails.push(`**Published:** ${publishedDate}`);
  }

  const site = metadata.site?.trim();
  if (site && !topPrefix.includes(site.toLowerCase())) {
    metaDetails.push(`**Site:** ${site}`);
  }

  if (metaDetails.length > 0) {
    headerLines.push(metaDetails.join(" | "));
  }

  if (headerLines.length > 0) {
    return normalizeWhitespace(`${headerLines.join("\n\n")}\n\n${cleanBody}`);
  }

  return cleanBody;
}

/**
 * Synchronous HTML content extractor.
 * Parses HTML, runs Readability extraction, extracts JSON-LD & Open Graph metadata,
 * detects RSS feed URLs, and evaluates content sparsity.
 */
export function extractHtmlContent(html: string, baseUrl: string): ExtractedWebpage {
  const rawSizeBytes = Buffer.byteLength(html || "", "utf-8");
  if (rawSizeBytes > MAX_RAW_HTML_BYTES) {
    const sizeMb = (rawSizeBytes / (1024 * 1024)).toFixed(2);
    throw new Error(
      `Webpage raw HTML size (${sizeMb} MB) exceeds the maximum supported size (10 MB).`
    );
  }

  const dom = new JSDOM(html, {
    url: baseUrl,
    virtualConsole: createExtractionVirtualConsole(),
  });
  const doc = dom.window.document;

  // Detect HTML lang attribute
  const lang =
    doc.documentElement.getAttribute("lang") ||
    doc.documentElement.getAttribute("xml:lang") ||
    undefined;

  // Resolve relative URLs in DOM
  resolveRelativeUrls(doc, baseUrl);

  // Extract structured JSON-LD metadata & Open Graph metadata
  const jsonLd = extractJsonLd(doc);
  const og = extractOpenGraph(doc);
  const rssFeedUrl = detectRssFeedUrl(doc, baseUrl);

  // Primary HTML Readability extraction
  const reader = new Readability(doc);
  const article = reader.parse();

  let articleDoc: Document | null = null;
  let primaryBodyMarkdown = "";

  if (article && article.content) {
    const articleDom = new JSDOM(article.content, {
      url: baseUrl,
      virtualConsole: createExtractionVirtualConsole(),
    });
    articleDoc = articleDom.window.document;

    const turndownService = createTurndownService();
    primaryBodyMarkdown = turndownService.turndown(article.content);
  }

  primaryBodyMarkdown = normalizeWhitespace(primaryBodyMarkdown);

  // Evaluate primary content sparsity
  let bodyMarkdown = primaryBodyMarkdown;
  const isSparseInitially = isContentSparse(bodyMarkdown);

  // Fallback 1: JSON-LD articleBody if primary extraction is sparse and articleBody is substantial
  if (isSparseInitially && jsonLd.articleBody && jsonLd.articleBody.length >= 100) {
    bodyMarkdown = normalizeWhitespace(jsonLd.articleBody);
  }

  if (!bodyMarkdown) {
    throw new Error(
      "No readable content could be extracted from this webpage. (Possible reasons: JavaScript-rendered page, authentication/paywall login required, or non-standard DOM structure.)"
    );
  }

  // Consolidated metadata merging (priority: JSON-LD > OG > Readability)
  const metadata = {
    site: jsonLd.publisher || og.siteName || article?.siteName?.trim() || undefined,
    title: jsonLd.headline || og.title || article?.title?.trim() || doc.title?.trim() || undefined,
    author: jsonLd.author || og.author || article?.byline?.trim() || undefined,
    publishedDate: jsonLd.datePublished || og.publishedDate || article?.publishedTime?.trim() || undefined,
    language: lang?.trim() || undefined,
    excerpt: jsonLd.description || og.description || article?.excerpt?.trim() || undefined,
  };

  const finalIsSparse = isContentSparse(bodyMarkdown);
  const fullMarkdown = formatDocumentMarkdown(metadata, bodyMarkdown);

  const hasJsonLd = Boolean(
    jsonLd.headline || jsonLd.articleBody || jsonLd.author || jsonLd.datePublished || jsonLd.description
  );
  const hasOg = Boolean(
    og.title || og.description || og.publishedDate || og.author || og.siteName
  );
  const hasTables = fullMarkdown.includes("| ---") || fullMarkdown.includes("| --- |");

  // Extract structured main content image markers strictly from articleDoc (Single Source of Truth)
  const imageExtractRes = articleDoc
    ? extractMainContentImageMarkers(articleDoc, baseUrl)
    : { markers: [], foundCount: 0, filteredCount: 0 };

  return {
    url: baseUrl,
    metadata,
    fullMarkdown,
    totalChars: fullMarkdown.length,
    isSparse: finalIsSparse,
    rssFeedUrl,
    rawHtmlLength: rawSizeBytes,
    readabilityLength: primaryBodyMarkdown.length,
    usedJsonLd: hasJsonLd,
    usedOg: hasOg,
    tablesDetected: hasTables,
    imageMarkers: imageExtractRes.markers,
    imagesFoundCount: imageExtractRes.foundCount,
    imagesFilteredCount: imageExtractRes.filteredCount,
  };
}

/**
 * Evaluates whether a destination URL is visibly presented to the reader as part of the webpage's content.
 */
export function isSemanticUrl(content: string, href: string): boolean {
  if (!href || !href.trim()) return false;
  const cleanHref = href.trim().toLowerCase();
  const cleanContent = content.trim().toLowerCase();

  return (
    cleanContent.startsWith("http://") ||
    cleanContent.startsWith("https://") ||
    cleanContent.startsWith("ftp://") ||
    cleanContent === cleanHref ||
    /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\/[^\s]*)?$/i.test(cleanContent)
  );
}