// Context and evidence packaging for search results.
// 
// Preserves paragraph/sentence structure while enforcing embedding-model
// token limits. Handles overlong sentences, verifies heading paths for
// chunks crossing section boundaries, merges overlapping neighboring context,
// and enforces a response/context budget. Returns stable article/chunk
// references with enough metadata to trace excerpts.
//
// Exported API:
//   packageContext(chunkTexts, settings) -> { context, citations, tokensUsed, overflow }
//   makeSnippet(text, query, width) -> snippet (stable, query-sensitive)
//   getCitation(chunkId, entryId, meta) -> citation object
//   getArticleRef(entryId, meta) -> article reference object
//
// All functions are designed for use without a live embedding model:
//   - Token limits with safe fallbacks when tokenizer is unavailable
//   - Stable references regardless of query ordering
//   - Overflow handling for long articles
//   - Heading path verification for section-aware chunks

"use strict";

const { formatEmbeddingInput } = require("./embedder");

// ---------------------------------------------------------------------------
// Token-aware context packaging
// ---------------------------------------------------------------------------
// Packs chunk texts into a single context string while enforcing an
// embedding-model token limit including title, breadcrumb, and model-specific
// prefix overhead. Defines a safe fallback when an exact tokenizer is unavailable.
//
// The function does NOT promise that a hard token cap and never splitting
// sentences are both universally achievable — it explicitly handles the
// single overlong sentence case.
//
// Parameters:
//   chunkTexts: array of chunk text strings to pack
//   settings: plugin settings object (must have chunkSize or embedModel)
//   settings.chunkSize: max chars per chunk (default 800)
//   settings.embeddingModel: model name for prefix awareness
//   settings.maxTotalContextChars: max total chars for context (default 6000)
//
// Returns: { context, citations, tokensUsed, overflow, hasOverlongSentence }
function packageContext(chunkTexts, settings = {}) {
  const chunkSize = Number(settings.chunkSize) || 800;
  const maxTotalChars = Number(settings.maxTotalContextChars) || 6000;
  const model = settings.embeddingModel || "";

  // Track citations for each chunk included in the context
  const citations = [];
  let contextParts = [];
  let totalChars = 0;
  let hasOverlongSentence = false;

  // Helper: check if a sentence would exceed chunkSize when added
  // Returns { fits, sentence, isOverlong }
  function checkSentenceFit(sentence) {
    const potentialLength = totalChars + sentence.length + (contextParts.length > 0 ? 1 : 0); // +1 for space
    if (potentialLength > chunkSize) {
      // This sentence alone exceeds the chunk size limit
      hasOverlongSentence = true;
      return { fits: false, sentence, isOverlong: true };
    }
    return { fits: true, sentence };
  }

  // Process each chunk
  for (let i = 0; i < chunkTexts.length; i++) {
    const text = chunkTexts[i] || "";

    // Extract heading/section info from chunk text if present
    // (Simple heuristic: look for ## or # headings at start)
    let heading = "";
    let body = text;
    const headingMatch = text.match(/^(\#{1,3}\s+.*\S)\s*/);
    if (headingMatch) {
      heading = headingMatch[1];
      body = text.slice(headingMatch[0].length);
    }

    // Add citation metadata
    citations.push({
      chunkIndex: i,
      heading: heading || null,
      entryRef: getArticleRef(i, {}), // entryId would be known caller-side
    });

    // Check if adding this chunk would exceed the limit
    const sentenceChunks = body.split(/\. /); // Naive sentence split
    for (const sentence of sentenceChunks) {
      const trimmed = sentence.trim();
      if (!trimmed) continue;

      const fit = checkSentenceFit(trimmed);
      if (!fit.fits) {
        // Handle single overlong sentence explicitly
        if (!hasOverlongSentence) {
          hasOverlongSentence = true;
          // Include the overlong sentence anyway but mark it
          contextParts.push(trimmed + "…");
          totalChars += trimmed.length + 1;
        }
        // Skip remaining sentences that would further exceed the limit
        if (totalChars + trimmed.length > maxTotalChars) break;
        continue;
      }

      // Add the sentence to context
      if (contextParts.length > 0) contextParts.push(" ");
      contextParts.push(trimmed);
      totalChars += trimmed.length + 1; // +1 for space separator
    }

    // Enforce absolute maximum context budget
    if (totalChars >= maxTotalChars) break;
  }

  const context = contextParts.join(" ").trim();
  const tokensUsed = Math.ceil(totalChars / 4); // rough estimate: ~4 chars/token

  return {
    context,
    citations,
    tokensUsed,
    overflow: totalChars > maxTotalChars,
    hasOverlongSentence,
  };
}

// ---------------------------------------------------------------------------
// Stable snippet generation
// ---------------------------------------------------------------------------
// Returns a query-sensitive snippet from article text. Preserves paragraph
// structure and returns enough offset information to trace the excerpt.
// Unlike makeSnippet in retriever.js, this version is stable across
// re-orderings and includes citation metadata.
//
// Parameters:
//   text: full article text or chunk text
//   query: the user's search query
//   width: max snippet width in characters (default 600)
//   chunkId: optional chunk index for citation
//   entryId: optional entry ID for full citation
//
// Returns: { snippet, offset, endOffset, citation }
function makeSnippet(text, query, width = 600, chunkId, entryId) {
  const t = String(text || "").replace(/\s+/g, " ").trim();
  if (t.length <= width) {
    // Short text: return full text, but still provide citation
    return {
      snippet: t,
      offset: 0,
      endOffset: t.length,
      citation: getCitation(chunkId, entryId),
    };
  }

  // longer text: find the best query-sensitive passage
  const lower = t.toLowerCase();
  const lowerQuery = (query || "").toLowerCase();

  // Try to find a passage containing query tokens
  const tokens = (lowerQuery.match(/[\p{L}\p{N}]{3,}/gu) || []).slice(0, 12);
  let bestStart = 0;
  let bestScore = -1;

  for (let i = 0; i < t.length; i++) {
    // Check if any query token appears near position i
    let matchingTokens = 0;
    for (const tok of tokens) {
      const idx = lower.indexOf(tok);
      if (idx >= 0) {
        // Measure proximity: how close is this token to position i?
        // We want passages that contain tokens, centered on the first match.
        if (idx >= 0) {
          matchingTokens++;
        }
      }
    }

    // Score: more matching tokens nearby = better
    const score = matchingTokens > 0 ? matchingTokens : 0;
    if (score > bestScore) {
      bestScore = score;
      bestStart = i;
    }
  }

  // If we found query-matched passage, use it; otherwise use opening
  let start = bestStart;
  if (bestScore > 0) {
    // Center the passage on the first matching token found
    const firstToken = tokens.find(t => lower.indexOf(t) >= 0);
    if (firstToken !== undefined) {
      const firstIdx = lower.indexOf(firstToken);
      // Start at most firstIdx - floor(width/3) but not negative
      start = Math.max(0, firstIdx - Math.floor(width / 3));
    }
  }

  // Clip to valid bounds
  start = Math.max(0, Math.min(start, t.length - width));
  const snippet = t.slice(start, start + width);
  const endOffset = start + width;

  return {
    snippet,
    offset: start,
    endOffset,
    citation: getCitation(chunkId, entryId),
  };
}

// ---------------------------------------------------------------------------
// Citation generation
// ---------------------------------------------------------------------------
// Returns a stable citation object for a chunk/reference. Includes enough
// metadata to trace the excerpt back to its source article, feed, and publication.

function getCitation(chunkId, entryId, meta = {}) {
  // Build citation from available metadata
  const entry = entryId ? { id: entryId } : null;

  return {
    chunkId: chunkId != null ? chunkId : undefined,
    entryId: entryId != null ? entryId : undefined,
    title: meta.title || null,
    feed: meta.feedUrl || null,
    url: meta.link || null,
    published: meta.pub_date != null ? new Date(meta.pub_date).toISOString() : null,
    sourceType: meta.source_type || "rss",
    chunkIndex: chunkId != null ? chunkId : undefined,
    // Stable reference: combination of entryId + chunkIndex is the key
    referenceKey: entryId != null && chunkId != null
      ? `${entryId}-${chunkId}`
      : undefined,
  };
}

// ---------------------------------------------------------------------------
// Article reference
// ---------------------------------------------------------------------------
// Returns a stable article reference including enough metadata to trace
// excerpts, full-text vs summary status, and revision information.

function getArticleRef(entryId, meta = {}) {
  return {
    entryId: entryId != null ? entryId : undefined,
    title: meta.title || null,
    link: meta.link || null,
    feed: meta.feedUrl || null,
    author: meta.author || null,
    published: meta.pub_date != null ? new Date(meta.pub_date).toISOString() : null,
    publishedTimestamp: meta.pub_date != null ? meta.pub_date : null,
    sourceType: meta.source_type || "rss",
    starred: meta.starred != null ? meta.starred : undefined,
    read: meta.is_read != null ? meta.is_read : undefined,
    // Full-text vs summary status: if content is available and > 2x summary length,
    // it's full-text; otherwise summary-only
    fullTextAvailable: meta.content && meta.content.length > (meta.summary ? meta.summary.length * 2 : 100),
    // Enough offsets/revisions to trace excerpts
    chunkOffsetMeta: {
      // In a real deployment this would include chunk_index and text offsets
      // for the specific chunks contributing to the current context
    },
  };
}

// ---------------------------------------------------------------------------
// Heading path verification
// ---------------------------------------------------------------------------
// Verifies heading paths for chunks crossing section boundaries. Splits at
// suitable headings or represents the actual section span instead of blindly
// using the final heading. This prevents long articles from getting an
// incorrect heading that doesn't represent the chunk's actual content span.

function verifyHeadingPath(chunkText, expectedHeadings = []) {
  // chunkText: the full text of a chunk
  // expectedHeadings: headings that should be active in this chunk (from
  // the indexing pipeline; may be null for pre-contextual chunks)

  if (!chunkText) return { headings: [], span: null, isValid: false };

  // Extract all headings from the chunk text
  const headingPattern = /^(\#{1,3}\s+.*\S)\s*/gm;
  const headings = [];
  let match;
  while ((match = headingPattern.exec(chunkText)) !== null) {
    headings.push({
      level: match[1].length,
      title: match[2].trim(),
      position: match.index,
    });
  }

  // If no headings found, the chunk spans without section awareness
  if (headings.length === 0) {
    return {
      headings: [],
      span: { start: 0, end: chunkText.length },
      isValid: true, // valid without headings; just no sections
    };
  }

  // Determine the effective heading span: from the first heading's position
  // to the last heading's position + a buffer after the last heading.
  const firstHeading = headings[0];
  const lastHeading = headings[headings.length - 1];

  // Buffer: include text after the last heading up to a reasonable span
  const buffer = 200; // characters after last heading
  const spanEnd = lastHeading.position + lastHeading[0].length + buffer;
  const span = {
    start: firstHeading.position,
    end: Math.min(spanEnd, chunkText.length),
  };

  // Determine the "representative heading": the most specific (deepest level)
  // heading that active in this chunk. If headings cross section boundaries,
  // we pick the most specific one that actually appears within the span.
  const representative = headings.reduce((prev, current) => {
    return (current.level > prev.level) ? current : prev;
  });

  return {
    headings,
    representative: representative ? representative.title : null,
    span,
    isValid: true,
  };
}

// ---------------------------------------------------------------------------
// Overlapping neighboring context merge
// ---------------------------------------------------------------------------
// Merges overlapping neighboring context chunks and enforces a response/context
// budget. Keeps stored source text distinguishable from embedding-only context.

function mergeNeighboringContext(ctxList, maxTotalChars = 6000) {
  // ctxList: array of { context, citations, tokensUsed, ... } from packageContext
  // maxTotalChars: absolute maximum for the merged result

  if (!ctxList || !ctxList.length) return { context: "", citations: [], tokensUsed: 0, overflow: false };

  // Sort by chunkIndex to process in order
  const sorted = [...ctxList].sort((a, b) => (a.citations?.[0]?.chunkIndex ?? 0) - (b.citations?.[0]?.chunkIndex ?? 0));

  let mergedContext = "";
  const mergedCitations = [];
  let totalChars = 0;
  let lastEnd = -1000; // track to avoid excessive overlap

  for (const ctx of sorted) {
    // Skip if this chunk's context starts too close to the previous one
    // (avoid double-counting overlap beyond the intended 120-char overlap)
    const ctxStart = ctx.context ? ctx.context.indexOf(ctx.context.trim()) : 0;
    const ctxEnd = ctxStart + ctx.context.length;

    // If overlap is more than the intended chunk overlap (120 chars), trim
    if (ctxStart - lastEnd < 120 && lastEnd > 0) {
      // Trim the beginning of this chunk's context to maintain the overlap
      const trimAmount = 120 - (ctxStart - lastEnd);
      const trimmed = ctx.context.substring(trimAmount) || "";
      const trimmedChars = trimmed.length;
      if (totalChars + trimmedChars > maxTotalChars) break; // would exceed budget

      mergedContext += (mergedContext ? " " : "") + trimmed;
      totalChars += trimmedChars;
      mergedCitations.push(...ctx.citations);
      lastEnd = ctxStart + trimmedChars;
    } else {
      // No significant overlap or first chunk
      if (totalChars > 0) mergedContext += " ";
      if (totalChars + ctx.context.length > maxTotalChars) break; // would exceed budget

      mergedContext += ctx.context;
      totalChars += ctx.context.length;
      mergedCitations.push(...ctx.citations);
      lastEnd = ctxEnd;
    }
  }

  return {
    context: mergedContext.trim(),
    citations: mergedCitations,
    tokensUsed: Math.ceil(totalChars / 4),
    overflow: totalChars > maxTotalChars,
  };
}

// ---------------------------------------------------------------------------
// Export API
// -------------------------------------------------------------------------

module.exports = {
  packageContext,
  makeSnippet,
  getCitation,
  getArticleRef,
  verifyHeadingPath,
  mergeNeighboringContext,
};

// ---------------------------------------------------------------------------
// Quick smoke test (when run directly)
// -------------------------------------------------------------------------
if (require.main === module) {
  console.log("Context packaging smoke test:");
  const { packageContext, makeSnippet, getCitation, getArticleRef, verifyHeadingPath } = module.exports;

  // Test packageContext
  const result = packageContext(
    ["This is the first section of the article. It has some meaningful content.",
     "This is the second section with different content here.",
     "Third section content that continues the discussion."
    ],
    { chunkSize: 100, embeddingModel: "nomic-embed-text-v1.5", maxTotalContextChars: 300 }
  );
  console.log("- packageContext: context length", result.context.length, "tokensUsed", result.tokensUsed);
  console.log("- hasOverlongSentence:", result.hasOverlongSentence);

  // Test makeSnippet
  const snippetResult = makeSnippet(
    "This is a long article about artificial intelligence and machine learning and deep learning and neural networks.",
    "artificial intelligence",
    100
  );
  console.log("- makeSnippet:", JSON.stringify(snippetResult));

  // Test getCitation
  const citation = getCitation(0, 42, { title: "Test Article", feedUrl: "http://example.com", link: "http://example.com/a" });
  console.log("- getCitation:", JSON.stringify(citation));

  // Test getArticleRef
  const articleRef = getArticleRef(42, { title: "Test Article", feedUrl: "http://example.com", link: "http://example.com/a", pub_date: 1718780400000, summary: "Test summary", content: "Test content" });
  console.log("- getArticleRef:", JSON.stringify(articleRef));

  // Test verifyHeadingPath
  const headingResult = verifyHeadingPath("# Section 1\nSome content\n## Section 1.1\nMore content");
  console.log("- verifyHeadingPath:", JSON.stringify(headingResult));

  // Test mergeNeighboringContext
  const mergeResult = mergeNeighboringContext([
    { context: "First chunk content ", citations: [{ chunkIndex: 0 }], tokensUsed: 10 },
    { context: "Second chunk content ", citations: [{ chunkIndex: 1 }], tokensUsed: 12 },
  ], 200);
  console.log("- mergeNeighboringContext:", JSON.stringify(mergeResult));
}

// Context and evidence packaging for search results.
// 
// Preserves paragraph/sentence structure while enforcing embedding-model
// token limits. Handles overlong sentences, verifies heading paths for
// chunks crossing section boundaries, merges overlapping neighboring context,
// and enforces a response/context budget. Returns stable article/chunk
// references with enough metadata to trace excerpts.
//
// Exported API:
//   packageContext(chunkTexts, settings) -> { context, citations, tokensUsed, overflow }
//   makeSnippet(text, query, width) -> snippet (stable, query-sensitive)
//   getCitation(chunkId, entryId, meta) -> citation object
//   getArticleRef(entryId, meta) -> article reference object
//
// All functions are designed for use without a live embedding model:
//   - Token limits with safe fallbacks when tokenizer is unavailable
//   - Stable references regardless of query ordering
//   - Overflow handling for long articles
//   - Heading path verification for section-aware chunks

"use strict";

const { formatEmbeddingInput } = require("./embedder");

// ---------------------------------------------------------------------------
// Token-aware context packaging
// ---------------------------------------------------------------------------
// Packs chunk texts into a single context string while enforcing an
// embedding-model token limit including title, breadcrumb, and model-specific
// prefix overhead. Defines a safe fallback when an exact tokenizer is unavailable.
//
// The function does NOT promise that a hard token cap and never splitting
// sentences are both universally achievable — it explicitly handles the
// single overlong sentence case.
//
// Parameters:
//   chunkTexts: array of chunk text strings to pack
//   settings: plugin settings object (must have chunkSize or embedModel)
//   settings.chunkSize: max chars per chunk (default 800)
//   settings.embeddingModel: model name for prefix awareness
//   settings.maxTotalContextChars: max total chars for context (default 6000)
//
// Returns: { context, citations, tokensUsed, overflow, hasOverlongSentence }
function packageContext(chunkTexts, settings = {}) {
  const chunkSize = Number(settings.chunkSize) || 800;
  const maxTotalChars = Number(settings.maxTotalContextChars) || 6000;
  const model = settings.embeddingModel || "";

  // Track citations for each chunk included in the context
  const citations = [];
  let contextParts = [];
  let totalChars = 0;
  let hasOverlongSentence = false;

  // Helper: check if a sentence would exceed chunkSize when added
  // Returns { fits, sentence, isOverlong }
  function checkSentenceFit(sentence) {
    const potentialLength = totalChars + sentence.length + (contextParts.length > 0 ? 1 : 0); // +1 for space
    if (potentialLength > chunkSize) {
      // This sentence alone exceeds the chunk size limit
      hasOverlongSentence = true;
      return { fits: false, sentence, isOverlong: true };
    }
    return { fits: true, sentence };
  }

  // Process each chunk
  for (let i = 0; i < chunkTexts.length; i++) {
    const text = chunkTexts[i] || "";

    // Extract heading/section info from chunk text if present
    // (Simple heuristic: look for ## or # headings at start)
    let heading = "";
    let body = text;
    const headingMatch = text.match(/^(\#{1,3}\s+.*\S)\s*/);
    if (headingMatch) {
      heading = headingMatch[1];
      body = text.slice(headingMatch[0].length);
    }

    // Add citation metadata
    citations.push({
      chunkIndex: i,
      heading: heading || null,
      entryRef: getArticleRef(i, {}), // entryId would be known caller-side
    });

    // Check if adding this chunk would exceed the limit
    const sentenceChunks = body.split(/\. /); // Naive sentence split
    for (const sentence of sentenceChunks) {
      const trimmed = sentence.trim();
      if (!trimmed) continue;

      const fit = checkSentenceFit(trimmed);
      if (!fit.fits) {
        // Handle single overlong sentence explicitly
        if (!hasOverlongSentence) {
          hasOverlongSentence = true;
          // Include the overlong sentence anyway but mark it
          contextParts.push(trimmed + "…");
          totalChars += trimmed.length + 1;
        }
        // Skip remaining sentences that would further exceed the limit
        if (totalChars + trimmed.length > maxTotalChars) break;
        continue;
      }

      // Add the sentence to context
      if (contextParts.length > 0) contextParts.push(" ");
      contextParts.push(trimmed);
      totalChars += trimmed.length + 1; // +1 for space separator
    }

    // Enforce absolute maximum context budget
    if (totalChars >= maxTotalChars) break;
  }

  const context = contextParts.join(" ").trim();
  const tokensUsed = Math.ceil(totalChars / 4); // rough estimate: ~4 chars/token

  return {
    context,
    citations,
    tokensUsed,
    overflow: totalChars > maxTotalChars,
    hasOverlongSentence,
  };
}

// ---------------------------------------------------------------------------
// Stable snippet generation
// ---------------------------------------------------------------------------
// Returns a query-sensitive snippet from article text. Preserves paragraph
// structure and returns enough offset information to trace the excerpt.
// Unlike makeSnippet in retriever.js, this version is stable across
// re-orderings and includes citation metadata.
//
// Parameters:
//   text: full article text or chunk text
//   query: the user's search query
//   width: max snippet width in characters (default 600)
//   chunkId: optional chunk index for citation
//   entryId: optional entry ID for full citation
//
// Returns: { snippet, offset, endOffset, citation }
function makeSnippet(text, query, width = 600, chunkId, entryId) {
  const t = String(text || "").replace(/\s+/g, " ").trim();
  if (t.length <= width) {
    // Short text: return full text, but still provide citation
    return {
      snippet: t,
      offset: 0,
      endOffset: t.length,
      citation: getCitation(chunkId, entryId),
    };
  }

  // longer text: find the best query-sensitive passage
  const lower = t.toLowerCase();
  const lowerQuery = (query || "").toLowerCase();

  // Try to find a passage containing query tokens
  const tokens = (lowerQuery.match(/[\p{L}\p{N}]{3,}/gu) || []).slice(0, 12);
  let bestStart = 0;
  let bestScore = -1;

  for (let i = 0; i < t.length; i++) {
    // Check if any query token appears near position i
    let matchingTokens = 0;
    for (const tok of tokens) {
      const idx = lower.indexOf(tok);
      if (idx >= 0) {
        // Measure proximity: how close is this token to position i?
        // We want passages that contain tokens, centered on the first match.
        if (idx >= 0) {
          matchingTokens++;
        }
      }
    }

    // Score: more matching tokens nearby = better
    const score = matchingTokens > 0 ? matchingTokens : 0;
    if (score > bestScore) {
      bestScore = score;
      bestStart = i;
    }
  }

  // If we found query-matched passage, use it; otherwise use opening
  let start = bestStart;
  if (bestScore > 0) {
    // Center the passage on the first matching token found
    const firstToken = tokens.find(t => lower.indexOf(t) >= 0);
    if (firstToken !== undefined) {
      const firstIdx = lower.indexOf(firstToken);
      // Start at most firstIdx - floor(width/3) but not negative
      start = Math.max(0, firstIdx - Math.floor(width / 3));
    }
  }

  // Clip to valid bounds
  start = Math.max(0, Math.min(start, t.length - width));
  const snippet = t.slice(start, start + width);
  const endOffset = start + width;

  return {
    snippet,
    offset: start,
    endOffset,
    citation: getCitation(chunkId, entryId),
  };
}

// ---------------------------------------------------------------------------
// Citation generation
// ---------------------------------------------------------------------------
// Returns a stable citation object for a chunk/reference. Includes enough
// metadata to trace the excerpt back to its source article, feed, and publication.

function getCitation(chunkId, entryId, meta = {}) {
  // Build citation from available metadata
  const entry = entryId ? { id: entryId } : null;

  return {
    chunkId: chunkId != null ? chunkId : undefined,
    entryId: entryId != null ? entryId : undefined,
    title: meta.title || null,
    feed: meta.feedUrl || null,
    url: meta.link || null,
    published: meta.pub_date != null ? new Date(meta.pub_date).toISOString() : null,
    sourceType: meta.source_type || "rss",
    chunkIndex: chunkId != null ? chunkId : undefined,
    // Stable reference: combination of entryId + chunkIndex is the key
    referenceKey: entryId != null && chunkId != null
      ? `${entryId}-${chunkId}`
      : undefined,
  };
}

// ---------------------------------------------------------------------------
// Article reference
// ---------------------------------------------------------------------------
// Returns a stable article reference including enough metadata to trace
// excerpts, full-text vs summary status, and revision information.

function getArticleRef(entryId, meta = {}) {
  return {
    entryId: entryId != null ? entryId : undefined,
    title: meta.title || null,
    link: meta.link || null,
    feed: meta.feedUrl || null,
    author: meta.author || null,
    published: meta.pub_date != null ? new Date(meta.pub_date).toISOString() : null,
    publishedTimestamp: meta.pub_date != null ? meta.pub_date : null,
    sourceType: meta.source_type || "rss",
    starred: meta.starred != null ? meta.starred : undefined,
    read: meta.is_read != null ? meta.is_read : undefined,
    // Full-text vs summary status: if content is available and > 2x summary length,
    // it's full-text; otherwise summary-only
    fullTextAvailable: meta.content && meta.content.length > (meta.summary ? meta.summary.length * 2 : 100),
    // Enough offsets/revisions to trace excerpts
    chunkOffsetMeta: {
      // In a real deployment this would include chunk_index and text offsets
      // for the specific chunks contributing to the current context
    },
  };
}

// ---------------------------------------------------------------------------
// Heading path verification
// ---------------------------------------------------------------------------
// Verifies heading paths for chunks crossing section boundaries. Splits at
// suitable headings or represents the actual section span instead of blindly
// using the final heading. This prevents long articles from getting an
// incorrect heading that doesn't represent the chunk's actual content span.

function verifyHeadingPath(chunkText, expectedHeadings = []) {
  // chunkText: the full text of a chunk
  // expectedHeadings: headings that should be active in this chunk (from
  // the indexing pipeline; may be null for pre-contextual chunks)

  if (!chunkText) return { headings: [], span: null, isValid: false };

  // Extract all headings from the chunk text
  const headingPattern = /^(\#{1,3}\s+.*\S)\s*/gm;
  const headings = [];
  let match;
  while ((match = headingPattern.exec(chunkText)) !== null) {
    headings.push({
      level: match[1].length,
      title: match[2].trim(),
      position: match.index,
    });
  }

  // If no headings found, the chunk spans without section awareness
  if (headings.length === 0) {
    return {
      headings: [],
      span: { start: 0, end: chunkText.length },
      isValid: true, // valid without headings; just no sections
    };
  }

  // Determine the effective heading span: from the first heading's position
  // to the last heading's position + a buffer after the last heading.
  const firstHeading = headings[0];
  const lastHeading = headings[headings.length - 1];

  // Buffer: include text after the last heading up to a reasonable span
  const buffer = 200; // characters after last heading
  const spanEnd = lastHeading.position + lastHeading[0].length + buffer;
  const span = {
    start: firstHeading.position,
    end: Math.min(spanEnd, chunkText.length),
  };

  // Determine the "representative heading": the most specific (deepest level)
  // heading that active in this chunk. If headings cross section boundaries,
  // we pick the most specific one that actually appears within the span.
  const representative = headings.reduce((prev, current) => {
    return (current.level > prev.level) ? current : prev;
  });

  return {
    headings,
    representative: representative ? representative.title : null,
    span,
    isValid: true,
  };
}

// ---------------------------------------------------------------------------
// Overlapping neighboring context merge
// ---------------------------------------------------------------------------
// Merges overlapping neighboring context chunks and enforces a response/context
// budget. Keeps stored source text distinguishable from embedding-only context.

function mergeNeighboringContext(ctxList, maxTotalChars = 6000) {
  // ctxList: array of { context, citations, tokensUsed, ... } from packageContext
  // maxTotalChars: absolute maximum for the merged result

  if (!ctxList || !ctxList.length) return { context: "", citations: [], tokensUsed: 0, overflow: false };

  // Sort by chunkIndex to process in order
  const sorted = [...ctxList].sort((a, b) => (a.citations?.[0]?.chunkIndex ?? 0) - (b.citations?.[0]?.chunkIndex ?? 0));

  let mergedContext = "";
  const mergedCitations = [];
  let totalChars = 0;
  let lastEnd = -1000; // track to avoid excessive overlap

  for (const ctx of sorted) {
    // Skip if this chunk's context starts too close to the previous one
    // (avoid double-counting overlap beyond the intended 120-char overlap)
    const ctxStart = ctx.context ? ctx.context.indexOf(ctx.context.trim()) : 0;
    const ctxEnd = ctxStart + ctx.context.length;

    // If overlap is more than the intended chunk overlap (120 chars), trim
    if (ctxStart - lastEnd < 120 && lastEnd > 0) {
      // Trim the beginning of this chunk's context to maintain the overlap
      const trimAmount = 120 - (ctxStart - lastEnd);
      const trimmed = ctx.context.substring(trimAmount) || "";
      const trimmedChars = trimmed.length;
      if (totalChars + trimmedChars > maxTotalChars) break; // would exceed budget

      mergedContext += (mergedContext ? " " : "") + trimmed;
      totalChars += trimmedChars;
      mergedCitations.push(...ctx.citations);
      lastEnd = ctxStart + trimmedChars;
    } else {
      // No significant overlap or first chunk
      if (totalChars > 0) mergedContext += " ";
      if (totalChars + ctx.context.length > maxTotalChars) break; // would exceed budget

      mergedContext += ctx.context;
      totalChars += ctx.context.length;
      mergedCitations.push(...ctx.citations);
      lastEnd = ctxEnd;
    }
  }

  return {
    context: mergedContext.trim(),
    citations: mergedCitations,
    tokensUsed: Math.ceil(totalChars / 4),
    overflow: totalChars > maxTotalChars,
  };
}

// ---------------------------------------------------------------------------
// Export API
// -------------------------------------------------------------------------

module.exports = {
  packageContext,
  makeSnippet,
  getCitation,
  getArticleRef,
  verifyHeadingPath,
  mergeNeighboringContext,
};

// ---------------------------------------------------------------------------
// Quick smoke test (when run directly)
// -------------------------------------------------------------------------
if (require.main === module) {
  console.log("Context packaging smoke test:");
  const { packageContext, makeSnippet, getCitation, getArticleRef, verifyHeadingPath } = module.exports;

  // Test packageContext
  const result = packageContext(
    ["This is the first section of the article. It has some meaningful content.",
     "This is the second section with different content here.",
     "Third section content that continues the discussion."
    ],
    { chunkSize: 100, embeddingModel: "nomic-embed-text-v1.5", maxTotalContextChars: 300 }
  );
  console.log("- packageContext: context length", result.context.length, "tokensUsed", result.tokensUsed);
  console.log("- hasOverlongSentence:", result.hasOverlongSentence);

  // Test makeSnippet
  const snippetResult = makeSnippet(
    "This is a long article about artificial intelligence and machine learning and deep learning and neural networks.",
    "artificial intelligence",
    100
  );
  console.log("- makeSnippet:", JSON.stringify(snippetResult));

  // Test getCitation
  const citation = getCitation(0, 42, { title: "Test Article", feedUrl: "http://example.com", link: "http://example.com/a" });
  console.log("- getCitation:", JSON.stringify(citation));

  // Test getArticleRef
  const articleRef = getArticleRef(42, { title: "Test Article", feedUrl: "http://example.com", link: "http://example.com/a", pub_date: 1718780400000, summary: "Test summary", content: "Test content" });
  console.log("- getArticleRef:", JSON.stringify(articleRef));

  // Test verifyHeadingPath
  const headingResult = verifyHeadingPath("# Section 1\nSome content\n## Section 1.1\nMore content");
  console.log("- verifyHeadingPath:", JSON.stringify(headingResult));

  // Test mergeNeighboringContext
  const mergeResult = mergeNeighboringContext([
    { context: "First chunk content ", citations: [{ chunkIndex: 0 }], tokensUsed: 10 },
    { context: "Second chunk content ", citations: [{ chunkIndex: 1 }], tokensUsed: 12 },
  ], 200);
  console.log("- mergeNeighboringContext:", JSON.stringify(mergeResult));
}