rag-ultimate

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.compressChunk = compressChunk;
exports.compressChunks = compressChunks;
const text_1 = require("../utils/text");
function scoreLine(line) {
    const trimmed = line.trim();
    if (!trimmed)
        return 0;
    if (/^#{1,6}\s+/.test(trimmed))
        return 4;
    if (/^[-*•]\s+/.test(trimmed))
        return 3;
    if (/^\d+[.)]\s+/.test(trimmed))
        return 3;
    if (trimmed.length < 80 && /^[A-Z0-9][A-Za-z0-9\s()\-_:/.]+$/.test(trimmed))
        return 2;
    return 1;
}
function extractCompactSummary(text) {
    const normalized = (0, text_1.normalizeWhitespace)(text);
    const lines = text
        .split(/\r?\n/)
        .map(line => line.trim())
        .filter(Boolean);
    if (lines.length === 0)
        return (0, text_1.truncate)(normalized, 1200);
    const kept = [];
    const seen = new Set();
    for (const line of lines) {
        const key = (0, text_1.normalizeWhitespace)(line).toLowerCase();
        if (seen.has(key))
            continue;
        seen.add(key);
        if (scoreLine(line) >= 3) {
            kept.push(line);
        }
        if (kept.length >= 6)
            break;
    }
    if (kept.length < 2) {
        const paragraphs = normalized.split(/\n\s*\n/g).map(p => p.trim()).filter(Boolean);
        kept.push(...paragraphs.slice(0, 3));
    }
    return (0, text_1.truncate)(kept.join("\n"), 1400);
}
function compressChunk(text) {
    return extractCompactSummary(text);
}
function compressChunks(chunks) {
    return chunks.map(compressChunk);
}