data-sampler

"use strict";
/**
 * @file length.ts
 * @description Length-based sampling strategy.
 * Splits text into fixed-size chunks with optional overlap between consecutive chunks.
 * Returns chunks in document order.
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.sampleByLength = sampleByLength;
/**
 * Default chunk size in characters when no explicit value is provided.
 */
const DEFAULT_CHUNK_SIZE = 2000;
/**
 * Default overlap between consecutive chunks (0 = no overlap).
 */
const DEFAULT_OVERLAP = 0;
/**
 * Split text into fixed-size chunks with optional overlap.
 *
 * @param text - The input text to chunk.
 * @param config - Sampling configuration (chunkSize, overlap, maxChunks).
 * @returns An array of Chunks in document order, each with uniform score of 0.
 */
function sampleByLength(text, config) {
    if (text.length === 0) {
        return [];
    }
    const chunkSize = Math.max(1, config.chunkSize ?? DEFAULT_CHUNK_SIZE);
    const overlap = Math.max(0, config.overlap ?? DEFAULT_OVERLAP);
    const maxChunks = config.maxChunks ?? Number.MAX_SAFE_INTEGER;
    const chunks = [];
    const step = Math.max(1, chunkSize - overlap);
    let offset = 0;
    let index = 0;
    while (offset < text.length && chunks.length < maxChunks) {
        const end = Math.min(offset + chunkSize, text.length);
        const chunkText = text.slice(offset, end);
        chunks.push({
            index,
            text: chunkText,
            score: 0,
            startOffset: offset,
            endOffset: end,
        });
        // If the remaining text fits in one more chunk of the same size,
        // emit it and stop — avoids a tiny tail chunk when overlap is 0.
        if (end >= text.length) {
            break;
        }
        offset += step;
        index++;
    }
    // If we hit maxChunks before exhausting the text, we still produced
    // a partial output. That is correct — maxChunks is a cap.
    return chunks;
}
//# sourceMappingURL=length.js.map