src / textChunker.ts
export interface TextChunk {
number: number;
text: string;
startCharacter: number;
endCharacter: number;
}
/**
* Split extracted text into bounded chunks.
* Paragraph, line, and word boundaries are preferred when possible.
*/
export function chunkText(
source: string,
chunkSize: number,
overlap: number,
): TextChunk[] {
if (!Number.isInteger(chunkSize) || chunkSize < 100) {
throw new Error("Chunk size must be an integer of at least 100.");
}
if (
!Number.isInteger(overlap) ||
overlap < 0 ||
overlap >= chunkSize
) {
throw new Error(
"Chunk overlap must be non-negative and smaller than the chunk size.",
);
}
const normalized = source.replace(/\r\n?/g, "\n").trim();
if (!normalized) {
return [];
}
const chunks: TextChunk[] = [];
let start = 0;
while (start < normalized.length) {
const maximumEnd = Math.min(start + chunkSize, normalized.length);
let end = maximumEnd;
if (maximumEnd < normalized.length) {
const window = normalized.slice(start, maximumEnd);
const minimumBreak = Math.floor(chunkSize * 0.6);
const paragraph = window.lastIndexOf("\n\n");
const newline = window.lastIndexOf("\n");
const space = window.lastIndexOf(" ");
const paragraphBreak =
paragraph >= chunkSize * 0.6 ? paragraph + 2 : -1;
const lineBreak =
newline >= chunkSize * 0.6 ? newline + 1 : -1;
const wordBreak =
space >= chunkSize * 0.6 ? space + 1 : -1;
const breakAt = Math.max(paragraphBreak, lineBreak, wordBreak);
if (breakAt > 0) {
end = start + breakAt;
}
}
const chunkBody = normalized.slice(start, end).trim();
if (chunkBody) {
chunks.push({
number: chunks.length + 1,
text: chunkBody,
startCharacter: start,
endCharacter: start + chunkBody.length,
});
}
if (end >= normalized.length) {
break;
}
start = Math.max(start + 1, end - overlap);
}
return chunks.map((chunk, index) => ({
...chunk,
number: index + 1,
}));
}
src / textChunker.ts
export interface TextChunk {
number: number;
text: string;
startCharacter: number;
endCharacter: number;
}
/**
* Split extracted text into bounded chunks.
* Paragraph, line, and word boundaries are preferred when possible.
*/
export function chunkText(
source: string,
chunkSize: number,
overlap: number,
): TextChunk[] {
if (!Number.isInteger(chunkSize) || chunkSize < 100) {
throw new Error("Chunk size must be an integer of at least 100.");
}
if (
!Number.isInteger(overlap) ||
overlap < 0 ||
overlap >= chunkSize
) {
throw new Error(
"Chunk overlap must be non-negative and smaller than the chunk size.",
);
}
const normalized = source.replace(/\r\n?/g, "\n").trim();
if (!normalized) {
return [];
}
const chunks: TextChunk[] = [];
let start = 0;
while (start < normalized.length) {
const maximumEnd = Math.min(start + chunkSize, normalized.length);
let end = maximumEnd;
if (maximumEnd < normalized.length) {
const window = normalized.slice(start, maximumEnd);
const minimumBreak = Math.floor(chunkSize * 0.6);
const paragraph = window.lastIndexOf("\n\n");
const newline = window.lastIndexOf("\n");
const space = window.lastIndexOf(" ");
const paragraphBreak =
paragraph >= chunkSize * 0.6 ? paragraph + 2 : -1;
const lineBreak =
newline >= chunkSize * 0.6 ? newline + 1 : -1;
const wordBreak =
space >= chunkSize * 0.6 ? space + 1 : -1;
const breakAt = Math.max(paragraphBreak, lineBreak, wordBreak);
if (breakAt > 0) {
end = start + breakAt;
}
}
const chunkBody = normalized.slice(start, end).trim();
if (chunkBody) {
chunks.push({
number: chunks.length + 1,
text: chunkBody,
startCharacter: start,
endCharacter: start + chunkBody.length,
});
}
if (end >= normalized.length) {
break;
}
start = Math.max(start + 1, end - overlap);
}
return chunks.map((chunk, index) => ({
...chunk,
number: index + 1,
}));
}