src / tests / textChunker.test.ts
import assert from "node:assert/strict";
import test from "node:test";
import { chunkText } from "../textChunker";
test("empty documents produce no chunks", () => {
assert.deepEqual(chunkText("", 100, 10), []);
assert.deepEqual(chunkText(" \n", 100, 10), []);
});
test("short documents produce one chunk", () => {
const chunks = chunkText("A short document.", 100, 10);
assert.equal(chunks.length, 1);
assert.equal(chunks[0].number, 1);
assert.equal(chunks[0].text, "A short document.");
});
test("long documents are split with bounded output and overlap", () => {
const source = "alpha beta gamma delta ".repeat(100);
const chunks = chunkText(source, 200, 20);
assert.ok(chunks.length > 1);
assert.equal(chunks[0].number, 1);
assert.equal(chunks.at(-1)?.number, chunks.length);
for (const chunk of chunks) {
assert.ok(chunk.text.length <= 200);
assert.ok(chunk.startCharacter >= 0);
assert.ok(chunk.endCharacter > chunk.startCharacter);
}
});
test("invalid overlap is rejected", () => {
assert.throws(() => chunkText("hello", 100, 100));
});
src / tests / textChunker.test.ts
import assert from "node:assert/strict";
import test from "node:test";
import { chunkText } from "../textChunker";
test("empty documents produce no chunks", () => {
assert.deepEqual(chunkText("", 100, 10), []);
assert.deepEqual(chunkText(" \n", 100, 10), []);
});
test("short documents produce one chunk", () => {
const chunks = chunkText("A short document.", 100, 10);
assert.equal(chunks.length, 1);
assert.equal(chunks[0].number, 1);
assert.equal(chunks[0].text, "A short document.");
});
test("long documents are split with bounded output and overlap", () => {
const source = "alpha beta gamma delta ".repeat(100);
const chunks = chunkText(source, 200, 20);
assert.ok(chunks.length > 1);
assert.equal(chunks[0].number, 1);
assert.equal(chunks.at(-1)?.number, chunks.length);
for (const chunk of chunks) {
assert.ok(chunk.text.length <= 200);
assert.ok(chunk.startCharacter >= 0);
assert.ok(chunk.endCharacter > chunk.startCharacter);
}
});
test("invalid overlap is rejected", () => {
assert.throws(() => chunkText("hello", 100, 100));
});