tests / promptPreprocessor.test.ts
tests / promptPreprocessor.test.ts
import { ChatMessage, type PromptPreprocessorController } from "@lmstudio/sdk";
import { describe, expect, it, vi } from "vitest";
import { createCatalog } from "../src/catalog";
import { createPreprocess } from "../src/promptPreprocessor";
import type { Judge } from "../src/types";
import { fakeFs } from "./fakeFs";
const ROOT = "/home/u/.agents/skills";
const fm = (name: string, desc: string, body: string) => `---\nname: ${name}\ndescription: ${desc}\n---\n${body}\n`;
const settings = { threshold: 0.6, maxSkills: 2, historyTurns: 4, maxInjectedChars: 12000, showHintWhenNoMatch: false };
function fakeCtl(opts: { history?: Array<{ role: "user" | "assistant"; text: string }>; settings?: Partial<typeof settings>; aborted?: boolean } = {}) {
const statuses: Array<{ status: string; text: string }> = [];
const citations: Array<{ text: string; source: unknown }> = [];
const debug: unknown[] = [];
const controller = new AbortController();
if (opts.aborted) controller.abort();
const merged = { ...settings, ...opts.settings };
const ctl = {
abortSignal: controller.signal,
getPluginConfig: () => ({ get: (k: keyof typeof merged) => merged[k] }),
getGlobalPluginConfig: () => ({ get: () => ROOT }),
pullHistory: async () => ({
getMessagesArray: () => (opts.history ?? []).map(m => ChatMessage.create(m.role, m.text)),
}),
createStatus: (s: { status: string; text: string }) => {
statuses.push(s);
return { setState: (n: { status: string; text: string }) => statuses.push(n), remove() {}, addSubStatus() { throw new Error("unused"); } };
},
createCitationBlock: (text: string, source: unknown) => citations.push({ text, source }),
debug: (...m: unknown[]) => debug.push(m),
};
return { ctl: ctl as unknown as PromptPreprocessorController, signal: controller.signal, statuses, citations, debug };
}
function setup(judgeImpl?: Judge) {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const judge = vi.fn<Judge>(judgeImpl ?? (async () => ({ pdf: 0.1, docx: 0.1 })));
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge });
return { preprocess, judge };
}
const user = (t: string) => ChatMessage.create("user", t);
describe("explicit activation", () => {
it("injects the named skill, strips the prefix and skips jev", async () => {
const { preprocess, judge } = setup();
const { ctl, citations } = fakeCtl();
const out = await preprocess(ctl, user("/pdf merge a.pdf b.pdf"));
expect(out).toBe(
[
'<skill name="pdf" path="/home/u/.agents/skills/pdf" source="explicit">\n# PDF\nMerge with qpdf.\n</skill>',
"<skill_note>Files under a skill (references/, scripts/) can be read with read_skill_file(skill, path). To run a script use shell_run if that tool is available; otherwise show the user the exact command to run.</skill_note>",
"<user_task>\nmerge a.pdf b.pdf\n</user_task>",
].join("\n\n"),
);
expect(judge).not.toHaveBeenCalled();
expect(citations).toEqual([{ text: "Work with PDF files.", source: { fileName: "pdf/SKILL.md", fileIdentifier: `${ROOT}/pdf/SKILL.md` } }]);
});
it("uses the placeholder task for a bare /name", async () => {
const { preprocess } = setup();
const out = (await preprocess(fakeCtl().ctl, user("/pdf"))) as string;
expect(out).toContain("<user_task>\n(no task text — ask the user what to do with this skill)\n</user_task>");
});
it("returns the original text plus an error for an unknown name", async () => {
const { preprocess, judge } = setup();
const out = await preprocess(fakeCtl().ctl, user("/nope do it"));
expect(out).toBe('/nope do it\n\n<skill_error>No skill named "nope". Available: docx, pdf</skill_error>');
expect(judge).not.toHaveBeenCalled();
});
it("lists a duplicated name once in the unknown-name error", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/a/SKILL.md`]: fm("same", "First.", "a"),
[`${ROOT}/b/SKILL.md`]: fm("same", "Second.", "b"),
},
});
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>() });
const out = await preprocess(fakeCtl().ctl, user("/nope do it"));
expect(out).toBe('/nope do it\n\n<skill_error>No skill named "nope". Available: same</skill_error>');
});
it("returns the original text plus an error when the skill body cannot be read", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const catalog = createCatalog(fs);
await catalog.load(ROOT);
const preprocess = createPreprocess({
catalog,
fs: {
...fs,
readFile: async (p: string) => {
if (p.endsWith("/pdf/SKILL.md")) throw Object.assign(new Error("ENOENT: gone"), { code: "ENOENT" });
return fs.readFile(p);
},
},
judge: vi.fn<Judge>(),
});
const { ctl, citations } = fakeCtl();
const out = await preprocess(ctl, user("/pdf do it"));
expect(out).toBe('/pdf do it\n\n<skill_error>Could not read SKILL.md for "pdf": ENOENT: gone</skill_error>');
expect(citations).toEqual([]);
});
it("cites the skill name when the description is empty", async () => {
const fs = fakeFs({ files: { [`${ROOT}/nodesc/SKILL.md`]: "---\nname: nodesc\n---\nbody" } });
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>() });
const { ctl, citations } = fakeCtl();
const out = (await preprocess(ctl, user("/nodesc go"))) as string;
expect(out).toContain('<skill name="nodesc"');
expect(citations).toEqual([{ text: "nodesc", source: { fileName: "nodesc/SKILL.md", fileIdentifier: `${ROOT}/nodesc/SKILL.md` } }]);
});
});
describe("jev routing", () => {
it("injects picks above threshold in probability order", async () => {
const { preprocess, judge } = setup(async () => ({ pdf: 0.7, docx: 0.95 }));
const { ctl, signal, statuses } = fakeCtl({ history: [{ role: "user", text: "hello" }, { role: "assistant", text: "hi" }] });
const out = (await preprocess(ctl, user("convert this report"))) as string;
expect(out.indexOf('<skill name="docx"')).toBeLessThan(out.indexOf('<skill name="pdf"'));
expect(out).toContain('source="jev" p="0.95"');
expect(out).toContain("<user_task>\nconvert this report\n</user_task>");
expect(judge).toHaveBeenCalledWith(
{ message: "convert this report", recentTurns: [{ role: "user", text: "hello" }, { role: "assistant", text: "hi" }] },
expect.arrayContaining([expect.objectContaining({ name: "pdf" }), expect.objectContaining({ name: "docx" })]),
signal,
);
expect(statuses.at(-1)).toEqual({ status: "done", text: "Applied skills: docx, pdf" });
});
it("returns the original message object when nothing matches and hint is off", async () => {
const { preprocess } = setup();
const msg = user("what's the weather");
const { ctl, statuses } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "done", text: "No skill applied" });
});
it("appends the hint when nothing matches and showHintWhenNoMatch is on", async () => {
const { preprocess } = setup();
const out = await preprocess(fakeCtl({ settings: { showHintWhenNoMatch: true } }).ctl, user("what's the weather"));
expect(out).toBe(
[
"<available_skills>",
`- docx: Word documents. (${ROOT}/docx)`,
`- pdf: Work with PDF files. (${ROOT}/pdf)`,
"Call read_skill_file(name) to load a skill's instructions.",
"</available_skills>",
"",
"<user_task>\nwhat's the weather\n</user_task>",
].join("\n"),
);
});
it("omits skills without a description from the hint list", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF"),
[`${ROOT}/nodesc/SKILL.md`]: "---\nname: nodesc\n---\nbody",
},
});
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>(async () => ({ pdf: 0.1 })) });
const out = (await preprocess(fakeCtl({ settings: { showHintWhenNoMatch: true } }).ctl, user("what's the weather"))) as string;
expect(out).toContain("<available_skills>");
expect(out).toContain(`- pdf: Work with PDF files. (${ROOT}/pdf)`);
expect(out).not.toContain("nodesc");
});
it("truncates history turns and limits their count", async () => {
const { preprocess, judge } = setup();
const history = Array.from({ length: 6 }, (_, i) => ({ role: "user" as const, text: `m${i} ${"x".repeat(600)}` }));
await preprocess(fakeCtl({ history, settings: { historyTurns: 2 } }).ctl, user("q"));
const input = judge.mock.calls[0][0];
expect(input.recentTurns).toHaveLength(2);
expect(input.recentTurns[0].text.startsWith("m4 ")).toBe(true);
expect(input.recentTurns[0].text).toHaveLength(500);
});
it("uses a short reference for a skill already injected in this chat", async () => {
const { preprocess } = setup(async () => ({ pdf: 0.9, docx: 0.1 }));
const prior = '<skill name="pdf" path="/x" source="jev" p="0.90">\nold\n</skill>\n\n<user_task>\nfirst\n</user_task>';
const out = (await preprocess(fakeCtl({ history: [{ role: "user", text: prior }] }).ctl, user("again"))) as string;
expect(out).toContain('<skill_active name="pdf" path="/home/u/.agents/skills/pdf">');
expect(out).not.toContain("Merge with qpdf");
});
it("skips jev when maxSkills is 0 and returns the original message", async () => {
const { preprocess, judge } = setup();
const msg = user("hi");
expect(await preprocess(fakeCtl({ settings: { maxSkills: 0 } }).ctl, msg)).toBe(msg);
expect(judge).not.toHaveBeenCalled();
});
});
describe("failure modes", () => {
it("falls back to the hint when jev rejects and reports the reason in the status", async () => {
const { preprocess } = setup(async () => {
throw new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable");
});
const { ctl, statuses } = fakeCtl();
const out = (await preprocess(ctl, user("merge pdfs"))) as string;
expect(out).toContain("<available_skills>");
expect(out).toContain("<user_task>\nmerge pdfs\n</user_task>");
expect(statuses.at(-1)).toEqual({
status: "done",
text: "Skills: jev unavailable (no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable); listed available skills instead",
});
});
it("returns the original message when aborted during jev", async () => {
const { preprocess } = setup(async () => {
throw Object.assign(new Error("aborted"), { name: "AbortError" });
});
const msg = user("merge pdfs");
const { ctl, statuses } = fakeCtl({ aborted: true });
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "canceled", text: "Skill routing canceled" });
});
it("reports a jev timeout via the hint fallback with the timeout reason", async () => {
const { preprocess } = setup(async () => {
throw new Error("jev timed out after 6000 ms");
});
const { ctl, statuses } = fakeCtl();
const out = (await preprocess(ctl, user("merge pdfs"))) as string;
expect(out).toContain("<available_skills>");
expect(statuses.at(-1)).toEqual({ status: "done", text: "Skills: jev unavailable (jev timed out after 6000 ms); listed available skills instead" });
});
it("is a no-op when the skills root is empty", async () => {
const fs = fakeFs({ files: {} });
const judge = vi.fn<Judge>();
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge });
const msg = user("hello");
expect(await preprocess(fakeCtl().ctl, msg)).toBe(msg);
expect(judge).not.toHaveBeenCalled();
});
it("marks the status as error and returns the original message when a skill body cannot be read after routing", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const catalog = createCatalog(fs);
await catalog.load(ROOT);
const judge = vi.fn<Judge>(async () => ({ pdf: 0.9, docx: 0.1 }));
const preprocess = createPreprocess({
catalog,
fs: {
...fs,
readFile: async (p: string) => {
if (p.endsWith("/pdf/SKILL.md")) throw Object.assign(new Error("ENOENT: gone"), { code: "ENOENT" });
return fs.readFile(p);
},
},
judge,
});
const msg = user("merge pdfs");
const { ctl, statuses, debug } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "error", text: "Skills: could not read a skill file (ENOENT: gone)" });
expect(JSON.stringify(debug)).toMatch(/ENOENT: gone/);
});
it("never throws: an unexpected error returns the original message", async () => {
const fs = fakeFs({ files: { [`${ROOT}/pdf/SKILL.md`]: fm("pdf", "PDFs.", "body") } });
const catalog = { load: async () => { throw new Error("disk on fire"); } };
const preprocess = createPreprocess({ catalog, fs, judge: vi.fn<Judge>() });
const msg = user("hello");
const { ctl, debug } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(JSON.stringify(debug)).toMatch(/disk on fire/);
});
});
import { ChatMessage, type PromptPreprocessorController } from "@lmstudio/sdk";
import { describe, expect, it, vi } from "vitest";
import { createCatalog } from "../src/catalog";
import { createPreprocess } from "../src/promptPreprocessor";
import type { Judge } from "../src/types";
import { fakeFs } from "./fakeFs";
const ROOT = "/home/u/.agents/skills";
const fm = (name: string, desc: string, body: string) => `---\nname: ${name}\ndescription: ${desc}\n---\n${body}\n`;
const settings = { threshold: 0.6, maxSkills: 2, historyTurns: 4, maxInjectedChars: 12000, showHintWhenNoMatch: false };
function fakeCtl(opts: { history?: Array<{ role: "user" | "assistant"; text: string }>; settings?: Partial<typeof settings>; aborted?: boolean } = {}) {
const statuses: Array<{ status: string; text: string }> = [];
const citations: Array<{ text: string; source: unknown }> = [];
const debug: unknown[] = [];
const controller = new AbortController();
if (opts.aborted) controller.abort();
const merged = { ...settings, ...opts.settings };
const ctl = {
abortSignal: controller.signal,
getPluginConfig: () => ({ get: (k: keyof typeof merged) => merged[k] }),
getGlobalPluginConfig: () => ({ get: () => ROOT }),
pullHistory: async () => ({
getMessagesArray: () => (opts.history ?? []).map(m => ChatMessage.create(m.role, m.text)),
}),
createStatus: (s: { status: string; text: string }) => {
statuses.push(s);
return { setState: (n: { status: string; text: string }) => statuses.push(n), remove() {}, addSubStatus() { throw new Error("unused"); } };
},
createCitationBlock: (text: string, source: unknown) => citations.push({ text, source }),
debug: (...m: unknown[]) => debug.push(m),
};
return { ctl: ctl as unknown as PromptPreprocessorController, signal: controller.signal, statuses, citations, debug };
}
function setup(judgeImpl?: Judge) {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const judge = vi.fn<Judge>(judgeImpl ?? (async () => ({ pdf: 0.1, docx: 0.1 })));
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge });
return { preprocess, judge };
}
const user = (t: string) => ChatMessage.create("user", t);
describe("explicit activation", () => {
it("injects the named skill, strips the prefix and skips jev", async () => {
const { preprocess, judge } = setup();
const { ctl, citations } = fakeCtl();
const out = await preprocess(ctl, user("/pdf merge a.pdf b.pdf"));
expect(out).toBe(
[
'<skill name="pdf" path="/home/u/.agents/skills/pdf" source="explicit">\n# PDF\nMerge with qpdf.\n</skill>',
"<skill_note>Files under a skill (references/, scripts/) can be read with read_skill_file(skill, path). To run a script use shell_run if that tool is available; otherwise show the user the exact command to run.</skill_note>",
"<user_task>\nmerge a.pdf b.pdf\n</user_task>",
].join("\n\n"),
);
expect(judge).not.toHaveBeenCalled();
expect(citations).toEqual([{ text: "Work with PDF files.", source: { fileName: "pdf/SKILL.md", fileIdentifier: `${ROOT}/pdf/SKILL.md` } }]);
});
it("uses the placeholder task for a bare /name", async () => {
const { preprocess } = setup();
const out = (await preprocess(fakeCtl().ctl, user("/pdf"))) as string;
expect(out).toContain("<user_task>\n(no task text — ask the user what to do with this skill)\n</user_task>");
});
it("returns the original text plus an error for an unknown name", async () => {
const { preprocess, judge } = setup();
const out = await preprocess(fakeCtl().ctl, user("/nope do it"));
expect(out).toBe('/nope do it\n\n<skill_error>No skill named "nope". Available: docx, pdf</skill_error>');
expect(judge).not.toHaveBeenCalled();
});
it("lists a duplicated name once in the unknown-name error", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/a/SKILL.md`]: fm("same", "First.", "a"),
[`${ROOT}/b/SKILL.md`]: fm("same", "Second.", "b"),
},
});
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>() });
const out = await preprocess(fakeCtl().ctl, user("/nope do it"));
expect(out).toBe('/nope do it\n\n<skill_error>No skill named "nope". Available: same</skill_error>');
});
it("returns the original text plus an error when the skill body cannot be read", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const catalog = createCatalog(fs);
await catalog.load(ROOT);
const preprocess = createPreprocess({
catalog,
fs: {
...fs,
readFile: async (p: string) => {
if (p.endsWith("/pdf/SKILL.md")) throw Object.assign(new Error("ENOENT: gone"), { code: "ENOENT" });
return fs.readFile(p);
},
},
judge: vi.fn<Judge>(),
});
const { ctl, citations } = fakeCtl();
const out = await preprocess(ctl, user("/pdf do it"));
expect(out).toBe('/pdf do it\n\n<skill_error>Could not read SKILL.md for "pdf": ENOENT: gone</skill_error>');
expect(citations).toEqual([]);
});
it("cites the skill name when the description is empty", async () => {
const fs = fakeFs({ files: { [`${ROOT}/nodesc/SKILL.md`]: "---\nname: nodesc\n---\nbody" } });
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>() });
const { ctl, citations } = fakeCtl();
const out = (await preprocess(ctl, user("/nodesc go"))) as string;
expect(out).toContain('<skill name="nodesc"');
expect(citations).toEqual([{ text: "nodesc", source: { fileName: "nodesc/SKILL.md", fileIdentifier: `${ROOT}/nodesc/SKILL.md` } }]);
});
});
describe("jev routing", () => {
it("injects picks above threshold in probability order", async () => {
const { preprocess, judge } = setup(async () => ({ pdf: 0.7, docx: 0.95 }));
const { ctl, signal, statuses } = fakeCtl({ history: [{ role: "user", text: "hello" }, { role: "assistant", text: "hi" }] });
const out = (await preprocess(ctl, user("convert this report"))) as string;
expect(out.indexOf('<skill name="docx"')).toBeLessThan(out.indexOf('<skill name="pdf"'));
expect(out).toContain('source="jev" p="0.95"');
expect(out).toContain("<user_task>\nconvert this report\n</user_task>");
expect(judge).toHaveBeenCalledWith(
{ message: "convert this report", recentTurns: [{ role: "user", text: "hello" }, { role: "assistant", text: "hi" }] },
expect.arrayContaining([expect.objectContaining({ name: "pdf" }), expect.objectContaining({ name: "docx" })]),
signal,
);
expect(statuses.at(-1)).toEqual({ status: "done", text: "Applied skills: docx, pdf" });
});
it("returns the original message object when nothing matches and hint is off", async () => {
const { preprocess } = setup();
const msg = user("what's the weather");
const { ctl, statuses } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "done", text: "No skill applied" });
});
it("appends the hint when nothing matches and showHintWhenNoMatch is on", async () => {
const { preprocess } = setup();
const out = await preprocess(fakeCtl({ settings: { showHintWhenNoMatch: true } }).ctl, user("what's the weather"));
expect(out).toBe(
[
"<available_skills>",
`- docx: Word documents. (${ROOT}/docx)`,
`- pdf: Work with PDF files. (${ROOT}/pdf)`,
"Call read_skill_file(name) to load a skill's instructions.",
"</available_skills>",
"",
"<user_task>\nwhat's the weather\n</user_task>",
].join("\n"),
);
});
it("omits skills without a description from the hint list", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF"),
[`${ROOT}/nodesc/SKILL.md`]: "---\nname: nodesc\n---\nbody",
},
});
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge: vi.fn<Judge>(async () => ({ pdf: 0.1 })) });
const out = (await preprocess(fakeCtl({ settings: { showHintWhenNoMatch: true } }).ctl, user("what's the weather"))) as string;
expect(out).toContain("<available_skills>");
expect(out).toContain(`- pdf: Work with PDF files. (${ROOT}/pdf)`);
expect(out).not.toContain("nodesc");
});
it("truncates history turns and limits their count", async () => {
const { preprocess, judge } = setup();
const history = Array.from({ length: 6 }, (_, i) => ({ role: "user" as const, text: `m${i} ${"x".repeat(600)}` }));
await preprocess(fakeCtl({ history, settings: { historyTurns: 2 } }).ctl, user("q"));
const input = judge.mock.calls[0][0];
expect(input.recentTurns).toHaveLength(2);
expect(input.recentTurns[0].text.startsWith("m4 ")).toBe(true);
expect(input.recentTurns[0].text).toHaveLength(500);
});
it("uses a short reference for a skill already injected in this chat", async () => {
const { preprocess } = setup(async () => ({ pdf: 0.9, docx: 0.1 }));
const prior = '<skill name="pdf" path="/x" source="jev" p="0.90">\nold\n</skill>\n\n<user_task>\nfirst\n</user_task>';
const out = (await preprocess(fakeCtl({ history: [{ role: "user", text: prior }] }).ctl, user("again"))) as string;
expect(out).toContain('<skill_active name="pdf" path="/home/u/.agents/skills/pdf">');
expect(out).not.toContain("Merge with qpdf");
});
it("skips jev when maxSkills is 0 and returns the original message", async () => {
const { preprocess, judge } = setup();
const msg = user("hi");
expect(await preprocess(fakeCtl({ settings: { maxSkills: 0 } }).ctl, msg)).toBe(msg);
expect(judge).not.toHaveBeenCalled();
});
});
describe("failure modes", () => {
it("falls back to the hint when jev rejects and reports the reason in the status", async () => {
const { preprocess } = setup(async () => {
throw new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable");
});
const { ctl, statuses } = fakeCtl();
const out = (await preprocess(ctl, user("merge pdfs"))) as string;
expect(out).toContain("<available_skills>");
expect(out).toContain("<user_task>\nmerge pdfs\n</user_task>");
expect(statuses.at(-1)).toEqual({
status: "done",
text: "Skills: jev unavailable (no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable); listed available skills instead",
});
});
it("returns the original message when aborted during jev", async () => {
const { preprocess } = setup(async () => {
throw Object.assign(new Error("aborted"), { name: "AbortError" });
});
const msg = user("merge pdfs");
const { ctl, statuses } = fakeCtl({ aborted: true });
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "canceled", text: "Skill routing canceled" });
});
it("reports a jev timeout via the hint fallback with the timeout reason", async () => {
const { preprocess } = setup(async () => {
throw new Error("jev timed out after 6000 ms");
});
const { ctl, statuses } = fakeCtl();
const out = (await preprocess(ctl, user("merge pdfs"))) as string;
expect(out).toContain("<available_skills>");
expect(statuses.at(-1)).toEqual({ status: "done", text: "Skills: jev unavailable (jev timed out after 6000 ms); listed available skills instead" });
});
it("is a no-op when the skills root is empty", async () => {
const fs = fakeFs({ files: {} });
const judge = vi.fn<Judge>();
const preprocess = createPreprocess({ catalog: createCatalog(fs), fs, judge });
const msg = user("hello");
expect(await preprocess(fakeCtl().ctl, msg)).toBe(msg);
expect(judge).not.toHaveBeenCalled();
});
it("marks the status as error and returns the original message when a skill body cannot be read after routing", async () => {
const fs = fakeFs({
files: {
[`${ROOT}/pdf/SKILL.md`]: fm("pdf", "Work with PDF files.", "# PDF\nMerge with qpdf."),
[`${ROOT}/docx/SKILL.md`]: fm("docx", "Word documents.", "# DOCX\nUse python-docx."),
},
});
const catalog = createCatalog(fs);
await catalog.load(ROOT);
const judge = vi.fn<Judge>(async () => ({ pdf: 0.9, docx: 0.1 }));
const preprocess = createPreprocess({
catalog,
fs: {
...fs,
readFile: async (p: string) => {
if (p.endsWith("/pdf/SKILL.md")) throw Object.assign(new Error("ENOENT: gone"), { code: "ENOENT" });
return fs.readFile(p);
},
},
judge,
});
const msg = user("merge pdfs");
const { ctl, statuses, debug } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(statuses.at(-1)).toEqual({ status: "error", text: "Skills: could not read a skill file (ENOENT: gone)" });
expect(JSON.stringify(debug)).toMatch(/ENOENT: gone/);
});
it("never throws: an unexpected error returns the original message", async () => {
const fs = fakeFs({ files: { [`${ROOT}/pdf/SKILL.md`]: fm("pdf", "PDFs.", "body") } });
const catalog = { load: async () => { throw new Error("disk on fire"); } };
const preprocess = createPreprocess({ catalog, fs, judge: vi.fn<Judge>() });
const msg = user("hello");
const { ctl, debug } = fakeCtl();
expect(await preprocess(ctl, msg)).toBe(msg);
expect(JSON.stringify(debug)).toMatch(/disk on fire/);
});
});