tests / judge.test.ts
tests / judge.test.ts
import { describe, expect, it, vi } from "vitest";
import { buildQuestions, jevJudge, resolveApiKey, routeWithJev, type JevClient } from "../src/judge";
import type { Skill } from "../src/types";
const skill = (name: string, description: string): Skill => ({
name,
description,
dir: `/s/${name}`,
skillMdPath: `/s/${name}/SKILL.md`,
routable: true,
disableModelInvocation: false,
});
const skills = [skill("pdf", "Work with PDF files."), skill("docx", "Word documents.")];
const input = { message: "merge these two pdfs", recentTurns: [{ role: "assistant", text: "Hi" }] };
function fakeClient(answers: Record<string, unknown>) {
const systemOne = vi.fn().mockResolvedValue({ answers });
return { client: { systemOne } as unknown as JevClient, systemOne };
}
describe("buildQuestions", () => {
it("creates one Noul per skill keyed by name, mentioning name and description", () => {
const q = buildQuestions(skills);
expect(Object.keys(q)).toEqual(["pdf", "docx"]);
expect(JSON.stringify(q.pdf)).toContain("skills[0]");
});
});
describe("routeWithJev", () => {
it("returns a probability per skill", async () => {
const { client } = fakeClient({ pdf: { noul: 0.92 }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).resolves.toEqual({ pdf: 0.92, docx: 0.1 });
});
it("sends message, recent turns and skill name/description as state, never bodies", async () => {
const { client, systemOne } = fakeClient({ pdf: { noul: 0.5 }, docx: { noul: 0.5 } });
await routeWithJev(client, input, skills);
const [request, options] = systemOne.mock.calls[0];
expect(request.state).toEqual({
message: "merge these two pdfs",
recentTurns: [{ role: "assistant", text: "Hi" }],
skills: [
{ name: "pdf", description: "Work with PDF files." },
{ name: "docx", description: "Word documents." },
],
});
expect(Object.keys(request.questions)).toEqual(["pdf", "docx"]);
expect(options.timeout).toBe(6000);
expect(options.signal).toBeInstanceOf(AbortSignal);
});
it("rejects on a non-finite probability", async () => {
const { client } = fakeClient({ pdf: { noul: NaN }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/pdf/);
});
it("rejects on a missing answer", async () => {
const { client } = fakeClient({ pdf: { noul: 0.3 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/docx/);
});
it("returns an empty record and makes no call when there are no skills", async () => {
const { client, systemOne } = fakeClient({});
await expect(routeWithJev(client, input, [])).resolves.toEqual({});
expect(systemOne).not.toHaveBeenCalled();
});
it("rejects on a probability outside 0..1", async () => {
const { client } = fakeClient({ pdf: { noul: 1.7 }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/pdf/);
});
it("reports a timeout as a timeout, not an abort", async () => {
// Mirrors the TypeSafe SDK: once the combined signal fires it throws APIUserAbortError.
const systemOne = vi.fn((_req: unknown, options: { signal: AbortSignal }) =>
new Promise((_resolve, reject) => {
options.signal.addEventListener("abort", () => reject(Object.assign(new Error("Request was aborted."), { name: "APIUserAbortError" })));
}),
);
const client = { systemOne } as unknown as JevClient;
await expect(routeWithJev(client, input, skills, undefined, 5)).rejects.toThrow(/timed out after 6000 ms|timed out after \d+ ms/);
});
it("propagates a caller abort unchanged", async () => {
const ctl = new AbortController();
ctl.abort();
const systemOne = vi.fn().mockRejectedValue(Object.assign(new Error("Request was aborted."), { name: "APIUserAbortError" }));
const client = { systemOne } as unknown as JevClient;
await expect(routeWithJev(client, input, skills, ctl.signal)).rejects.toThrow("Request was aborted.");
});
});
describe("resolveApiKey / jevJudge", () => {
it("prefers the plugin setting over the environment variable", () => {
expect(resolveApiKey(" setting-key ", { JEV_API_KEY: "env-key" })).toBe("setting-key");
expect(resolveApiKey("", { JEV_API_KEY: "env-key" })).toBe("env-key");
expect(resolveApiKey(" ", {})).toBeUndefined();
});
it("rejects when neither the setting nor JEV_API_KEY is set", async () => {
await expect(jevJudge(() => undefined)(input, skills)).rejects.toThrow(/no jev API key/);
});
it("re-reads the key on every call so a changed setting takes effect", async () => {
let key = "first";
const judge = jevJudge(() => key);
// Empty skills short-circuits before any network call, so this only exercises key resolution + client construction.
await expect(judge(input, [])).resolves.toEqual({});
key = "";
await expect(judge(input, skills)).rejects.toThrow(/no jev API key/);
});
});
import { describe, expect, it, vi } from "vitest";
import { buildQuestions, jevJudge, resolveApiKey, routeWithJev, type JevClient } from "../src/judge";
import type { Skill } from "../src/types";
const skill = (name: string, description: string): Skill => ({
name,
description,
dir: `/s/${name}`,
skillMdPath: `/s/${name}/SKILL.md`,
routable: true,
disableModelInvocation: false,
});
const skills = [skill("pdf", "Work with PDF files."), skill("docx", "Word documents.")];
const input = { message: "merge these two pdfs", recentTurns: [{ role: "assistant", text: "Hi" }] };
function fakeClient(answers: Record<string, unknown>) {
const systemOne = vi.fn().mockResolvedValue({ answers });
return { client: { systemOne } as unknown as JevClient, systemOne };
}
describe("buildQuestions", () => {
it("creates one Noul per skill keyed by name, mentioning name and description", () => {
const q = buildQuestions(skills);
expect(Object.keys(q)).toEqual(["pdf", "docx"]);
expect(JSON.stringify(q.pdf)).toContain("skills[0]");
});
});
describe("routeWithJev", () => {
it("returns a probability per skill", async () => {
const { client } = fakeClient({ pdf: { noul: 0.92 }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).resolves.toEqual({ pdf: 0.92, docx: 0.1 });
});
it("sends message, recent turns and skill name/description as state, never bodies", async () => {
const { client, systemOne } = fakeClient({ pdf: { noul: 0.5 }, docx: { noul: 0.5 } });
await routeWithJev(client, input, skills);
const [request, options] = systemOne.mock.calls[0];
expect(request.state).toEqual({
message: "merge these two pdfs",
recentTurns: [{ role: "assistant", text: "Hi" }],
skills: [
{ name: "pdf", description: "Work with PDF files." },
{ name: "docx", description: "Word documents." },
],
});
expect(Object.keys(request.questions)).toEqual(["pdf", "docx"]);
expect(options.timeout).toBe(6000);
expect(options.signal).toBeInstanceOf(AbortSignal);
});
it("rejects on a non-finite probability", async () => {
const { client } = fakeClient({ pdf: { noul: NaN }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/pdf/);
});
it("rejects on a missing answer", async () => {
const { client } = fakeClient({ pdf: { noul: 0.3 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/docx/);
});
it("returns an empty record and makes no call when there are no skills", async () => {
const { client, systemOne } = fakeClient({});
await expect(routeWithJev(client, input, [])).resolves.toEqual({});
expect(systemOne).not.toHaveBeenCalled();
});
it("rejects on a probability outside 0..1", async () => {
const { client } = fakeClient({ pdf: { noul: 1.7 }, docx: { noul: 0.1 } });
await expect(routeWithJev(client, input, skills)).rejects.toThrow(/pdf/);
});
it("reports a timeout as a timeout, not an abort", async () => {
// Mirrors the TypeSafe SDK: once the combined signal fires it throws APIUserAbortError.
const systemOne = vi.fn((_req: unknown, options: { signal: AbortSignal }) =>
new Promise((_resolve, reject) => {
options.signal.addEventListener("abort", () => reject(Object.assign(new Error("Request was aborted."), { name: "APIUserAbortError" })));
}),
);
const client = { systemOne } as unknown as JevClient;
await expect(routeWithJev(client, input, skills, undefined, 5)).rejects.toThrow(/timed out after 6000 ms|timed out after \d+ ms/);
});
it("propagates a caller abort unchanged", async () => {
const ctl = new AbortController();
ctl.abort();
const systemOne = vi.fn().mockRejectedValue(Object.assign(new Error("Request was aborted."), { name: "APIUserAbortError" }));
const client = { systemOne } as unknown as JevClient;
await expect(routeWithJev(client, input, skills, ctl.signal)).rejects.toThrow("Request was aborted.");
});
});
describe("resolveApiKey / jevJudge", () => {
it("prefers the plugin setting over the environment variable", () => {
expect(resolveApiKey(" setting-key ", { JEV_API_KEY: "env-key" })).toBe("setting-key");
expect(resolveApiKey("", { JEV_API_KEY: "env-key" })).toBe("env-key");
expect(resolveApiKey(" ", {})).toBeUndefined();
});
it("rejects when neither the setting nor JEV_API_KEY is set", async () => {
await expect(jevJudge(() => undefined)(input, skills)).rejects.toThrow(/no jev API key/);
});
it("re-reads the key on every call so a changed setting takes effect", async () => {
let key = "first";
const judge = jevJudge(() => key);
// Empty skills short-circuits before any network call, so this only exercises key resolution + client construction.
await expect(judge(input, [])).resolves.toEqual({});
key = "";
await expect(judge(input, skills)).rejects.toThrow(/no jev API key/);
});
});