src / judge.ts
src / judge.ts
import { noul, TypeSafeClient } from "@typesafe-ai/sdk";
import { JEV_TIMEOUT_MS, type Judge, type RouteInput, type Skill } from "./types";
export type JevClient = Pick<TypeSafeClient, "systemOne">;
/** One Noul per skill. Yes means "this skill would materially help with the request". */
export function buildQuestions(skills: Skill[]) {
const questions: Record<string, ReturnType<typeof noul>> = {};
skills.forEach((skill, i) => {
questions[skill.name] = noul(
`Would following the instructions of the skill \`skills[${i}]\` (name \`skills[${i}].name\`, described by \`skills[${i}].description\`) materially help complete the request in \`message\`? Use \`recentTurns\` only as context for what the user is working on.`,
{
true: "The request is the kind of task this skill exists for, or the skill's description directly names what the user is asking about or working with.",
false: "The skill is unrelated or only tangentially related, or the request is general conversation that needs no specialised instructions.",
},
);
});
return questions;
}
/**
* One systemOne request; all skill questions run in parallel over the same state.
* `deadlineMs` exists so tests can exercise the timeout path quickly; production uses JEV_TIMEOUT_MS.
*/
export async function routeWithJev(
client: JevClient,
input: RouteInput,
skills: Skill[],
signal?: AbortSignal,
deadlineMs: number = JEV_TIMEOUT_MS,
): Promise<Record<string, number>> {
if (skills.length === 0) return {};
const deadline = AbortSignal.timeout(deadlineMs);
let answers: Awaited<ReturnType<JevClient["systemOne"]>>["answers"];
try {
({ answers } = await client.systemOne(
{
state: {
message: input.message,
recentTurns: input.recentTurns.map(t => ({ role: t.role, text: t.text })),
skills: skills.map(s => ({ name: s.name, description: s.description })),
},
questions: buildQuestions(skills),
},
{
timeout: JEV_TIMEOUT_MS,
retry: { maxRetries: 1 },
signal: signal ? AbortSignal.any([signal, deadline]) : deadline,
},
));
} catch (e) {
// The SDK reports our own deadline as a generic "Request was aborted."; only the caller's signal means "user cancelled".
if (deadline.aborted && !signal?.aborted) throw new Error(`jev timed out after ${deadlineMs} ms`);
throw e;
}
const out: Record<string, number> = {};
for (const skill of skills) {
const p = (answers as Record<string, { noul?: number } | undefined>)[skill.name]?.noul;
if (typeof p !== "number" || !Number.isFinite(p) || p < 0 || p > 1) throw new Error(`jev returned an invalid answer for ${skill.name}`);
out[skill.name] = p;
}
return out;
}
/** The key from the plugin settings if one is entered, otherwise from the JEV_API_KEY environment variable. */
export function resolveApiKey(setting: string | undefined, env: NodeJS.ProcessEnv = process.env): string | undefined {
return setting?.trim() || env.JEV_API_KEY?.trim() || undefined;
}
/** Builds a Judge that asks for the key on every call, so a changed setting takes effect without a reload. */
export function jevJudge(getKey: () => string | undefined): Judge {
let cached: { key: string; client: TypeSafeClient } | undefined;
return (input, skills, signal) => {
const key = getKey()?.trim();
if (!key) {
return Promise.reject(new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable"));
}
if (cached?.key !== key) cached = { key, client: new TypeSafeClient({ apiKey: key }) };
return routeWithJev(cached.client, input, skills, signal);
};
}
import { noul, TypeSafeClient } from "@typesafe-ai/sdk";
import { JEV_TIMEOUT_MS, type Judge, type RouteInput, type Skill } from "./types";
export type JevClient = Pick<TypeSafeClient, "systemOne">;
/** One Noul per skill. Yes means "this skill would materially help with the request". */
export function buildQuestions(skills: Skill[]) {
const questions: Record<string, ReturnType<typeof noul>> = {};
skills.forEach((skill, i) => {
questions[skill.name] = noul(
`Would following the instructions of the skill \`skills[${i}]\` (name \`skills[${i}].name\`, described by \`skills[${i}].description\`) materially help complete the request in \`message\`? Use \`recentTurns\` only as context for what the user is working on.`,
{
true: "The request is the kind of task this skill exists for, or the skill's description directly names what the user is asking about or working with.",
false: "The skill is unrelated or only tangentially related, or the request is general conversation that needs no specialised instructions.",
},
);
});
return questions;
}
/**
* One systemOne request; all skill questions run in parallel over the same state.
* `deadlineMs` exists so tests can exercise the timeout path quickly; production uses JEV_TIMEOUT_MS.
*/
export async function routeWithJev(
client: JevClient,
input: RouteInput,
skills: Skill[],
signal?: AbortSignal,
deadlineMs: number = JEV_TIMEOUT_MS,
): Promise<Record<string, number>> {
if (skills.length === 0) return {};
const deadline = AbortSignal.timeout(deadlineMs);
let answers: Awaited<ReturnType<JevClient["systemOne"]>>["answers"];
try {
({ answers } = await client.systemOne(
{
state: {
message: input.message,
recentTurns: input.recentTurns.map(t => ({ role: t.role, text: t.text })),
skills: skills.map(s => ({ name: s.name, description: s.description })),
},
questions: buildQuestions(skills),
},
{
timeout: JEV_TIMEOUT_MS,
retry: { maxRetries: 1 },
signal: signal ? AbortSignal.any([signal, deadline]) : deadline,
},
));
} catch (e) {
// The SDK reports our own deadline as a generic "Request was aborted."; only the caller's signal means "user cancelled".
if (deadline.aborted && !signal?.aborted) throw new Error(`jev timed out after ${deadlineMs} ms`);
throw e;
}
const out: Record<string, number> = {};
for (const skill of skills) {
const p = (answers as Record<string, { noul?: number } | undefined>)[skill.name]?.noul;
if (typeof p !== "number" || !Number.isFinite(p) || p < 0 || p > 1) throw new Error(`jev returned an invalid answer for ${skill.name}`);
out[skill.name] = p;
}
return out;
}
/** The key from the plugin settings if one is entered, otherwise from the JEV_API_KEY environment variable. */
export function resolveApiKey(setting: string | undefined, env: NodeJS.ProcessEnv = process.env): string | undefined {
return setting?.trim() || env.JEV_API_KEY?.trim() || undefined;
}
/** Builds a Judge that asks for the key on every call, so a changed setting takes effect without a reload. */
export function jevJudge(getKey: () => string | undefined): Judge {
let cached: { key: string; client: TypeSafeClient } | undefined;
return (input, skills, signal) => {
const key = getKey()?.trim();
if (!key) {
return Promise.reject(new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable"));
}
if (cached?.key !== key) cached = { key, client: new TypeSafeClient({ apiKey: key }) };
return routeWithJev(cached.client, input, skills, signal);
};
}