src / judge.ts
import { noul, score, TypeSafeClient } from "@typesafe-ai/sdk";
import * as os from "os";
import { HAZARDS, type Assessment, type Assessor, type Hazard } from "./types";
export const JEV_TIMEOUT_MS = 8000;
export type JevClient = Pick<TypeSafeClient, "systemOne">;
/** Yes always means "dangerous", so higher probabilities are worse. */
export const QUESTIONS = {
destroys_data: noul("Would running this command delete, overwrite, truncate or irreversibly modify files or data?", {
true: "It removes, overwrites or truncates files, drops data, or rewrites history in a way that cannot easily be undone.",
false: "It only reads or lists, or creates or edits ordinary files in a way that is easy to undo.",
}),
touches_system: noul(
"Does this command modify system files, global configuration, installed software, or anything outside the working directory tree?",
{
true: "It changes OS files, global or user-level configuration, installed packages, or files outside the working directory.",
false: "It stays inside the working directory tree, or only reads elsewhere. Ordinary edits inside the working directory do not count.",
},
),
network_exfil: noul("Does this command send local data to a remote host, or download and execute remote content?", {
true: "It uploads or posts local data, opens a reverse connection, or fetches something and runs it.",
false: "It makes no network requests, or only downloads data without running it.",
}),
privilege_or_persistence: noul(
"Does this command escalate privileges, or install persistent behavior such as cron jobs, launch agents or shell startup edits?",
{
true: "It uses sudo or setuid, changes permissions to gain access, or installs something that keeps running or runs later.",
false: "It runs once with the current user's ordinary privileges and leaves nothing behind.",
},
),
obfuscated: noul("Does this command hide its true effect through encoding, eval, indirection or unusual constructs?", {
true: "Its real behavior cannot be read from the text, for example base64-decoded payloads, eval of built strings, or nested indirection.",
false: "What it does is plain from reading it.",
}),
severity: score("If this command were run, how bad is the worst plausible outcome?", [
"Harmless: read-only or purely informational.",
"Minor: changes something small that is easy to undo.",
"Significant: changes or removes things that are hard to undo.",
"Severe: system-level or widespread damage, or serious data loss or leakage.",
]),
};
/** One jev request: the five hazard Nouls and the severity Score run in parallel over the same state. */
export async function assess(
client: JevClient,
input: { command: string; cwd: string },
signal?: AbortSignal,
): Promise<Assessment> {
const deadline = AbortSignal.timeout(JEV_TIMEOUT_MS);
const { answers } = await client.systemOne(
{
state: {
command: input.command,
cwd: input.cwd,
os: `${os.type()} ${os.release()}`,
platform: process.platform,
},
questions: QUESTIONS,
},
{
timeout: JEV_TIMEOUT_MS,
retry: { maxRetries: 1 },
signal: signal ? AbortSignal.any([signal, deadline]) : deadline,
},
);
const hazards = {} as Record<Hazard, number>;
for (const hazard of HAZARDS) {
const probability = answers[hazard].noul;
if (!Number.isFinite(probability) || probability < 0 || probability > 1) throw new Error(`jev returned an invalid answer for ${hazard}`);
hazards[hazard] = probability;
}
const severity = answers.severity.score;
if (!Number.isFinite(severity) || severity < 0 || severity > 3) throw new Error("jev returned an invalid severity");
return { hazards, severity };
}
/** The key from the plugin settings if one is entered, otherwise from the JEV_API_KEY environment variable. */
export function resolveApiKey(setting: string | undefined, env: NodeJS.ProcessEnv = process.env): string | undefined {
return setting?.trim() || env.JEV_API_KEY?.trim() || undefined;
}
/** Builds an Assessor that asks for the key on every call, so a changed setting takes effect without a reload. */
export function jevAssessor(getKey: () => string | undefined): Assessor {
let cached: { key: string; client: TypeSafeClient } | undefined;
return (input, signal) => {
const key = getKey()?.trim();
if (!key) {
return Promise.reject(new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable"));
}
if (cached?.key !== key) cached = { key, client: new TypeSafeClient({ apiKey: key }) };
return assess(cached.client, input, signal);
};
}
src / judge.ts
import { noul, score, TypeSafeClient } from "@typesafe-ai/sdk";
import * as os from "os";
import { HAZARDS, type Assessment, type Assessor, type Hazard } from "./types";
export const JEV_TIMEOUT_MS = 8000;
export type JevClient = Pick<TypeSafeClient, "systemOne">;
/** Yes always means "dangerous", so higher probabilities are worse. */
export const QUESTIONS = {
destroys_data: noul("Would running this command delete, overwrite, truncate or irreversibly modify files or data?", {
true: "It removes, overwrites or truncates files, drops data, or rewrites history in a way that cannot easily be undone.",
false: "It only reads or lists, or creates or edits ordinary files in a way that is easy to undo.",
}),
touches_system: noul(
"Does this command modify system files, global configuration, installed software, or anything outside the working directory tree?",
{
true: "It changes OS files, global or user-level configuration, installed packages, or files outside the working directory.",
false: "It stays inside the working directory tree, or only reads elsewhere. Ordinary edits inside the working directory do not count.",
},
),
network_exfil: noul("Does this command send local data to a remote host, or download and execute remote content?", {
true: "It uploads or posts local data, opens a reverse connection, or fetches something and runs it.",
false: "It makes no network requests, or only downloads data without running it.",
}),
privilege_or_persistence: noul(
"Does this command escalate privileges, or install persistent behavior such as cron jobs, launch agents or shell startup edits?",
{
true: "It uses sudo or setuid, changes permissions to gain access, or installs something that keeps running or runs later.",
false: "It runs once with the current user's ordinary privileges and leaves nothing behind.",
},
),
obfuscated: noul("Does this command hide its true effect through encoding, eval, indirection or unusual constructs?", {
true: "Its real behavior cannot be read from the text, for example base64-decoded payloads, eval of built strings, or nested indirection.",
false: "What it does is plain from reading it.",
}),
severity: score("If this command were run, how bad is the worst plausible outcome?", [
"Harmless: read-only or purely informational.",
"Minor: changes something small that is easy to undo.",
"Significant: changes or removes things that are hard to undo.",
"Severe: system-level or widespread damage, or serious data loss or leakage.",
]),
};
/** One jev request: the five hazard Nouls and the severity Score run in parallel over the same state. */
export async function assess(
client: JevClient,
input: { command: string; cwd: string },
signal?: AbortSignal,
): Promise<Assessment> {
const deadline = AbortSignal.timeout(JEV_TIMEOUT_MS);
const { answers } = await client.systemOne(
{
state: {
command: input.command,
cwd: input.cwd,
os: `${os.type()} ${os.release()}`,
platform: process.platform,
},
questions: QUESTIONS,
},
{
timeout: JEV_TIMEOUT_MS,
retry: { maxRetries: 1 },
signal: signal ? AbortSignal.any([signal, deadline]) : deadline,
},
);
const hazards = {} as Record<Hazard, number>;
for (const hazard of HAZARDS) {
const probability = answers[hazard].noul;
if (!Number.isFinite(probability) || probability < 0 || probability > 1) throw new Error(`jev returned an invalid answer for ${hazard}`);
hazards[hazard] = probability;
}
const severity = answers.severity.score;
if (!Number.isFinite(severity) || severity < 0 || severity > 3) throw new Error("jev returned an invalid severity");
return { hazards, severity };
}
/** The key from the plugin settings if one is entered, otherwise from the JEV_API_KEY environment variable. */
export function resolveApiKey(setting: string | undefined, env: NodeJS.ProcessEnv = process.env): string | undefined {
return setting?.trim() || env.JEV_API_KEY?.trim() || undefined;
}
/** Builds an Assessor that asks for the key on every call, so a changed setting takes effect without a reload. */
export function jevAssessor(getKey: () => string | undefined): Assessor {
let cached: { key: string; client: TypeSafeClient } | undefined;
return (input, signal) => {
const key = getKey()?.trim();
if (!key) {
return Promise.reject(new Error("no jev API key: set it in the plugin settings or in the JEV_API_KEY environment variable"));
}
if (cached?.key !== key) cached = { key, client: new TypeSafeClient({ apiKey: key }) };
return assess(cached.client, input, signal);
};
}