bench / metrics.ts
bench / metrics.ts
/** Pure measurement helpers for the benchmark suite. */
import { CanonMessage } from "../src/view";
const normalize = (s: string): string =>
s.toLowerCase().replace(/\s+/g, " ").trim();
/** Case/whitespace-normalized substring match against accepted answers. */
export function gradeAnswer(response: string, accepted: string[]): boolean {
const norm = normalize(response);
return accepted.some((candidate) => norm.includes(normalize(candidate)));
}
export interface StructuralReport {
singleLeadingSystem: boolean;
noOrphanToolResults: boolean;
hasUserMessage: boolean;
ok: boolean;
}
/** Validate a compacted view: system text + tail messages. */
export function checkViewStructure(
systemText: string | undefined,
tail: CanonMessage[],
): StructuralReport {
void systemText; // the system text lives outside the tail by construction
const singleLeadingSystem = tail.every((m) => m.role !== "system");
let noOrphanToolResults = true;
tail.forEach((m, i) => {
if (m.role !== "tool") return;
const prev = tail[i - 1];
const anchored =
prev !== undefined &&
(prev.role === "tool" ||
(prev.role === "assistant" && (prev.toolCalls ?? []).length > 0));
if (!anchored) noOrphanToolResults = false;
});
const hasUserMessage = tail.some((m) => m.role === "user");
return {
singleLeadingSystem,
noOrphanToolResults,
hasUserMessage,
ok: singleLeadingSystem && noOrphanToolResults && hasUserMessage,
};
}
export function reductionPercent(
beforeTokens: number,
afterTokens: number,
): number {
if (beforeTokens <= 0) return 0;
return Math.round((1 - afterTokens / beforeTokens) * 100);
}
export function latencyStats(samplesMs: number[]): {
mean: number;
p50: number;
max: number;
} {
if (samplesMs.length === 0) return { mean: 0, p50: 0, max: 0 };
const sorted = [...samplesMs].sort((x, y) => x - y);
const mean =
Math.round(
(sorted.reduce((total, v) => total + v, 0) / sorted.length) * 10,
) / 10;
const p50 = sorted[Math.floor((sorted.length - 1) / 2)];
const max = sorted[sorted.length - 1];
return { mean, p50, max };
}
/** Pure measurement helpers for the benchmark suite. */
import { CanonMessage } from "../src/view";
const normalize = (s: string): string =>
s.toLowerCase().replace(/\s+/g, " ").trim();
/** Case/whitespace-normalized substring match against accepted answers. */
export function gradeAnswer(response: string, accepted: string[]): boolean {
const norm = normalize(response);
return accepted.some((candidate) => norm.includes(normalize(candidate)));
}
export interface StructuralReport {
singleLeadingSystem: boolean;
noOrphanToolResults: boolean;
hasUserMessage: boolean;
ok: boolean;
}
/** Validate a compacted view: system text + tail messages. */
export function checkViewStructure(
systemText: string | undefined,
tail: CanonMessage[],
): StructuralReport {
void systemText; // the system text lives outside the tail by construction
const singleLeadingSystem = tail.every((m) => m.role !== "system");
let noOrphanToolResults = true;
tail.forEach((m, i) => {
if (m.role !== "tool") return;
const prev = tail[i - 1];
const anchored =
prev !== undefined &&
(prev.role === "tool" ||
(prev.role === "assistant" && (prev.toolCalls ?? []).length > 0));
if (!anchored) noOrphanToolResults = false;
});
const hasUserMessage = tail.some((m) => m.role === "user");
return {
singleLeadingSystem,
noOrphanToolResults,
hasUserMessage,
ok: singleLeadingSystem && noOrphanToolResults && hasUserMessage,
};
}
export function reductionPercent(
beforeTokens: number,
afterTokens: number,
): number {
if (beforeTokens <= 0) return 0;
return Math.round((1 - afterTokens / beforeTokens) * 100);
}
export function latencyStats(samplesMs: number[]): {
mean: number;
p50: number;
max: number;
} {
if (samplesMs.length === 0) return { mean: 0, p50: 0, max: 0 };
const sorted = [...samplesMs].sort((x, y) => x - y);
const mean =
Math.round(
(sorted.reduce((total, v) => total + v, 0) / sorted.length) * 10,
) / 10;
const p50 = sorted[Math.floor((sorted.length - 1) / 2)];
const max = sorted[sorted.length - 1];
return { mean, p50, max };
}