bench / fixtures.ts
bench / fixtures.ts
/**
* Deterministic synthetic conversations for benchmarking. Pure data — a
* seeded PRNG, no I/O — emitting the same CanonMessage[] the plugin's pure
* modules consume, with seeded facts whose recall can be graded after
* compaction.
*/
import { CanonMessage } from "../src/view";
export interface SeededFact {
id: string;
question: string;
/** Any of these substrings in the answer counts as correct. */
accepted: string[];
}
export interface Fixture {
name: string;
msgs: CanonMessage[];
facts: SeededFact[];
}
/** Small deterministic PRNG (mulberry32). */
export function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
a |= 0;
a = (a + 0x6d2b79f5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
const WORDS =
"system module pipeline buffer index worker queue schema branch commit deploy replica shard cache token stream handler adapter registry payload".split(
" ",
);
function filler(rng: () => number, words: number): string {
const parts: string[] = [];
for (let i = 0; i < words; i++) {
parts.push(WORDS[Math.floor(rng() * WORDS.length)]);
}
return parts.join(" ");
}
const u = (text: string): CanonMessage => ({ role: "user", text });
const a = (text: string): CanonMessage => ({ role: "assistant", text });
const toolRound = (
name: string,
args: string,
result: string,
): CanonMessage[] => [
{
role: "assistant",
text: "",
toolCalls: [{ name, args }],
},
{ role: "tool", text: "", toolResults: [{ content: result }] },
];
export function codingAgentFixture(seed = 1): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u(
"We are building project VELVET-OTTER-19. Remember: the database password hint is " +
'"quantum-pistachio-7788", we chose PostgreSQL over MongoDB for transactional ' +
"integrity, my favorite constant is e = 2.71828, the deadline is March 14th and " +
"the budget is $47,250. Now start implementing the ingestion service.",
),
a(
"Noted all project constraints for VELVET-OTTER-19. Starting on the ingestion service.",
),
];
for (let round = 0; round < 10; round++) {
msgs.push(u(`Continue with step ${round + 1}: ${filler(rng, 12)}.`));
msgs.push(
...toolRound(
"write_file",
`{"path":"src/step${round + 1}.ts"}`,
`wrote src/step${round + 1}.ts\n${filler(rng, 120)}`,
),
);
msgs.push(a(`Step ${round + 1} done: ${filler(rng, 30)}.`));
}
msgs.push(u("Good. Pause here."));
msgs.push(a("Paused."));
return {
name: "coding-agent",
msgs,
facts: [
{
id: "codename",
question: "What is the project codename?",
accepted: ["VELVET-OTTER-19"],
},
{
id: "password-hint",
question: "What is the database password hint?",
accepted: ["quantum-pistachio-7788"],
},
{
id: "db-choice",
question:
"Which database did we choose and why?",
accepted: ["PostgreSQL"],
},
{
id: "constant",
question: "What is my favorite constant's value?",
accepted: ["2.71828"],
},
{
id: "budget",
question: "What is the project budget?",
accepted: ["$47,250", "47,250", "47250"],
},
],
};
}
export function researchFixture(seed = 2): Fixture {
const rng = mulberry32(seed);
const findings = [
{
id: "gallium",
text: "Gallium melts at 29.76 C (source: CRC Handbook).",
question: "At what temperature does gallium melt?",
accepted: ["29.76"],
},
{
id: "voyager",
text: "Voyager 1 is 24.4 billion km from Earth (source: NASA JPL, 2024).",
question: "How far is Voyager 1 from Earth?",
accepted: ["24.4"],
},
{
id: "trieste",
text: "The bathyscaphe Trieste reached 10,911 m depth in 1960 (source: US Navy records).",
question: "What depth did the Trieste reach?",
accepted: ["10,911", "10911"],
},
];
const msgs: CanonMessage[] = [
u("Research these physical facts for me and keep careful sources."),
];
for (const f of findings) {
msgs.push(u(`Look into: ${f.question} ${filler(rng, 8)}`));
msgs.push(
...toolRound(
"web_search",
`{"query":"${f.id}"}`,
`search results: ${filler(rng, 100)}`,
),
);
msgs.push(a(`Finding: ${f.text} Context: ${filler(rng, 40)}.`));
}
for (let i = 0; i < 4; i++) {
msgs.push(u(`Also summarize ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 60)));
}
return {
name: "research",
msgs,
facts: findings.map((f) => ({
id: f.id,
question: f.question,
accepted: f.accepted,
})),
};
}
export function contradictoryUpdatesFixture(seed = 3): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u("Project plan: the budget is $10,000 and the launch city is Berlin."),
a("Recorded: budget $10,000, launch in Berlin."),
];
for (let i = 0; i < 4; i++) {
msgs.push(u(`Work on ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 50)));
}
msgs.push(
u(
"Update: the budget is revised to $12,500 (was $10,000), and launch moves from Berlin to Lisbon.",
),
);
msgs.push(a("Updated: budget now $12,500; launch city now Lisbon."));
for (let i = 0; i < 4; i++) {
msgs.push(u(`Continue with ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 50)));
}
return {
name: "contradictory-updates",
msgs,
facts: [
{
id: "budget-revised",
question: "What is the current project budget?",
accepted: ["12,500", "12500"],
},
{
id: "city-revised",
question: "What is the current launch city?",
accepted: ["Lisbon"],
},
{
id: "original-city",
question: "Which city was the launch originally planned for?",
accepted: ["Berlin"],
},
],
};
}
export function toolSpamFixture(seed = 4): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u(
"Analyze these logs. Important: the incident ticket is INC-88412, the root " +
"cause was a leap-second bug, and the affected service is billing-gateway.",
),
a("Noted INC-88412. Reading the logs."),
];
for (let i = 0; i < 6; i++) {
msgs.push(
...toolRound(
"read_file_range",
`{"path":"logs/app-${i}.log"}`,
`log lines:\n${filler(rng, 1400)}`,
),
);
msgs.push(a(`Batch ${i + 1} scanned: ${filler(rng, 20)}.`));
}
msgs.push(u("Summarize what matters."));
msgs.push(a("The signal is in INC-88412 on billing-gateway."));
return {
name: "tool-spam",
msgs,
facts: [
{
id: "ticket",
question: "What is the incident ticket number?",
accepted: ["INC-88412"],
},
{
id: "root-cause",
question: "What was the root cause?",
accepted: ["leap-second", "leap second"],
},
{
id: "service",
question: "Which service was affected?",
accepted: ["billing-gateway"],
},
],
};
}
export function allFixtures(seed = 1): Fixture[] {
return [
codingAgentFixture(seed),
researchFixture(seed + 1),
contradictoryUpdatesFixture(seed + 2),
toolSpamFixture(seed + 3),
];
}
/**
* Deterministic synthetic conversations for benchmarking. Pure data — a
* seeded PRNG, no I/O — emitting the same CanonMessage[] the plugin's pure
* modules consume, with seeded facts whose recall can be graded after
* compaction.
*/
import { CanonMessage } from "../src/view";
export interface SeededFact {
id: string;
question: string;
/** Any of these substrings in the answer counts as correct. */
accepted: string[];
}
export interface Fixture {
name: string;
msgs: CanonMessage[];
facts: SeededFact[];
}
/** Small deterministic PRNG (mulberry32). */
export function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
a |= 0;
a = (a + 0x6d2b79f5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
const WORDS =
"system module pipeline buffer index worker queue schema branch commit deploy replica shard cache token stream handler adapter registry payload".split(
" ",
);
function filler(rng: () => number, words: number): string {
const parts: string[] = [];
for (let i = 0; i < words; i++) {
parts.push(WORDS[Math.floor(rng() * WORDS.length)]);
}
return parts.join(" ");
}
const u = (text: string): CanonMessage => ({ role: "user", text });
const a = (text: string): CanonMessage => ({ role: "assistant", text });
const toolRound = (
name: string,
args: string,
result: string,
): CanonMessage[] => [
{
role: "assistant",
text: "",
toolCalls: [{ name, args }],
},
{ role: "tool", text: "", toolResults: [{ content: result }] },
];
export function codingAgentFixture(seed = 1): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u(
"We are building project VELVET-OTTER-19. Remember: the database password hint is " +
'"quantum-pistachio-7788", we chose PostgreSQL over MongoDB for transactional ' +
"integrity, my favorite constant is e = 2.71828, the deadline is March 14th and " +
"the budget is $47,250. Now start implementing the ingestion service.",
),
a(
"Noted all project constraints for VELVET-OTTER-19. Starting on the ingestion service.",
),
];
for (let round = 0; round < 10; round++) {
msgs.push(u(`Continue with step ${round + 1}: ${filler(rng, 12)}.`));
msgs.push(
...toolRound(
"write_file",
`{"path":"src/step${round + 1}.ts"}`,
`wrote src/step${round + 1}.ts\n${filler(rng, 120)}`,
),
);
msgs.push(a(`Step ${round + 1} done: ${filler(rng, 30)}.`));
}
msgs.push(u("Good. Pause here."));
msgs.push(a("Paused."));
return {
name: "coding-agent",
msgs,
facts: [
{
id: "codename",
question: "What is the project codename?",
accepted: ["VELVET-OTTER-19"],
},
{
id: "password-hint",
question: "What is the database password hint?",
accepted: ["quantum-pistachio-7788"],
},
{
id: "db-choice",
question:
"Which database did we choose and why?",
accepted: ["PostgreSQL"],
},
{
id: "constant",
question: "What is my favorite constant's value?",
accepted: ["2.71828"],
},
{
id: "budget",
question: "What is the project budget?",
accepted: ["$47,250", "47,250", "47250"],
},
],
};
}
export function researchFixture(seed = 2): Fixture {
const rng = mulberry32(seed);
const findings = [
{
id: "gallium",
text: "Gallium melts at 29.76 C (source: CRC Handbook).",
question: "At what temperature does gallium melt?",
accepted: ["29.76"],
},
{
id: "voyager",
text: "Voyager 1 is 24.4 billion km from Earth (source: NASA JPL, 2024).",
question: "How far is Voyager 1 from Earth?",
accepted: ["24.4"],
},
{
id: "trieste",
text: "The bathyscaphe Trieste reached 10,911 m depth in 1960 (source: US Navy records).",
question: "What depth did the Trieste reach?",
accepted: ["10,911", "10911"],
},
];
const msgs: CanonMessage[] = [
u("Research these physical facts for me and keep careful sources."),
];
for (const f of findings) {
msgs.push(u(`Look into: ${f.question} ${filler(rng, 8)}`));
msgs.push(
...toolRound(
"web_search",
`{"query":"${f.id}"}`,
`search results: ${filler(rng, 100)}`,
),
);
msgs.push(a(`Finding: ${f.text} Context: ${filler(rng, 40)}.`));
}
for (let i = 0; i < 4; i++) {
msgs.push(u(`Also summarize ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 60)));
}
return {
name: "research",
msgs,
facts: findings.map((f) => ({
id: f.id,
question: f.question,
accepted: f.accepted,
})),
};
}
export function contradictoryUpdatesFixture(seed = 3): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u("Project plan: the budget is $10,000 and the launch city is Berlin."),
a("Recorded: budget $10,000, launch in Berlin."),
];
for (let i = 0; i < 4; i++) {
msgs.push(u(`Work on ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 50)));
}
msgs.push(
u(
"Update: the budget is revised to $12,500 (was $10,000), and launch moves from Berlin to Lisbon.",
),
);
msgs.push(a("Updated: budget now $12,500; launch city now Lisbon."));
for (let i = 0; i < 4; i++) {
msgs.push(u(`Continue with ${filler(rng, 10)}.`));
msgs.push(a(filler(rng, 50)));
}
return {
name: "contradictory-updates",
msgs,
facts: [
{
id: "budget-revised",
question: "What is the current project budget?",
accepted: ["12,500", "12500"],
},
{
id: "city-revised",
question: "What is the current launch city?",
accepted: ["Lisbon"],
},
{
id: "original-city",
question: "Which city was the launch originally planned for?",
accepted: ["Berlin"],
},
],
};
}
export function toolSpamFixture(seed = 4): Fixture {
const rng = mulberry32(seed);
const msgs: CanonMessage[] = [
u(
"Analyze these logs. Important: the incident ticket is INC-88412, the root " +
"cause was a leap-second bug, and the affected service is billing-gateway.",
),
a("Noted INC-88412. Reading the logs."),
];
for (let i = 0; i < 6; i++) {
msgs.push(
...toolRound(
"read_file_range",
`{"path":"logs/app-${i}.log"}`,
`log lines:\n${filler(rng, 1400)}`,
),
);
msgs.push(a(`Batch ${i + 1} scanned: ${filler(rng, 20)}.`));
}
msgs.push(u("Summarize what matters."));
msgs.push(a("The signal is in INC-88412 on billing-gateway."));
return {
name: "tool-spam",
msgs,
facts: [
{
id: "ticket",
question: "What is the incident ticket number?",
accepted: ["INC-88412"],
},
{
id: "root-cause",
question: "What was the root cause?",
accepted: ["leap-second", "leap second"],
},
{
id: "service",
question: "Which service was affected?",
accepted: ["billing-gateway"],
},
],
};
}
export function allFixtures(seed = 1): Fixture[] {
return [
codingAgentFixture(seed),
researchFixture(seed + 1),
contradictoryUpdatesFixture(seed + 2),
toolSpamFixture(seed + 3),
];
}