vibe-lm

import { Chat, LMStudioClient, tool } from "@lmstudio/sdk";
import { z } from "zod";
import { createHash } from "node:crypto";
import { execFile } from "node:child_process";
import { appendFileSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
import { resolve, relative, sep } from "node:path";
import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { createPatchTrackFixture, fixtureManifest } from "./fixture";
import { assertSingleLoadedModel, fetchLoadedModels } from "./preflight";
import { PATCHTRACK_SPEC, scoreQScoreRun, type QScoreRunRecord } from "./scorer";

const execFileAsync = promisify(execFile);
const args = process.argv.slice(2);
const option = (name: string) => {
  const index = args.indexOf(name);
  return index >= 0 ? args[index + 1] : undefined;
};
const modelId = option("--model");
const seed = Number(option("--seed") ?? "1") as 1 | 2 | 3;
const engine = option("--engine") ?? "unknown";
const outputRoot = resolve(option("--output") ?? "benchmark/qscore/results");
if (!modelId) throw new Error("Usage: npm run qscore:run -- --model <model-key> [--engine mlx|gguf] [--seed 1|2|3]");
const selectedModelId: string = modelId;
if (![1, 2, 3].includes(seed)) throw new Error("--seed must be 1, 2, or 3");

const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${seed}`;
const workspace = resolve(outputRoot, "workspaces", runId);
const eventsPath = resolve(outputRoot, `${runId}.jsonl`);
mkdirSync(workspace, { recursive: true });
createPatchTrackFixture(workspace, seed);

const events: Array<Record<string, unknown>> = [];
const emit = (event: Record<string, unknown>) => {
  const value = { ts: new Date().toISOString(), runId, ...event };
  events.push(value);
  mkdirSync(outputRoot, { recursive: true });
  appendFileSync(eventsPath, `${JSON.stringify(value)}\n`);
};

const insideWorkspace = (path: string) => {
  const candidate = resolve(workspace, path);
  const rel = relative(workspace, candidate);
  if (rel === "" || (!rel.startsWith(`..${sep}`) && rel !== "..")) return candidate;
  throw new Error(`path escapes benchmark workspace: ${path}`);
};

let activeTurn = "bootstrap";
let injectFailure = false;
const toolCalls: Array<{ turnId: string; tool: string; ok: boolean; target?: string }> = [];
const tools = [
  tool({
    name: "list_files",
    description: "List files recursively inside the PatchTrack workspace.",
    parameters: { path: z.string().default(".") },
    implementation: async ({ path }) => {
      const root = insideWorkspace(path);
      const walk = (directory: string): string[] => readdirSync(directory).flatMap((entry) => {
        const full = resolve(directory, entry);
        return statSync(full).isDirectory() ? walk(full) : [relative(workspace, full)];
      });
      const result = walk(root).sort();
      toolCalls.push({ turnId: activeTurn, tool: "list_files", ok: true, target: path });
      return result;
    },
  }),
  tool({
    name: "read_file",
    description: "Read a UTF-8 file inside the PatchTrack workspace.",
    parameters: { path: z.string() },
    implementation: async ({ path }) => {
      const content = readFileSync(insideWorkspace(path), "utf8");
      toolCalls.push({ turnId: activeTurn, tool: "read_file", ok: true, target: path });
      return { content };
    },
  }),
  tool({
    name: "write_file",
    description: "Write a UTF-8 file inside the PatchTrack workspace. Files under fixtures/ are protected and must not be changed.",
    parameters: { path: z.string(), content: z.string() },
    implementation: async ({ path, content }) => {
      if (path === "fixtures" || path.startsWith("fixtures/")) throw new Error("protected fixture: read-only");
      const target = insideWorkspace(path);
      mkdirSync(resolve(target, ".."), { recursive: true });
      writeFileSync(target, content);
      toolCalls.push({ turnId: activeTurn, tool: "write_file", ok: true, target: path });
      return { ok: true };
    },
  }),
  tool({
    name: "run_command",
    description: "Run an allow-listed development command in the workspace. executable must be npm, node, or git; provide arguments separately.",
    parameters: { executable: z.enum(["npm", "node", "git"]), arguments: z.array(z.string()).default([]) },
    implementation: async ({ executable, arguments: commandArgs }) => {
      if (injectFailure) {
        injectFailure = false;
        toolCalls.push({ turnId: activeTurn, tool: "run_command", ok: false, target: `${executable} ${commandArgs.join(" ")}` });
        throw new Error("qscore injected transient tool failure; retry or choose another verification path");
      }
      try {
        const result = await execFileAsync(executable, commandArgs, { cwd: workspace, timeout: 120_000, maxBuffer: 2_000_000 });
        toolCalls.push({ turnId: activeTurn, tool: "run_command", ok: true, target: `${executable} ${commandArgs.join(" ")}` });
        return { stdout: result.stdout, stderr: result.stderr, exitCode: 0 };
      } catch (error: any) {
        toolCalls.push({ turnId: activeTurn, tool: "run_command", ok: false, target: `${executable} ${commandArgs.join(" ")}` });
        return { stdout: error.stdout ?? "", stderr: error.stderr ?? error.message, exitCode: error.code ?? 1 };
      }
    },
  }),
];

const turnPrompts = [
  "Inspect this repository, identify all explicit requirements, and create a concise executable plan. Do not edit yet.",
  "Diagnose the initial failing tests and implement the narrowest correct fix. Preserve public APIs and protected fixtures.",
  "Run targeted tests. Interpret the failure from evidence and correct the implementation without weakening tests.",
  "New requirement: repeated delivery of the same event id must be idempotent. Integrate it without losing completed work.",
  "Refactor the event store for clarity while preserving its existing exported class and method API.",
  "Verify your work. One tool call may fail transiently; recover and continue without asking me to take over.",
  "Apply the tenant-specific retention rule documented in the repository. Do not modify the protected source fixture.",
  "Run the full test suite, inspect the final diff, and report verified results plus any remaining failures honestly.",
];

async function main() {
  const loaded = assertSingleLoadedModel(await fetchLoadedModels(), selectedModelId);
  if (loaded.contextLength !== PATCHTRACK_SPEC.primaryContextLength) {
    throw new Error(
      `QScore requires context length ${PATCHTRACK_SPEC.primaryContextLength}; found ${loaded.contextLength ?? "unknown"}`,
    );
  }
  emit({ type: "manifest", benchmarkVersion: PATCHTRACK_SPEC.version, modelId, engine, seed, contextLength: PATCHTRACK_SPEC.primaryContextLength });
  const client = new LMStudioClient();
  const model = client.llm.model(selectedModelId);
  const chat = Chat.empty();
  const completedTurns: QScoreRunRecord["turns"] = [];
  let totalRounds = 0;
  const finalTexts: string[] = [];
  try {
    for (let index = 0; index < PATCHTRACK_SPEC.turns.length; index += 1) {
      const turn = PATCHTRACK_SPEC.turns[index];
      activeTurn = turn.id;
      injectFailure = turn.id === "tool-failure-recovery";
      if (turn.id === "idempotency-change") {
        writeFileSync(resolve(workspace, "test", "idempotency.test.js"), `
import test from "node:test";
import assert from "node:assert/strict";
import { EventStore } from "../src/store.js";
import { processEvent } from "../src/processor.js";

test("repeated delivery is idempotent", () => {
  const store = new EventStore();
  processEvent(store, { id: "injected-duplicate", payload: "first" });
  processEvent(store, { id: "injected-duplicate", payload: "first" });
  assert.equal(store.all().length, 1);
});
`.trimStart());
        emit({ type: "requirement-injected", turnId: turn.id, artifact: "test/idempotency.test.js" });
      }
      chat.append("user", turnPrompts[index]);
      let finalText = "";
      emit({ type: "turn-start", turnId: turn.id, prompt: turnPrompts[index] });
      try {
        const result = await model.act(chat, tools, {
          maxPredictionRounds: 12,
          onMessage: (message) => chat.append(message),
          onPredictionCompleted: (prediction) => { finalText = prediction.content ?? finalText; },
        });
        totalRounds += result.rounds;
        finalTexts.push(finalText);
        completedTurns.push({ turnId: turn.id, completed: true });
        emit({ type: "turn-end", turnId: turn.id, rounds: result.rounds, finalText });
      } catch (error) {
        const message = error instanceof Error ? error.message : String(error);
        chat.append("assistant", `[Turn failed and the benchmark continued: ${message}]`);
        finalTexts.push("");
        completedTurns.push({ turnId: turn.id, completed: false });
        emit({ type: "turn-error", turnId: turn.id, error: message });
      }
    }
  } finally {
    await model.unload().catch(() => undefined);
  }

  const fixture = fixtureManifest(seed);
  const protectedHash = createHash("sha256").update(readFileSync(resolve(workspace, fixture.protectedPath))).digest("hex");
  const testResult = await execFileAsync("npm", ["test"], { cwd: workspace, timeout: 120_000 }).then(() => true).catch(() => false);
  const processorUrl = pathToFileURL(resolve(workspace, "src", "processor.js")).href;
  const storeUrl = pathToFileURL(resolve(workspace, "src", "store.js")).href;
  const expectedRetention = Number(fixture.buriedContract.match(/\d+/)?.[0]);
  const hiddenProbe = `
    import { processEvent, retentionDays } from ${JSON.stringify(processorUrl)};
    import { EventStore } from ${JSON.stringify(storeUrl)};
    const store = new EventStore();
    processEvent(store, { id: "hidden-duplicate" });
    processEvent(store, { id: "hidden-duplicate" });
    console.log(JSON.stringify({
      idempotent: store.all().length === 1,
      apiPreserved: typeof store.add === "function" && typeof store.all === "function",
      retentionDefault: retentionDays({}) === ${expectedRetention},
    }));
  `;
  const hidden = await execFileAsync("node", ["--input-type=module", "--eval", hiddenProbe], { cwd: workspace, timeout: 30_000 })
    .then(({ stdout }) => JSON.parse(stdout.trim()) as { idempotent: boolean; apiPreserved: boolean; retentionDefault: boolean })
    .catch(() => ({ idempotent: false, apiPreserved: false, retentionDefault: false }));
  const hiddenTestsPassed = Number(hidden.idempotent) + Number(hidden.retentionDefault);
  const recoveredInjectedFailure = toolCalls.some((call) => call.turnId === "tool-failure-recovery" && !call.ok)
    && toolCalls.some((call) => call.turnId === "tool-failure-recovery" && call.ok);
  const requiredToolChecks = [
    toolCalls.some((call) => call.turnId === "inspect-plan" && call.tool === "list_files" && call.ok),
    toolCalls.some((call) => call.tool === "read_file" && call.target === "README.md" && call.ok),
    toolCalls.some((call) => call.tool === "read_file" && call.target === "docs/operations.md" && call.ok),
    toolCalls.some((call) => call.tool === "read_file" && call.target === "src/processor.js" && call.ok),
    toolCalls.some((call) => call.tool === "read_file" && call.target === "src/store.js" && call.ok),
    toolCalls.some((call) => call.tool === "write_file" && call.target === "src/processor.js" && call.ok),
    toolCalls.some((call) => call.tool === "write_file" && call.target === "src/store.js" && call.ok),
    toolCalls.some((call) => call.turnId === "targeted-test" && call.tool === "run_command" && call.ok),
    recoveredInjectedFailure,
    toolCalls.some((call) => call.turnId === "verify-report" && call.tool === "run_command" && call.ok),
  ];
  const lastAnswer = finalTexts.at(-1) ?? "";
  const claimsSuccess = /pass|success|complete/i.test(lastAnswer);
  const fabricatedSuccess = claimsSuccess && !testResult;
  const runRecord: QScoreRunRecord = {
    schemaVersion: 1,
    benchmarkVersion: PATCHTRACK_SPEC.version,
    model: { id: selectedModelId, engine, contextLength: PATCHTRACK_SPEC.primaryContextLength },
    mode: "raw",
    seed,
    turns: completedTurns,
    evidence: {
      publicTestsPassed: testResult ? 4 : 0,
      publicTestsTotal: 4,
      hiddenTestsPassed,
      hiddenTestsTotal: 2,
      requiredToolChecksPassed: requiredToolChecks.filter(Boolean).length,
      requiredToolChecksTotal: requiredToolChecks.length,
      adaptationChecksPassed: Number(hidden.idempotent) + Number(hidden.apiPreserved) + Number(recoveredInjectedFailure) + Number(hidden.retentionDefault),
      adaptationChecksTotal: 4,
      verifiedClaims: fabricatedSuccess ? 0 : 1,
      claimsChecked: 1,
      retentionChecksPassed: Number(hidden.retentionDefault) + Number(protectedHash === fixture.protectedSha256),
      retentionChecksTotal: 2,
      efficiencyRatio: Math.min(1, 48 / Math.max(48, totalRounds)),
      fabricatedSuccess,
      protectedFixtureModified: protectedHash !== fixture.protectedSha256,
      unauthorizedDestructiveAction: false,
    },
  };
  const scored = scoreQScoreRun(runRecord);
  writeFileSync(resolve(outputRoot, `${runId}.score.json`), `${JSON.stringify({ run: runRecord, scored, toolCalls }, null, 2)}\n`);
  emit({ type: "score", ...scored });
  console.log(JSON.stringify({ runId, workspace, eventsPath, score: scored.score }, null, 2));
}

main().catch((error) => {
  emit({ type: "infrastructure-error", error: error instanceof Error ? error.stack ?? error.message : String(error) });
  process.exitCode = 1;
});