{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3.8-27b",
  "driver": "in-process",
  "contextLength": 8192,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 14454,
        "contextOverflow": false,
        "promptTokens": 26628,
        "predictedTokens": 1101
      },
      "contextLength": 8192
    },
    {
      "scenario": "S2",
      "title": "out-of-band edit, conflict recovery",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "out-of-band edit injected",
          "ok": true
        },
        {
          "name": "reviewer note preserved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "recovered from the out-of-band change",
          "ok": true
        },
        {
          "name": "conflict recovery path: targeted replace kept the note",
          "ok": true
        },
        {
          "name": "EDIT_CONFLICT envelopes after injection: 0",
          "ok": true
        },
        {
          "name": "EDIT_CONFLICT envelopes total: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 12,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_tasks": 4,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 9,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 29778,
        "contextOverflow": false,
        "promptTokens": 65378,
        "predictedTokens": 2352
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: completed",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: true",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 10,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_plan": 2,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 9,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 29425,
        "contextOverflow": false,
        "promptTokens": 73102,
        "predictedTokens": 2288
      },
      "contextLength": 16384
    },
    {
      "scenario": "S4",
      "title": "manual mode: stage, /accept, resume",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "nothing applied before approval",
          "ok": true
        },
        {
          "name": "edit staged as pending_approval",
          "ok": true
        },
        {
          "name": "at least one approval consumed",
          "ok": true
        },
        {
          "name": "transaction applied",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "used resume call: yes",
          "ok": true
        },
        {
          "name": "approvals consumed: 2",
          "ok": true
        },
        {
          "name": "pending approvals left: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 14,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_tasks": 4,
          "workspace_edit": 2,
          "workspace_command": 2
        },
        "rounds": 12,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 33232,
        "contextOverflow": false,
        "promptTokens": 108881,
        "predictedTokens": 2671
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 6,
        "toolCallsByName": {
          "workspace_notes": 1,
          "workspace_inspect": 1,
          "workspace_command": 4
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 23118,
        "contextOverflow": false,
        "promptTokens": 28953,
        "predictedTokens": 1773
      },
      "contextLength": 8192
    },
    {
      "scenario": "S6",
      "title": "git branch, add, commit",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "NOTES.md committed",
          "ok": false,
          "detail": "README.md"
        },
        {
          "name": "two commits on the history",
          "ok": false,
          "detail": "6405418 initial"
        },
        {
          "name": "commit message mentions live",
          "ok": false,
          "detail": "6405418 initial"
        },
        {
          "name": "feature/live branch exists",
          "ok": true
        },
        {
          "name": "working tree clean",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 4,
        "toolCallsByName": {
          "workspace_inspect": 1,
          "workspace_vcs": 2,
          "workspace_notes": 1
        },
        "rounds": 3,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 44570,
        "contextOverflow": true,
        "promptTokens": 11113,
        "predictedTokens": 3197
      },
      "error": "context overflow: prompt 8351 > 8192",
      "contextLength": 8192
    },
    {
      "scenario": "S6",
      "title": "git branch, add, commit",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "NOTES.md committed",
          "ok": true
        },
        {
          "name": "two commits on the history",
          "ok": true
        },
        {
          "name": "commit message mentions live",
          "ok": true
        },
        {
          "name": "feature/live branch exists",
          "ok": true
        },
        {
          "name": "working tree clean",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 11,
        "toolCallsByName": {
          "workspace_inspect": 2,
          "workspace_notes": 1,
          "workspace_vcs": 7,
          "workspace_edit": 1
        },
        "rounds": 10,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 42312,
        "contextOverflow": false,
        "promptTokens": 67527,
        "predictedTokens": 3018
      },
      "contextLength": 12288
    },
    {
      "scenario": "S7",
      "title": "web research: fetch a page, record a sourced note",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "research project created",
          "ok": true
        },
        {
          "name": "source recorded with a sha256",
          "ok": true
        },
        {
          "name": "source content stored on disk",
          "ok": true
        },
        {
          "name": "note references a recorded source",
          "ok": true
        },
        {
          "name": "research projects: 1",
          "ok": true
        },
        {
          "name": "sources: 1",
          "ok": true
        },
        {
          "name": "notes: 1",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 10,
        "toolCallsByName": {
          "workspace_research": 3,
          "workspace_inspect": 7
        },
        "rounds": 11,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 53676,
        "contextOverflow": false,
        "promptTokens": 106436,
        "predictedTokens": 4141
      },
      "contextLength": 16384
    },
    {
      "scenario": "S8",
      "title": "sub-agent run: delegate the fix",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "a sub-agent run was started",
          "ok": true
        },
        {
          "name": "run reached completed",
          "ok": true
        },
        {
          "name": "filesChanged is non-empty",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "agent runs: 1",
          "ok": true
        },
        {
          "name": "agent tool calls: 7",
          "ok": true
        },
        {
          "name": "agent passes used: 2",
          "ok": true
        },
        {
          "name": "agent commitEdits: true",
          "ok": true
        },
        {
          "name": "files changed: src/math.js",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_notes": 1,
          "workspace_inspect": 5,
          "workspace_agent": 1
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 52856,
        "contextOverflow": false,
        "promptTokens": 30831,
        "predictedTokens": 1728
      },
      "contextLength": 16384
    },
    {
      "scenario": "S9",
      "title": "full project from an empty directory",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "package.json parses with scripts.test running node --test",
          "ok": true
        },
        {
          "name": "at least one file under src/",
          "ok": true
        },
        {
          "name": "at least one test file under tests/",
          "ok": true
        },
        {
          "name": "node --test exits 0 with at least three ok lines",
          "ok": true
        },
        {
          "name": "score: 4/4",
          "ok": true
        },
        {
          "name": "files created: package.json, sample.txt, src/wordcount.js, tests/wordcount.test.js",
          "ok": true
        },
        {
          "name": "ok lines: 4",
          "ok": true
        },
        {
          "name": "node --test exit: 0",
          "ok": true
        },
        {
          "name": "package private: true",
          "ok": true
        },
        {
          "name": "applied transactions: 2",
          "ok": true
        },
        {
          "name": "model ran node --test: yes (1)",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 12,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 1,
          "workspace_tasks": 4,
          "workspace_edit": 2,
          "workspace_command": 2,
          "workspace_vcs": 1
        },
        "rounds": 10,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 49340,
        "contextOverflow": true,
        "promptTokens": 101472,
        "predictedTokens": 4046
      },
      "contextLength": 16384
    }
  ]
}

{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3.8-27b",
  "driver": "in-process",
  "contextLength": 8192,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 14454,
        "contextOverflow": false,
        "promptTokens": 26628,
        "predictedTokens": 1101
      },
      "contextLength": 8192
    },
    {
      "scenario": "S2",
      "title": "out-of-band edit, conflict recovery",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "out-of-band edit injected",
          "ok": true
        },
        {
          "name": "reviewer note preserved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "recovered from the out-of-band change",
          "ok": true
        },
        {
          "name": "conflict recovery path: targeted replace kept the note",
          "ok": true
        },
        {
          "name": "EDIT_CONFLICT envelopes after injection: 0",
          "ok": true
        },
        {
          "name": "EDIT_CONFLICT envelopes total: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 12,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_tasks": 4,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 9,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 29778,
        "contextOverflow": false,
        "promptTokens": 65378,
        "predictedTokens": 2352
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: completed",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: true",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 10,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_plan": 2,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 9,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 29425,
        "contextOverflow": false,
        "promptTokens": 73102,
        "predictedTokens": 2288
      },
      "contextLength": 16384
    },
    {
      "scenario": "S4",
      "title": "manual mode: stage, /accept, resume",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "nothing applied before approval",
          "ok": true
        },
        {
          "name": "edit staged as pending_approval",
          "ok": true
        },
        {
          "name": "at least one approval consumed",
          "ok": true
        },
        {
          "name": "transaction applied",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "used resume call: yes",
          "ok": true
        },
        {
          "name": "approvals consumed: 2",
          "ok": true
        },
        {
          "name": "pending approvals left: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 14,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 4,
          "workspace_tasks": 4,
          "workspace_edit": 2,
          "workspace_command": 2
        },
        "rounds": 12,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 33232,
        "contextOverflow": false,
        "promptTokens": 108881,
        "predictedTokens": 2671
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 6,
        "toolCallsByName": {
          "workspace_notes": 1,
          "workspace_inspect": 1,
          "workspace_command": 4
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 23118,
        "contextOverflow": false,
        "promptTokens": 28953,
        "predictedTokens": 1773
      },
      "contextLength": 8192
    },
    {
      "scenario": "S6",
      "title": "git branch, add, commit",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "NOTES.md committed",
          "ok": false,
          "detail": "README.md"
        },
        {
          "name": "two commits on the history",
          "ok": false,
          "detail": "6405418 initial"
        },
        {
          "name": "commit message mentions live",
          "ok": false,
          "detail": "6405418 initial"
        },
        {
          "name": "feature/live branch exists",
          "ok": true
        },
        {
          "name": "working tree clean",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 4,
        "toolCallsByName": {
          "workspace_inspect": 1,
          "workspace_vcs": 2,
          "workspace_notes": 1
        },
        "rounds": 3,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 44570,
        "contextOverflow": true,
        "promptTokens": 11113,
        "predictedTokens": 3197
      },
      "error": "context overflow: prompt 8351 > 8192",
      "contextLength": 8192
    },
    {
      "scenario": "S6",
      "title": "git branch, add, commit",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "NOTES.md committed",
          "ok": true
        },
        {
          "name": "two commits on the history",
          "ok": true
        },
        {
          "name": "commit message mentions live",
          "ok": true
        },
        {
          "name": "feature/live branch exists",
          "ok": true
        },
        {
          "name": "working tree clean",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 11,
        "toolCallsByName": {
          "workspace_inspect": 2,
          "workspace_notes": 1,
          "workspace_vcs": 7,
          "workspace_edit": 1
        },
        "rounds": 10,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 42312,
        "contextOverflow": false,
        "promptTokens": 67527,
        "predictedTokens": 3018
      },
      "contextLength": 12288
    },
    {
      "scenario": "S7",
      "title": "web research: fetch a page, record a sourced note",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "research project created",
          "ok": true
        },
        {
          "name": "source recorded with a sha256",
          "ok": true
        },
        {
          "name": "source content stored on disk",
          "ok": true
        },
        {
          "name": "note references a recorded source",
          "ok": true
        },
        {
          "name": "research projects: 1",
          "ok": true
        },
        {
          "name": "sources: 1",
          "ok": true
        },
        {
          "name": "notes: 1",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 10,
        "toolCallsByName": {
          "workspace_research": 3,
          "workspace_inspect": 7
        },
        "rounds": 11,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 53676,
        "contextOverflow": false,
        "promptTokens": 106436,
        "predictedTokens": 4141
      },
      "contextLength": 16384
    },
    {
      "scenario": "S8",
      "title": "sub-agent run: delegate the fix",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "a sub-agent run was started",
          "ok": true
        },
        {
          "name": "run reached completed",
          "ok": true
        },
        {
          "name": "filesChanged is non-empty",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "agent runs: 1",
          "ok": true
        },
        {
          "name": "agent tool calls: 7",
          "ok": true
        },
        {
          "name": "agent passes used: 2",
          "ok": true
        },
        {
          "name": "agent commitEdits: true",
          "ok": true
        },
        {
          "name": "files changed: src/math.js",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_notes": 1,
          "workspace_inspect": 5,
          "workspace_agent": 1
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 52856,
        "contextOverflow": false,
        "promptTokens": 30831,
        "predictedTokens": 1728
      },
      "contextLength": 16384
    },
    {
      "scenario": "S9",
      "title": "full project from an empty directory",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "package.json parses with scripts.test running node --test",
          "ok": true
        },
        {
          "name": "at least one file under src/",
          "ok": true
        },
        {
          "name": "at least one test file under tests/",
          "ok": true
        },
        {
          "name": "node --test exits 0 with at least three ok lines",
          "ok": true
        },
        {
          "name": "score: 4/4",
          "ok": true
        },
        {
          "name": "files created: package.json, sample.txt, src/wordcount.js, tests/wordcount.test.js",
          "ok": true
        },
        {
          "name": "ok lines: 4",
          "ok": true
        },
        {
          "name": "node --test exit: 0",
          "ok": true
        },
        {
          "name": "package private: true",
          "ok": true
        },
        {
          "name": "applied transactions: 2",
          "ok": true
        },
        {
          "name": "model ran node --test: yes (1)",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 12,
        "toolCallsByName": {
          "workspace_notes": 2,
          "workspace_inspect": 1,
          "workspace_tasks": 4,
          "workspace_edit": 2,
          "workspace_command": 2,
          "workspace_vcs": 1
        },
        "rounds": 10,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 49340,
        "contextOverflow": true,
        "promptTokens": 101472,
        "predictedTokens": 4046
      },
      "contextLength": 16384
    }
  ]
}