{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3.8-27b",
  "driver": "in-process",
  "contextLength": 16384,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 8,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_command": 2,
          "workspace_edit": 1
        },
        "rounds": 6,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 18230,
        "contextOverflow": false,
        "promptTokens": 36997,
        "predictedTokens": 1377
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: completed",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: true",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 9,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_plan": 2,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 8,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 23072,
        "contextOverflow": false,
        "promptTokens": 61491,
        "predictedTokens": 1810
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_inspect": 2,
          "workspace_notes": 1,
          "workspace_command": 4
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 22578,
        "contextOverflow": false,
        "promptTokens": 31204,
        "predictedTokens": 1596
      },
      "contextLength": 16384
    }
  ]
}

{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3.8-27b",
  "driver": "in-process",
  "contextLength": 16384,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 8,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_command": 2,
          "workspace_edit": 1
        },
        "rounds": 6,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 18230,
        "contextOverflow": false,
        "promptTokens": 36997,
        "predictedTokens": 1377
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: completed",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: true",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 0",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 9,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_notes": 1,
          "workspace_plan": 2,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 8,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 23072,
        "contextOverflow": false,
        "promptTokens": 61491,
        "predictedTokens": 1810
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 7,
        "toolCallsByName": {
          "workspace_inspect": 2,
          "workspace_notes": 1,
          "workspace_command": 4
        },
        "rounds": 5,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 22578,
        "contextOverflow": false,
        "promptTokens": 31204,
        "predictedTokens": 1596
      },
      "contextLength": 16384
    }
  ]
}