{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3-1.7b",
  "driver": "in-process",
  "contextLength": 16384,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 6,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 7,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 6673,
        "contextOverflow": false,
        "promptTokens": 40806,
        "predictedTokens": 2074
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": false,
          "detail": "module.exports = { add: (a, b) => a + b + 1 }; |"
        },
        {
          "name": "node --test passes",
          "ok": false,
          "detail": "0) | at Test.runInAsyncScope (node:async_hooks:214:14) | at Test.run (node:internal/test_runner/test:1106:25) | at Test.start (node:internal/test_runner/test:1003:17) | at startSubtestAfterBootstrap (node:internal/test_runner/harness:358:17) { | generatedMessage: true, | code: 'ERR_ASSERTION', | actual: 5, | expected: 4, | operator: 'strictEqual', | diff: 'simple' | } |"
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: superseded/approved",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: false",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 1",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 26,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_plan": 6,
          "workspace_edit": 15,
          "workspace_command": 1
        },
        "rounds": 28,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 34594,
        "contextOverflow": false,
        "promptTokens": 312094,
        "predictedTokens": 9758
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 8,
        "toolCallsByName": {
          "workspace_command": 8
        },
        "rounds": 3,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 4402,
        "contextOverflow": false,
        "promptTokens": 16186,
        "predictedTokens": 1369
      },
      "contextLength": 16384
    },
    {
      "scenario": "S9",
      "title": "full project from an empty directory",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "package.json parses with scripts.test running node --test",
          "ok": true
        },
        {
          "name": "at least one file under src/",
          "ok": true
        },
        {
          "name": "at least one test file under tests/",
          "ok": true
        },
        {
          "name": "node --test exits 0 with at least three ok lines",
          "ok": false,
          "detail": "exit 1, 0 ok lines | not ok 3 - tests\\\\test-punctuation.js | --- | duration_ms: 30.3679 | type: 'test' | location: 'C:\\\\Users\\\\rootm\\\\AppData\\\\Local\\\\Temp\\\\live-s9-akXQtO\\\\tests\\\\test-punctuation.js:1:1' | failureType: 'testCodeFailure' | exitCode: 1 | signal: ~ | error: 'test failed' | code: 'ERR_TEST_FAILURE' | ... | 1..3 | # tests 3 | # suites 0 | # pass 0 | # fail 3 | # cancelled 0 | # skipped 0 | # todo 0 | # duration_ms 49.8157 |"
        },
        {
          "name": "score: 3/4",
          "ok": true
        },
        {
          "name": "files created: package.json, src/wordcount.js, tests/test-empty-input.js, tests/test-multiple-spaces.js, tests/test-punctuation.js",
          "ok": true
        },
        {
          "name": "ok lines: 0",
          "ok": true
        },
        {
          "name": "node --test exit: 1",
          "ok": true
        },
        {
          "name": "package private: true",
          "ok": true
        },
        {
          "name": "applied transactions: 5",
          "ok": true
        },
        {
          "name": "model ran node --test: no",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 28,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_edit": 10,
          "workspace_vcs": 14
        },
        "rounds": 22,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 20948,
        "contextOverflow": true,
        "promptTokens": 246653,
        "predictedTokens": 6126
      },
      "contextLength": 16384,
      "error": "context overflow: prompt 16461 > 16384"
    }
  ],
  "note": "Composite file: S3, S5 and S9 were re-run on fix commit `addeac1`; S1 is the original matrix row, recorded before it."
}

{
  "date": "2026-08-26",
  "modelId": "qwen/qwen3-1.7b",
  "driver": "in-process",
  "contextLength": 16384,
  "results": [
    {
      "scenario": "S1",
      "title": "inspect → read → edit → verify",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "bug fixed in src/math.js",
          "ok": true
        },
        {
          "name": "node --test passes",
          "ok": true
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "at least one applied transaction",
          "ok": true
        },
        {
          "name": "inspected before editing",
          "ok": true
        },
        {
          "name": "ran node --test through workspace_command",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 6,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_edit": 1,
          "workspace_command": 1
        },
        "rounds": 7,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 6673,
        "contextOverflow": false,
        "promptTokens": 40806,
        "predictedTokens": 2074
      },
      "contextLength": 16384
    },
    {
      "scenario": "S3",
      "title": "plan mode: propose, /accept, execute",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "no mutation before approval",
          "ok": true
        },
        {
          "name": "plan proposed in phase 1",
          "ok": true
        },
        {
          "name": "exactly one plan approved",
          "ok": true
        },
        {
          "name": "bug fixed in src/math.js",
          "ok": false,
          "detail": "module.exports = { add: (a, b) => a + b + 1 }; |"
        },
        {
          "name": "node --test passes",
          "ok": false,
          "detail": "0) | at Test.runInAsyncScope (node:async_hooks:214:14) | at Test.run (node:internal/test_runner/test:1106:25) | at Test.start (node:internal/test_runner/test:1003:17) | at startSubtestAfterBootstrap (node:internal/test_runner/harness:358:17) { | generatedMessage: true, | code: 'ERR_ASSERTION', | actual: 5, | expected: 4, | operator: 'strictEqual', | diff: 'simple' | } |"
        },
        {
          "name": "test file untouched",
          "ok": true
        },
        {
          "name": "plan proposals after /accept: 0",
          "ok": true
        },
        {
          "name": "plan final status: superseded/approved",
          "ok": true
        },
        {
          "name": "workspace_plan complete called: false",
          "ok": true
        },
        {
          "name": "APPROVAL_REQUIRED before approval: 1",
          "ok": true
        },
        {
          "name": "stray planned transactions: 0",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 26,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_plan": 6,
          "workspace_edit": 15,
          "workspace_command": 1
        },
        "rounds": 28,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 34594,
        "contextOverflow": false,
        "promptTokens": 312094,
        "predictedTokens": 9758
      },
      "contextLength": 16384
    },
    {
      "scenario": "S5",
      "title": "foreground command, background job, cancel",
      "driver": "in-process",
      "success": true,
      "checks": [
        {
          "name": "foreground run reported an exit code",
          "ok": true
        },
        {
          "name": "background job started",
          "ok": true
        },
        {
          "name": "job was canceled",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 8,
        "toolCallsByName": {
          "workspace_command": 8
        },
        "rounds": 3,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 4402,
        "contextOverflow": false,
        "promptTokens": 16186,
        "predictedTokens": 1369
      },
      "contextLength": 16384
    },
    {
      "scenario": "S9",
      "title": "full project from an empty directory",
      "driver": "in-process",
      "success": false,
      "checks": [
        {
          "name": "package.json parses with scripts.test running node --test",
          "ok": true
        },
        {
          "name": "at least one file under src/",
          "ok": true
        },
        {
          "name": "at least one test file under tests/",
          "ok": true
        },
        {
          "name": "node --test exits 0 with at least three ok lines",
          "ok": false,
          "detail": "exit 1, 0 ok lines | not ok 3 - tests\\\\test-punctuation.js | --- | duration_ms: 30.3679 | type: 'test' | location: 'C:\\\\Users\\\\rootm\\\\AppData\\\\Local\\\\Temp\\\\live-s9-akXQtO\\\\tests\\\\test-punctuation.js:1:1' | failureType: 'testCodeFailure' | exitCode: 1 | signal: ~ | error: 'test failed' | code: 'ERR_TEST_FAILURE' | ... | 1..3 | # tests 3 | # suites 0 | # pass 0 | # fail 3 | # cancelled 0 | # skipped 0 | # todo 0 | # duration_ms 49.8157 |"
        },
        {
          "name": "score: 3/4",
          "ok": true
        },
        {
          "name": "files created: package.json, src/wordcount.js, tests/test-empty-input.js, tests/test-multiple-spaces.js, tests/test-punctuation.js",
          "ok": true
        },
        {
          "name": "ok lines: 0",
          "ok": true
        },
        {
          "name": "node --test exit: 1",
          "ok": true
        },
        {
          "name": "package private: true",
          "ok": true
        },
        {
          "name": "applied transactions: 5",
          "ok": true
        },
        {
          "name": "model ran node --test: no",
          "ok": true
        }
      ],
      "metrics": {
        "toolCalls": 28,
        "toolCallsByName": {
          "workspace_inspect": 4,
          "workspace_edit": 10,
          "workspace_vcs": 14
        },
        "rounds": 22,
        "invalidToolRequests": 0,
        "envelopeViolations": 0,
        "wallMs": 20948,
        "contextOverflow": true,
        "promptTokens": 246653,
        "predictedTokens": 6126
      },
      "contextLength": 16384,
      "error": "context overflow: prompt 16461 > 16384"
    }
  ],
  "note": "Composite file: S3, S5 and S9 were re-run on fix commit `addeac1`; S1 is the original matrix row, recorded before it."
}