bench / results / 2026-08-24-gemma-3-12b-it-heretic.json
bench / results / 2026-08-24-gemma-3-12b-it-heretic.json
{
"date": "2026-08-24",
"modelId": "gemma-3-12b-it-heretic",
"variant": "full",
"settings": {
"limitTokens": 500,
"floorTokens": 400,
"keepRecentTokens": 300,
"chunkTokens": 800,
"summarizerMaxTokens": 400,
"consolidationBudgetTokens": 600,
"consolidationKeepNewest": 2,
"mergeMaxTokens": 400,
"maxExcerptChars": 32000,
"quizMaxTokens": 120
},
"fixtures": [
{
"fixture": "coding-agent",
"asked": 5,
"correctAfterCompaction": 5,
"reductionPercent": 43,
"latency": {
"mean": 3224.3,
"p50": 2885,
"max": 4441
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "research",
"asked": 3,
"correctAfterCompaction": 3,
"reductionPercent": 40,
"latency": {
"mean": 3403,
"p50": 3403,
"max": 3403
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "contradictory-updates",
"asked": 3,
"correctAfterCompaction": 3,
"reductionPercent": 31,
"latency": {
"mean": 1497,
"p50": 1497,
"max": 1497
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "tool-spam",
"asked": 3,
"correctAfterCompaction": 0,
"correctAfterConsolidation": 0,
"reductionPercent": 63,
"latency": {
"mean": 1829.5,
"p50": 1698,
"max": 3363
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
}
]
}{
"date": "2026-08-24",
"modelId": "gemma-3-12b-it-heretic",
"variant": "full",
"settings": {
"limitTokens": 500,
"floorTokens": 400,
"keepRecentTokens": 300,
"chunkTokens": 800,
"summarizerMaxTokens": 400,
"consolidationBudgetTokens": 600,
"consolidationKeepNewest": 2,
"mergeMaxTokens": 400,
"maxExcerptChars": 32000,
"quizMaxTokens": 120
},
"fixtures": [
{
"fixture": "coding-agent",
"asked": 5,
"correctAfterCompaction": 5,
"reductionPercent": 43,
"latency": {
"mean": 3224.3,
"p50": 2885,
"max": 4441
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "research",
"asked": 3,
"correctAfterCompaction": 3,
"reductionPercent": 40,
"latency": {
"mean": 3403,
"p50": 3403,
"max": 3403
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "contradictory-updates",
"asked": 3,
"correctAfterCompaction": 3,
"reductionPercent": 31,
"latency": {
"mean": 1497,
"p50": 1497,
"max": 1497
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
},
{
"fixture": "tool-spam",
"asked": 3,
"correctAfterCompaction": 0,
"correctAfterConsolidation": 0,
"reductionPercent": 63,
"latency": {
"mean": 1829.5,
"p50": 1698,
"max": 3363
},
"structural": {
"singleLeadingSystem": true,
"noOrphanToolResults": true,
"hasUserMessage": true,
"ok": true
},
"cacheDeterministic": true
}
]
}