src / digest.ts
src / digest.ts
// What the model sees of the ledger: a short header on most turns, the full state when the
// last full view has scrolled out of the part of the context kept word for word, and a
// one-line form in chats with attached documents (whose retrieval reads the user message).
import { currentStep, Ledger, LedgerEvent, listProjects, loadLedger, progress, stepLabel } from "./ledger";
import { clip, estimateTokens } from "./text";
export const HEADER_OPEN = "[throughLine: background state of this chat's project, for the assistant. Use it; do not mention it unless asked.]";
export const FULL_OPEN = "[throughLine full state: background state of this chat's project, for the assistant. Use it; do not mention it unless asked.]";
export const BLOCK_CLOSE = "[end throughLine]";
export const SCOPE_LINE = "Stay within the goal: add steps beyond it only if the user asks, and when every step is done, give the user the result and stop.";
function nowLine(l: Ledger): string {
const p = progress(l);
const cur = currentStep(l);
if (!p.total) return "Now: no plan yet.";
const tally = `${p.done} of ${p.total} done${p.blocked ? `, ${p.blocked} blocked` : ""}`;
if (cur) return `Now: ${clip(stepLabel(cur), 160)} | ${tally}`;
if (p.done === p.total) return `Now: all steps done | ${tally}`;
return `Now: no step in progress | ${tally}`;
}
// A short account of ledger changes since the chat last saw the ledger. Added steps are grouped,
// and a step that changed several times is reported once, with its latest status.
export function describeChanges(events: LedgerEvent[]): string {
const parts: string[] = [];
const added: number[] = [];
const stepStatus = new Map<string, string>();
const flushAdded = () => {
if (!added.length) return;
parts.push(added.length === 1 ? `step ${added[0]} added` : `steps ${added[0]} to ${added.at(-1)} added`);
added.length = 0;
};
for (const e of events) {
if (e.op === "step_added") { added.push(Number(e.id)); continue; }
flushAdded();
switch (e.op) {
case "finding_added": parts.push(`${e.id} added`); break;
case "decision_added": parts.push(`${e.id} added`); break;
case "question_added": parts.push(`${e.id} opened`); break;
case "question_resolved": parts.push(`${e.id} resolved`); break;
case "step_updated": {
const status = /now (\w+)/.exec(e.text ?? "")?.[1];
stepStatus.set(e.id ?? "", status ? `now ${status}` : "updated");
break;
}
case "goal_set": parts.push("goal changed by the user"); break;
case "constraint_added": parts.push("constraint added by the user"); break;
case "hand_edit": parts.push(`the user edited the ledger by hand (${e.text ?? "changes"})`); break;
default: break;
}
}
flushAdded();
for (const [id, what] of stepStatus) parts.push(`step ${id} ${what}`);
const unique = [...new Set(parts)];
return unique.length > 8 ? `${unique.slice(-8).join("; ")}; and ${unique.length - 8} earlier changes` : unique.join("; ");
}
export function renderHeader(l: Ledger, changes: string, extras: string[] = []): string {
const lines = [HEADER_OPEN, `Project: ${l.name} | Goal: ${clip(l.goal || "not set; ask the user", 200)}`, nowLine(l)];
if (changes) lines.push(`Changed since the last header: ${changes}`);
const p = progress(l);
if (p.total && p.done === p.total) lines.push(SCOPE_LINE);
lines.push(...extras, BLOCK_CLOSE);
return lines.join("\n");
}
export function renderCompact(l: Ledger): string {
const cur = currentStep(l);
const p = progress(l);
const where = cur ? `step ${cur.id} of ${p.total}: ${clip(cur.text, 80)}` : p.total ? `${p.done} of ${p.total} steps done` : "no plan yet";
return `[throughLine: project ${l.name}, ${where}. Background state for the assistant; ledger_view has the rest.]`;
}
// The full state, cut to a token cap. Goal, constraints and the plan's current position always
// appear; findings and decisions are filled newest first until the cap is reached.
export function renderFull(l: Ledger, capTokens: number, extras: string[] = []): string {
const goalLabel = l.goalSource === "user_message" ? "Goal (taken from the user's message; the user can change it with /tl goal)" : "Goal";
const out: string[] = [FULL_OPEN, `Project: ${l.name}`, `${goalLabel}: ${clip(l.goal || "not set; ask the user", 600)}`];
out.push(l.constraints.length ? `Constraints: ${l.constraints.map((c) => clip(c, 200)).join(" | ")}` : "Constraints: none");
if (l.plan.length) {
const p = progress(l);
out.push(`Plan (${p.done} of ${p.total} done):`);
const curIdx = l.plan.findIndex((s) => s.status === "doing");
const firstOpen = l.plan.findIndex((s) => s.status !== "done");
const anchor = curIdx >= 0 ? curIdx : firstOpen >= 0 ? firstOpen : l.plan.length - 1;
const hi = Math.min(l.plan.length - 1, anchor + 4);
const lo = Math.max(0, Math.min(anchor - 1, hi - 4));
const shown = l.plan.filter((s, i) => (i >= lo && i <= hi) || s.status === "blocked");
const before = l.plan.slice(0, lo).filter((s) => !shown.includes(s));
const after = l.plan.slice(hi + 1).filter((s) => !shown.includes(s));
if (before.length) out.push(` (${before.length} earlier step${before.length === 1 ? "" : "s"}, ${before.filter((s) => s.status === "done").length} done)`);
for (const s of shown) out.push(` ${clip(stepLabel(s), 160)}`);
if (after.length) out.push(` (${after.length} later step${after.length === 1 ? "" : "s"})`);
} else {
out.push("Plan: no steps yet.");
}
const open = l.questions.filter((q) => q.status === "open");
const tail: string[] = [];
if (open.length) tail.push(`Open questions: ${open.map((q) => `${q.id} ${clip(q.text, 140)}`).join(" | ")}`);
tail.push(SCOPE_LINE, ...extras, BLOCK_CLOSE);
// Decisions first (few and lasting), then findings newest first, until the cap is reached.
// Each section's heading and the closing note are reserved before entries are added.
const reserve = 40;
let used = estimateTokens(out.join("\n") + "\n" + tail.join("\n")) + reserve;
let omitted = 0;
const take = <T>(items: T[], line: (x: T) => string): string[] => {
const lines: string[] = [];
for (const x of [...items].reverse()) {
const t = line(x);
const cost = estimateTokens(t) + 1;
if (used + cost > capTokens) { omitted++; continue; }
lines.unshift(t);
used += cost;
}
return lines;
};
const dLines = take(l.decisions, (d) => ` ${d.id} ${clip(d.text, 180)}${d.reason ? ` (because ${clip(d.reason, 100)})` : ""}`);
const fLines = take(l.findings, (f) => ` ${f.id} ${clip(f.text, 220)} [${clip(f.source, 80)}${f.verification === "not_read" ? ", not read" : ""}]`);
const extra: string[] = [];
if (dLines.length) extra.push(`Decisions${dLines.length < l.decisions.length ? ` (latest ${dLines.length} of ${l.decisions.length})` : ""}:`, ...dLines);
if (fLines.length) extra.push(`Findings${fLines.length < l.findings.length ? ` (latest ${fLines.length} of ${l.findings.length})` : ""}:`, ...fLines);
if (omitted) extra.push(`${omitted} older entries not shown; ledger_view returns everything.`);
return [...out, ...extra, ...tail].join("\n");
}
// A list of all projects, attached to a message that asks about them.
export function renderProjectList(currentProject?: string): string {
const names = listProjects();
const lines = ["[throughLine: the user's projects, for answering this message.]"];
if (!names.length) lines.push("There are no throughLine projects yet.");
for (const n of names) {
const l = loadLedger(n);
if (!l) continue;
const p = progress(l);
const cur = currentStep(l);
const mark = n === currentProject ? " (this chat)" : "";
const where = cur ? `now on step ${cur.id}: ${clip(cur.text, 60)}` : !p.total ? "no plan yet" : p.done === p.total ? "all steps done" : `${p.done} of ${p.total} steps done`;
const nf = l.findings.length;
lines.push(`${n}${mark}: ${clip(l.goal || "no goal", 90)} | ${where} | ${nf} finding${nf === 1 ? "" : "s"} | updated ${l.updated.slice(0, 10)}`);
}
lines.push(BLOCK_CLOSE);
return lines.join("\n");
}
// A message that asks about the user's projects, in the user's own words.
export function asksAboutProjects(text: string): boolean {
const t = text.trim();
if (!t || t.length > 400 || t.startsWith("/")) return false;
return /\b(projects?|throughline|ledgers?)\b/i.test(t) && /\b(status|progress|state|where|update|overview|list|which|what|how)\b/i.test(t);
}
// What the model sees of the ledger: a short header on most turns, the full state when the
// last full view has scrolled out of the part of the context kept word for word, and a
// one-line form in chats with attached documents (whose retrieval reads the user message).
import { currentStep, Ledger, LedgerEvent, listProjects, loadLedger, progress, stepLabel } from "./ledger";
import { clip, estimateTokens } from "./text";
export const HEADER_OPEN = "[throughLine: background state of this chat's project, for the assistant. Use it; do not mention it unless asked.]";
export const FULL_OPEN = "[throughLine full state: background state of this chat's project, for the assistant. Use it; do not mention it unless asked.]";
export const BLOCK_CLOSE = "[end throughLine]";
export const SCOPE_LINE = "Stay within the goal: add steps beyond it only if the user asks, and when every step is done, give the user the result and stop.";
function nowLine(l: Ledger): string {
const p = progress(l);
const cur = currentStep(l);
if (!p.total) return "Now: no plan yet.";
const tally = `${p.done} of ${p.total} done${p.blocked ? `, ${p.blocked} blocked` : ""}`;
if (cur) return `Now: ${clip(stepLabel(cur), 160)} | ${tally}`;
if (p.done === p.total) return `Now: all steps done | ${tally}`;
return `Now: no step in progress | ${tally}`;
}
// A short account of ledger changes since the chat last saw the ledger. Added steps are grouped,
// and a step that changed several times is reported once, with its latest status.
export function describeChanges(events: LedgerEvent[]): string {
const parts: string[] = [];
const added: number[] = [];
const stepStatus = new Map<string, string>();
const flushAdded = () => {
if (!added.length) return;
parts.push(added.length === 1 ? `step ${added[0]} added` : `steps ${added[0]} to ${added.at(-1)} added`);
added.length = 0;
};
for (const e of events) {
if (e.op === "step_added") { added.push(Number(e.id)); continue; }
flushAdded();
switch (e.op) {
case "finding_added": parts.push(`${e.id} added`); break;
case "decision_added": parts.push(`${e.id} added`); break;
case "question_added": parts.push(`${e.id} opened`); break;
case "question_resolved": parts.push(`${e.id} resolved`); break;
case "step_updated": {
const status = /now (\w+)/.exec(e.text ?? "")?.[1];
stepStatus.set(e.id ?? "", status ? `now ${status}` : "updated");
break;
}
case "goal_set": parts.push("goal changed by the user"); break;
case "constraint_added": parts.push("constraint added by the user"); break;
case "hand_edit": parts.push(`the user edited the ledger by hand (${e.text ?? "changes"})`); break;
default: break;
}
}
flushAdded();
for (const [id, what] of stepStatus) parts.push(`step ${id} ${what}`);
const unique = [...new Set(parts)];
return unique.length > 8 ? `${unique.slice(-8).join("; ")}; and ${unique.length - 8} earlier changes` : unique.join("; ");
}
export function renderHeader(l: Ledger, changes: string, extras: string[] = []): string {
const lines = [HEADER_OPEN, `Project: ${l.name} | Goal: ${clip(l.goal || "not set; ask the user", 200)}`, nowLine(l)];
if (changes) lines.push(`Changed since the last header: ${changes}`);
const p = progress(l);
if (p.total && p.done === p.total) lines.push(SCOPE_LINE);
lines.push(...extras, BLOCK_CLOSE);
return lines.join("\n");
}
export function renderCompact(l: Ledger): string {
const cur = currentStep(l);
const p = progress(l);
const where = cur ? `step ${cur.id} of ${p.total}: ${clip(cur.text, 80)}` : p.total ? `${p.done} of ${p.total} steps done` : "no plan yet";
return `[throughLine: project ${l.name}, ${where}. Background state for the assistant; ledger_view has the rest.]`;
}
// The full state, cut to a token cap. Goal, constraints and the plan's current position always
// appear; findings and decisions are filled newest first until the cap is reached.
export function renderFull(l: Ledger, capTokens: number, extras: string[] = []): string {
const goalLabel = l.goalSource === "user_message" ? "Goal (taken from the user's message; the user can change it with /tl goal)" : "Goal";
const out: string[] = [FULL_OPEN, `Project: ${l.name}`, `${goalLabel}: ${clip(l.goal || "not set; ask the user", 600)}`];
out.push(l.constraints.length ? `Constraints: ${l.constraints.map((c) => clip(c, 200)).join(" | ")}` : "Constraints: none");
if (l.plan.length) {
const p = progress(l);
out.push(`Plan (${p.done} of ${p.total} done):`);
const curIdx = l.plan.findIndex((s) => s.status === "doing");
const firstOpen = l.plan.findIndex((s) => s.status !== "done");
const anchor = curIdx >= 0 ? curIdx : firstOpen >= 0 ? firstOpen : l.plan.length - 1;
const hi = Math.min(l.plan.length - 1, anchor + 4);
const lo = Math.max(0, Math.min(anchor - 1, hi - 4));
const shown = l.plan.filter((s, i) => (i >= lo && i <= hi) || s.status === "blocked");
const before = l.plan.slice(0, lo).filter((s) => !shown.includes(s));
const after = l.plan.slice(hi + 1).filter((s) => !shown.includes(s));
if (before.length) out.push(` (${before.length} earlier step${before.length === 1 ? "" : "s"}, ${before.filter((s) => s.status === "done").length} done)`);
for (const s of shown) out.push(` ${clip(stepLabel(s), 160)}`);
if (after.length) out.push(` (${after.length} later step${after.length === 1 ? "" : "s"})`);
} else {
out.push("Plan: no steps yet.");
}
const open = l.questions.filter((q) => q.status === "open");
const tail: string[] = [];
if (open.length) tail.push(`Open questions: ${open.map((q) => `${q.id} ${clip(q.text, 140)}`).join(" | ")}`);
tail.push(SCOPE_LINE, ...extras, BLOCK_CLOSE);
// Decisions first (few and lasting), then findings newest first, until the cap is reached.
// Each section's heading and the closing note are reserved before entries are added.
const reserve = 40;
let used = estimateTokens(out.join("\n") + "\n" + tail.join("\n")) + reserve;
let omitted = 0;
const take = <T>(items: T[], line: (x: T) => string): string[] => {
const lines: string[] = [];
for (const x of [...items].reverse()) {
const t = line(x);
const cost = estimateTokens(t) + 1;
if (used + cost > capTokens) { omitted++; continue; }
lines.unshift(t);
used += cost;
}
return lines;
};
const dLines = take(l.decisions, (d) => ` ${d.id} ${clip(d.text, 180)}${d.reason ? ` (because ${clip(d.reason, 100)})` : ""}`);
const fLines = take(l.findings, (f) => ` ${f.id} ${clip(f.text, 220)} [${clip(f.source, 80)}${f.verification === "not_read" ? ", not read" : ""}]`);
const extra: string[] = [];
if (dLines.length) extra.push(`Decisions${dLines.length < l.decisions.length ? ` (latest ${dLines.length} of ${l.decisions.length})` : ""}:`, ...dLines);
if (fLines.length) extra.push(`Findings${fLines.length < l.findings.length ? ` (latest ${fLines.length} of ${l.findings.length})` : ""}:`, ...fLines);
if (omitted) extra.push(`${omitted} older entries not shown; ledger_view returns everything.`);
return [...out, ...extra, ...tail].join("\n");
}
// A list of all projects, attached to a message that asks about them.
export function renderProjectList(currentProject?: string): string {
const names = listProjects();
const lines = ["[throughLine: the user's projects, for answering this message.]"];
if (!names.length) lines.push("There are no throughLine projects yet.");
for (const n of names) {
const l = loadLedger(n);
if (!l) continue;
const p = progress(l);
const cur = currentStep(l);
const mark = n === currentProject ? " (this chat)" : "";
const where = cur ? `now on step ${cur.id}: ${clip(cur.text, 60)}` : !p.total ? "no plan yet" : p.done === p.total ? "all steps done" : `${p.done} of ${p.total} steps done`;
const nf = l.findings.length;
lines.push(`${n}${mark}: ${clip(l.goal || "no goal", 90)} | ${where} | ${nf} finding${nf === 1 ? "" : "s"} | updated ${l.updated.slice(0, 10)}`);
}
lines.push(BLOCK_CLOSE);
return lines.join("\n");
}
// A message that asks about the user's projects, in the user's own words.
export function asksAboutProjects(text: string): boolean {
const t = text.trim();
if (!t || t.length > 400 || t.startsWith("/")) return false;
return /\b(projects?|throughline|ledgers?)\b/i.test(t) && /\b(status|progress|state|where|update|overview|list|which|what|how)\b/i.test(t);
}