dist / memory / preprocessor.js
dist / memory / preprocessor.js
"use strict";
/**
* @file preprocessor.ts
* Prompt preprocessor — silently injects relevant memories into the
* system context before each user message reaches the model.
*
* This is what makes the memory system "feel" persistent:
* the user never has to say "recall" — the model just knows.
*
* Flow:
* 1. User types a message
* 2. Preprocessor fires, extracts key terms from the message
* 3. Retrieves top-N memories by composite score
* 4. Formats them and prepends to the system prompt
* 5. Model sees the memories as part of its context
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.promptPreprocessor = promptPreprocessor;
const config_1 = require("./config");
const toolsProvider_1 = require("./toolsProvider");
const ai_1 = require("./processing/ai");
const constants_1 = require("./constants");
const projectMemory_1 = require("./projectMemory");
const sessionCompact_1 = require("./sessionCompact");
const pendingQueue_1 = require("./processing/pendingQueue");
const stateManager_1 = require("../tools/stateManager");
const contextGauge_1 = require("../tools/contextGauge");
function readConfig(ctl) {
const c = ctl.getPluginConfig(config_1.configSchematics);
return {
autoInject: c.get("autoInjectMemories") === "on",
contextCount: c.get("contextMemoryCount") || 5,
decayDays: 30,
storagePath: "",
enableAI: c.get("enableAIExtraction") === "on",
activeProject: c.get("activeProject") || "",
injectionCategories: [], // all categories
};
}
/** Format a single memory into a one-line string. */
function formatMemoryLine(mem) {
const scopeTag = mem.scope !== "global"
? ` [${mem.scope}${mem.project ? `:${mem.project}` : ""}]`
: "";
return `${mem.content} [${mem.category}${mem.tags.length > 0 ? `, tags: ${mem.tags.join(", ")}` : ""}${scopeTag}]`;
}
/**
* Build tiered memory context:
* - L0: Identity memories (always loaded, ~400 chars)
* - L1: Essential memories (always loaded, ~1600 chars)
* - L2: Query-relevant memories (on-demand)
*/
function buildTieredContext(identityMems, essentialMems, queryMems, charFactor = 1) {
const sections = [];
const usedIds = new Set();
// L0: Identity layer
if (identityMems.length > 0) {
const l0Lines = [];
let l0Chars = 0;
for (const mem of identityMems) {
const line = mem.content;
if (l0Chars + line.length > constants_1.L0_MAX_CHARS)
break;
l0Lines.push(line);
l0Chars += line.length;
usedIds.add(mem.id);
}
if (l0Lines.length > 0) {
sections.push(`[User Identity]${constants_1.MEMORY_SEPARATOR}${l0Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
// L1: Essential layer
if (essentialMems.length > 0) {
const l1Lines = [];
let l1Chars = 0;
for (const mem of essentialMems) {
if (usedIds.has(mem.id))
continue;
const line = formatMemoryLine(mem);
if (l1Chars + line.length > Math.floor(constants_1.L1_MAX_CHARS * charFactor))
break;
l1Lines.push(line);
l1Chars += line.length;
usedIds.add(mem.id);
}
if (l1Lines.length > 0) {
sections.push(`[Essential Knowledge]${constants_1.MEMORY_SEPARATOR}${l1Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
// L2: Query-relevant layer (dedup against L0/L1)
const remainingBudget = Math.floor(constants_1.MAX_INJECTED_CONTEXT_CHARS * charFactor) - sections.join("\n").length;
if (queryMems.length > 0 && remainingBudget > 100) {
const l2Lines = [];
let l2Chars = 0;
for (const mem of queryMems) {
if (usedIds.has(mem.id))
continue;
const line = formatMemoryLine(mem);
if (l2Chars + line.length > remainingBudget)
break;
l2Lines.push(line);
l2Chars += line.length;
usedIds.add(mem.id);
}
if (l2Lines.length > 0) {
sections.push(`[Relevant Context]${constants_1.MEMORY_SEPARATOR}${l2Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
if (sections.length === 0)
return "";
const totalCount = usedIds.size;
return [
`[Persistent Memory — ${totalCount} memories loaded]`,
...sections,
``,
`Use this knowledge naturally. Do not mention the memory system unless asked.`,
].join("\n");
}
/**
* C1: extraction + storage of facts for one text. Originally the inline
* .then-body of the preprocessor; now shared by the live path and the
* crash-recovery drain. Per-fact errors are tolerated (as before); a fatal
* throw marks the queue entry for retry.
*/
async function processExtraction(text, existingSummary, cfg, engine, db, model) {
const facts = await (0, ai_1.extractFacts)(text, existingSummary, model);
for (const fact of facts) {
try {
// Dedup check — same logic as the Remember tool
const similar = await engine.retrieve(fact.content, 3, cfg.decayDays);
if (similar && similar.memories) {
const normalize = (s) => s.toLowerCase().replace(/[^a-záà âãéèêÃïóôõúüç\s]/g, "").replace(/\s+/g, " ").trim();
const n1 = normalize(fact.content);
let isDup = false;
for (const mem of similar.memories) {
if (mem.relevanceScore > 0.7 && mem.content) {
const n2 = normalize(mem.content);
if (n1.includes(n2) || n2.includes(n1)) {
isDup = true;
break;
}
const words1 = new Set(n1.split(" "));
const words2 = new Set(n2.split(" "));
const inter = [...words1].filter(w => words2.has(w)).length;
const union = new Set([...words1, ...words2]).size;
if (union > 0 && inter / union > 0.75) {
isDup = true;
break;
}
}
}
if (isDup)
continue; // skip duplicate
}
const target = (0, projectMemory_1.resolveProjectMemoryTarget)(cfg, {
forceActiveProject: true,
});
const normalizedTags = target.scope === "project" && target.project
? (0, projectMemory_1.buildProjectMemoryTags)(fact.tags, target.project)
: fact.tags;
const id = db.store(fact.content, fact.category, normalizedTags, fact.confidence, "ai-extracted", null, target.scope, target.project);
engine.indexMemory(id, fact.content, normalizedTags, fact.category);
}
catch {
}
}
}
/** Ids being processed right now — the drain must not double-process them. */
const liveIds = new Set();
let drainRunning = false;
/** C1: recover pending extractions left over from a crash/restart. */
function drainPending(queue, cfg, engine, db, model) {
if (drainRunning)
return;
drainRunning = true;
(async () => {
const entries = await queue.pending(constants_1.PENDING_QUEUE_DRAIN_LIMIT, liveIds);
for (const e of entries) {
liveIds.add(e.id);
try {
await processExtraction(e.text, e.existingSummary, cfg, engine, db, model);
await queue.complete(e.id);
}
catch {
await queue.fail(e.id);
}
finally {
liveIds.delete(e.id);
}
}
})()
.catch(() => { })
.finally(() => { drainRunning = false; });
}
async function promptPreprocessor(ctl, userMessage) {
try { (0, ai_1.setPluginClient)(ctl.client); } catch { }
const cfg = readConfig(ctl);
if (!cfg.autoInject)
return userMessage;
if (userMessage.length < 10)
return userMessage;
try {
const { engine, db } = await (0, toolsProvider_1.getSharedInstances)(cfg.storagePath);
let injectScale = { countFactor: 1, charFactor: 1 };
try {
const pst = await stateManager_1.getPersistedState();
injectScale = (0, contextGauge_1.scaleMemoryInjection)(pst?.contextGauge?.lastPct ?? 0);
}
catch { }
// F2: session compaction hook — digest pending request into memory
let compactLine = "";
try {
const st = await (0, sessionCompact_1.runCompactHook)(ctl, engine, db, cfg);
if (st)
compactLine = st;
}
catch { }
const compactTail = compactLine ? `\n\n${compactLine}` : "";
let chatModel = null;
try {
const ts = typeof ctl.tokenSource === "function" ? await ctl.tokenSource() : null;
if (ts && typeof ts.respond === "function")
chatModel = ts;
}
catch { }
const extractionModel = chatModel ?? undefined;
// L0: Identity memories (always loaded)
const identityMems = db.getIdentityMemories();
// L1: Essential memories (always loaded — most important/accessed)
const essentialMems = db.getEssentialMemories(constants_1.L1_ESSENTIAL_COUNT);
// L2: Query-relevant memories (on-demand based on user message)
const result = await engine.retrieve(userMessage, cfg.contextCount, cfg.decayDays, false);
let queryMemories = [...result.memories];
if (cfg.activeProject) {
const projectMems = db.getByProject(cfg.activeProject, cfg.contextCount);
const existingIds = new Set(result.memories.map((m) => m.id));
for (const pm of projectMems) {
if (!existingIds.has(pm.id))
queryMemories.push(pm);
}
}
// Apply category filter if configured
if (cfg.injectionCategories.length > 0) {
queryMemories = queryMemories.filter((m) => cfg.injectionCategories.includes(m.category.toLowerCase()));
}
queryMemories = queryMemories.slice(0, Math.max(2, Math.round(cfg.contextCount * injectScale.countFactor)));
if (cfg.enableAI && userMessage.length >= 100) {
const existingSummary = result.memories.map((m) => m.content).join("; ");
// C1: register BEFORE extracting — a restart mid-extraction
// leaves the entry on disk; drainPending() recovers it next run.
const queue = (0, pendingQueue_1.getPendingQueue)();
const entryId = queue.enqueue({ text: userMessage, existingSummary });
liveIds.add(entryId);
processExtraction(userMessage, existingSummary, cfg, engine, db, extractionModel)
.then(() => queue.complete(entryId))
.catch(() => queue.fail(entryId))
.finally(() => liveIds.delete(entryId));
drainPending(queue, cfg, engine, db, extractionModel);
}
// Build tiered context (L0 + L1 + L2)
const hasAnyMemories = identityMems.length > 0 || essentialMems.length > 0 || queryMemories.length > 0;
// Identity onboarding: only when NO memories exist at all (true first conversation)
if (!hasAnyMemories) {
return `${userMessage}\n\n---\n\n[Memory System: No memories found. If the user hasn't introduced themselves yet, naturally ask their name, role, and preferred language early in the conversation, then store with Remember(content, "identity"). This makes future conversations personalized.]${compactTail}`;
}
const context = buildTieredContext(identityMems, essentialMems, queryMemories, injectScale.charFactor);
if (!context)
return `${userMessage}${compactTail}`;
return `${userMessage}\n\n---\n\n${context}${compactTail}`;
}
catch {
return userMessage;
}
}
//# sourceMappingURL=preprocessor.js.map"use strict";
/**
* @file preprocessor.ts
* Prompt preprocessor — silently injects relevant memories into the
* system context before each user message reaches the model.
*
* This is what makes the memory system "feel" persistent:
* the user never has to say "recall" — the model just knows.
*
* Flow:
* 1. User types a message
* 2. Preprocessor fires, extracts key terms from the message
* 3. Retrieves top-N memories by composite score
* 4. Formats them and prepends to the system prompt
* 5. Model sees the memories as part of its context
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.promptPreprocessor = promptPreprocessor;
const config_1 = require("./config");
const toolsProvider_1 = require("./toolsProvider");
const ai_1 = require("./processing/ai");
const constants_1 = require("./constants");
const projectMemory_1 = require("./projectMemory");
const sessionCompact_1 = require("./sessionCompact");
const pendingQueue_1 = require("./processing/pendingQueue");
const stateManager_1 = require("../tools/stateManager");
const contextGauge_1 = require("../tools/contextGauge");
function readConfig(ctl) {
const c = ctl.getPluginConfig(config_1.configSchematics);
return {
autoInject: c.get("autoInjectMemories") === "on",
contextCount: c.get("contextMemoryCount") || 5,
decayDays: 30,
storagePath: "",
enableAI: c.get("enableAIExtraction") === "on",
activeProject: c.get("activeProject") || "",
injectionCategories: [], // all categories
};
}
/** Format a single memory into a one-line string. */
function formatMemoryLine(mem) {
const scopeTag = mem.scope !== "global"
? ` [${mem.scope}${mem.project ? `:${mem.project}` : ""}]`
: "";
return `${mem.content} [${mem.category}${mem.tags.length > 0 ? `, tags: ${mem.tags.join(", ")}` : ""}${scopeTag}]`;
}
/**
* Build tiered memory context:
* - L0: Identity memories (always loaded, ~400 chars)
* - L1: Essential memories (always loaded, ~1600 chars)
* - L2: Query-relevant memories (on-demand)
*/
function buildTieredContext(identityMems, essentialMems, queryMems, charFactor = 1) {
const sections = [];
const usedIds = new Set();
// L0: Identity layer
if (identityMems.length > 0) {
const l0Lines = [];
let l0Chars = 0;
for (const mem of identityMems) {
const line = mem.content;
if (l0Chars + line.length > constants_1.L0_MAX_CHARS)
break;
l0Lines.push(line);
l0Chars += line.length;
usedIds.add(mem.id);
}
if (l0Lines.length > 0) {
sections.push(`[User Identity]${constants_1.MEMORY_SEPARATOR}${l0Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
// L1: Essential layer
if (essentialMems.length > 0) {
const l1Lines = [];
let l1Chars = 0;
for (const mem of essentialMems) {
if (usedIds.has(mem.id))
continue;
const line = formatMemoryLine(mem);
if (l1Chars + line.length > Math.floor(constants_1.L1_MAX_CHARS * charFactor))
break;
l1Lines.push(line);
l1Chars += line.length;
usedIds.add(mem.id);
}
if (l1Lines.length > 0) {
sections.push(`[Essential Knowledge]${constants_1.MEMORY_SEPARATOR}${l1Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
// L2: Query-relevant layer (dedup against L0/L1)
const remainingBudget = Math.floor(constants_1.MAX_INJECTED_CONTEXT_CHARS * charFactor) - sections.join("\n").length;
if (queryMems.length > 0 && remainingBudget > 100) {
const l2Lines = [];
let l2Chars = 0;
for (const mem of queryMems) {
if (usedIds.has(mem.id))
continue;
const line = formatMemoryLine(mem);
if (l2Chars + line.length > remainingBudget)
break;
l2Lines.push(line);
l2Chars += line.length;
usedIds.add(mem.id);
}
if (l2Lines.length > 0) {
sections.push(`[Relevant Context]${constants_1.MEMORY_SEPARATOR}${l2Lines.join(constants_1.MEMORY_SEPARATOR)}`);
}
}
if (sections.length === 0)
return "";
const totalCount = usedIds.size;
return [
`[Persistent Memory — ${totalCount} memories loaded]`,
...sections,
``,
`Use this knowledge naturally. Do not mention the memory system unless asked.`,
].join("\n");
}
/**
* C1: extraction + storage of facts for one text. Originally the inline
* .then-body of the preprocessor; now shared by the live path and the
* crash-recovery drain. Per-fact errors are tolerated (as before); a fatal
* throw marks the queue entry for retry.
*/
async function processExtraction(text, existingSummary, cfg, engine, db, model) {
const facts = await (0, ai_1.extractFacts)(text, existingSummary, model);
for (const fact of facts) {
try {
// Dedup check — same logic as the Remember tool
const similar = await engine.retrieve(fact.content, 3, cfg.decayDays);
if (similar && similar.memories) {
const normalize = (s) => s.toLowerCase().replace(/[^a-záà âãéèêÃïóôõúüç\s]/g, "").replace(/\s+/g, " ").trim();
const n1 = normalize(fact.content);
let isDup = false;
for (const mem of similar.memories) {
if (mem.relevanceScore > 0.7 && mem.content) {
const n2 = normalize(mem.content);
if (n1.includes(n2) || n2.includes(n1)) {
isDup = true;
break;
}
const words1 = new Set(n1.split(" "));
const words2 = new Set(n2.split(" "));
const inter = [...words1].filter(w => words2.has(w)).length;
const union = new Set([...words1, ...words2]).size;
if (union > 0 && inter / union > 0.75) {
isDup = true;
break;
}
}
}
if (isDup)
continue; // skip duplicate
}
const target = (0, projectMemory_1.resolveProjectMemoryTarget)(cfg, {
forceActiveProject: true,
});
const normalizedTags = target.scope === "project" && target.project
? (0, projectMemory_1.buildProjectMemoryTags)(fact.tags, target.project)
: fact.tags;
const id = db.store(fact.content, fact.category, normalizedTags, fact.confidence, "ai-extracted", null, target.scope, target.project);
engine.indexMemory(id, fact.content, normalizedTags, fact.category);
}
catch {
}
}
}
/** Ids being processed right now — the drain must not double-process them. */
const liveIds = new Set();
let drainRunning = false;
/** C1: recover pending extractions left over from a crash/restart. */
function drainPending(queue, cfg, engine, db, model) {
if (drainRunning)
return;
drainRunning = true;
(async () => {
const entries = await queue.pending(constants_1.PENDING_QUEUE_DRAIN_LIMIT, liveIds);
for (const e of entries) {
liveIds.add(e.id);
try {
await processExtraction(e.text, e.existingSummary, cfg, engine, db, model);
await queue.complete(e.id);
}
catch {
await queue.fail(e.id);
}
finally {
liveIds.delete(e.id);
}
}
})()
.catch(() => { })
.finally(() => { drainRunning = false; });
}
async function promptPreprocessor(ctl, userMessage) {
try { (0, ai_1.setPluginClient)(ctl.client); } catch { }
const cfg = readConfig(ctl);
if (!cfg.autoInject)
return userMessage;
if (userMessage.length < 10)
return userMessage;
try {
const { engine, db } = await (0, toolsProvider_1.getSharedInstances)(cfg.storagePath);
let injectScale = { countFactor: 1, charFactor: 1 };
try {
const pst = await stateManager_1.getPersistedState();
injectScale = (0, contextGauge_1.scaleMemoryInjection)(pst?.contextGauge?.lastPct ?? 0);
}
catch { }
// F2: session compaction hook — digest pending request into memory
let compactLine = "";
try {
const st = await (0, sessionCompact_1.runCompactHook)(ctl, engine, db, cfg);
if (st)
compactLine = st;
}
catch { }
const compactTail = compactLine ? `\n\n${compactLine}` : "";
let chatModel = null;
try {
const ts = typeof ctl.tokenSource === "function" ? await ctl.tokenSource() : null;
if (ts && typeof ts.respond === "function")
chatModel = ts;
}
catch { }
const extractionModel = chatModel ?? undefined;
// L0: Identity memories (always loaded)
const identityMems = db.getIdentityMemories();
// L1: Essential memories (always loaded — most important/accessed)
const essentialMems = db.getEssentialMemories(constants_1.L1_ESSENTIAL_COUNT);
// L2: Query-relevant memories (on-demand based on user message)
const result = await engine.retrieve(userMessage, cfg.contextCount, cfg.decayDays, false);
let queryMemories = [...result.memories];
if (cfg.activeProject) {
const projectMems = db.getByProject(cfg.activeProject, cfg.contextCount);
const existingIds = new Set(result.memories.map((m) => m.id));
for (const pm of projectMems) {
if (!existingIds.has(pm.id))
queryMemories.push(pm);
}
}
// Apply category filter if configured
if (cfg.injectionCategories.length > 0) {
queryMemories = queryMemories.filter((m) => cfg.injectionCategories.includes(m.category.toLowerCase()));
}
queryMemories = queryMemories.slice(0, Math.max(2, Math.round(cfg.contextCount * injectScale.countFactor)));
if (cfg.enableAI && userMessage.length >= 100) {
const existingSummary = result.memories.map((m) => m.content).join("; ");
// C1: register BEFORE extracting — a restart mid-extraction
// leaves the entry on disk; drainPending() recovers it next run.
const queue = (0, pendingQueue_1.getPendingQueue)();
const entryId = queue.enqueue({ text: userMessage, existingSummary });
liveIds.add(entryId);
processExtraction(userMessage, existingSummary, cfg, engine, db, extractionModel)
.then(() => queue.complete(entryId))
.catch(() => queue.fail(entryId))
.finally(() => liveIds.delete(entryId));
drainPending(queue, cfg, engine, db, extractionModel);
}
// Build tiered context (L0 + L1 + L2)
const hasAnyMemories = identityMems.length > 0 || essentialMems.length > 0 || queryMemories.length > 0;
// Identity onboarding: only when NO memories exist at all (true first conversation)
if (!hasAnyMemories) {
return `${userMessage}\n\n---\n\n[Memory System: No memories found. If the user hasn't introduced themselves yet, naturally ask their name, role, and preferred language early in the conversation, then store with Remember(content, "identity"). This makes future conversations personalized.]${compactTail}`;
}
const context = buildTieredContext(identityMems, essentialMems, queryMemories, injectScale.charFactor);
if (!context)
return `${userMessage}${compactTail}`;
return `${userMessage}\n\n---\n\n${context}${compactTail}`;
}
catch {
return userMessage;
}
}
//# sourceMappingURL=preprocessor.js.map