Forked from crunch3r/ai-toolbox
Forked from crunch3r/ai-toolbox
src / promptPreprocessor.ts
src / promptPreprocessor.ts
/**
* Document RAG Prompt Preprocessor + Working Directory Detection + Temporal Awareness
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { type ChatMessage, type FileHandle, type PromptPreprocessorController } from '@lmstudio/sdk';
import { configSchematics } from './config';
import pdfParse from 'pdf-parse';
import type { ContextGuard } from './contextGuard';
import { setAttachments, listAttachments } from './attachmentManager';
// i18n-CONFIRM arc (06.10): confirmation prompts follow the user's configured language ("🌐 Language" menu);
// acceptance stays multilingual via the derived union sets — single source of truth in src/locales/.
import { getConfirmationWords, ALL_CONFIRM_WORDS, ALL_DECLINE_WORDS, getRuntimeTexts, LANGUAGE_DISPLAY_NAMES, fillTemplate } from './locales';
// UNIFIED LANGUAGE v2 (step 4, 27.10): per-turn host/OS language fallback chain — the existing '^\.\/hostLanguage\.js$'
// jest mapper entry resolves this; no new config needed (verified 26.10). Option B: the turn resolver below consumes it here ('message layer').
import { getHostLocale, getOsLocale } from './hostLanguage.js';
// UNIFIED LANGUAGE v2 step 5 (27.10): per-turn state-file sync — extensionless on purpose: ts-jest resolves it via
// moduleFileExtensions and tsup/tsc accept the form ('./locales' precedent), so NO jest mapper entry is required.
import { persistConcretePick } from './uiLabelBake';
import type { LanguageCode, RuntimeTexts } from './locales/types.js'; // type-only — erased at runtime, never jest-resolved (RC#4 class avoided)
/** Shipped locale IDs keyed by their lowercase config form — the concrete values resolveTurnLanguage() accepts. */
const SHIPPED_LANGUAGE_CODES: Readonly<Record<string, LanguageCode>> = {
en: 'en',
de: 'de',
es: 'es',
'zh-cn': 'zh-CN',
'zh-tw': 'zh-TW',
};
/**
* UNIFIED LANGUAGE v2 (step 4): resolve the language for THIS turn's injected runtime texts.
* Chain: (a) a concretely configured language (trimmed/lowercased, matched against the shipped set — ANY
* non-matching value falls through by construction; step 5 will add 'auto', which must keep falling
* through); (b) the LM Studio host preference (getHostLocale() !== ''); (c) the OS locale's first tag
* ('de-DE' → 'de') matched against the same set; (d) English. Pure function of its inputs — exported so the
* fallback chain stays jest-testable without SDK mocks.
*/
export function resolveTurnLanguage(configured?: string | null): LanguageCode {
if (typeof configured === 'string') {
const code = SHIPPED_LANGUAGE_CODES[configured.trim().toLowerCase()];
if (code) return code; // concrete pick wins — anything else (incl. future 'auto') falls through by design
}
const hostLocale = getHostLocale();
if (hostLocale !== '') return hostLocale;
const osTag = getOsLocale().split('-')[0];
return SHIPPED_LANGUAGE_CODES[osTag] ?? 'en';
}
import { autoTracker } from './autoTracker';
import { TokenStatsManager, drainActiveToolTurns, estimateTokensFromChars, setAgenticCharsPerToken } from './tokenStatsManager';
// C compaction family (24.09): pre-summarization pruning of oversized tool payloads (DeepSeek harness item C)
import { DEFAULT_MAX_BYTES_PER_RESULT, pruneOversizedToolPayloads } from './utils/toolPayloadCompaction.js';
import { collectPrunedPayloadRefs, storePrunedPayload } from './utils/toolPayloadStorage.js';
import { getToolOverheadChars } from './toolOverhead.js';
import { getWorkingDir, setWorkingDir, listRegisteredProjects } from './workingDir.js';
// CONTAMINATION-FIX Part A (01.10): identity rebind after Step 0.7 CWD switches. No import cycle: only index.ts
// imports both modules; toolsProvider does not reference promptPreprocessor.
import { getStateManager } from './toolsProvider.js';
// UPDATE-ANNOUNCE (19.10, owner GO): one-shot in-chat update announcement accessor — direct import of the pure state+fetch module is safe here
// (no cycle; toolsProvider already imports it). Jest resolution: this specifier matches the existing '^\.\/updateChecker\.js$' mapper entry (Phase-1), so no new config.
import { getPendingUpdateAnnouncement, LOCAL_VERSION } from './updateChecker.js';
// Arc C (07.10): forced structured session-memory save at the compression trigger — persists via the canonical
// save_session_summary writer path (see src/sessionSummaryPersist.ts); cycle-free edge set (only index.ts imports both sides).
import { persistGeneratedSessionSummary } from './sessionSummaryPersist.js';
// CONTAMINATION-FIX follow-up C / Gap-2 (01.10): static decode import for the post-switch session-memory read in
// executeProjectSwitch(). Replaces a per-call `await import('@msgpack/msgpack')` — dynamic native-import under a
// CJS build is fragile, and this aligns with how every other module in src/ consumes @msgpack/msgpack.
import { decode as decodeMsgpack } from '@msgpack/msgpack';
interface ExtendedMessage {
role?: string;
content?: unknown;
toolCalls?: unknown[];
tool_calls?: unknown[];
files?: unknown[];
images?: unknown[];
}
/** Minimal interface for LM Studio LLM model objects with optional metadata */
interface LLMModelWithOptionalFields {
modelKey?: string;
id?: string;
}
// 🔹 F1 (17.09 G-B fix): PURE decision/formatting helper for the one-shot "auto-checkpoint saved before
// compression" notice. Extracted out of preprocess() PART B so it is jest-testable without SDK mocks.
// Origin: 17.09 log verdict (memory_1789659390790) — the threshold prompt generated at 17:16:31 was consumed
// by consumePendingConfirmation() in the same run with ZERO user-visible acknowledgement.
export interface CheckpointNoticeInput {
/** The pre-compression snapshot save succeeded this turn. */
snapshotSaved: boolean;
/** A checkpoint warning was live during the same preprocess run (generated here, or pending from a prior turn). */
warningWasLive: boolean;
/** Pre-compression usage in percent of the model context window (tokenCount/maxTokens*100); null when unknown. */
usagePercent: number | null;
/** Saved snapshot id, when available. */
sessionId?: string;
}
export function buildCheckpointSavedNotice(input: CheckpointNoticeInput): string | null {
if (!input.snapshotSaved || !input.warningWasLive) {
return null; // warning path (PART B-2 last-chance note) or plain compression — unchanged behavior
}
const pct = typeof input.usagePercent === 'number' && Number.isFinite(input.usagePercent)
? ` at ~${(Math.round(input.usagePercent * 10) / 10)}% of context` // one-decimal rounding (matches toFixed(1) percent convention elsewhere); outer Math.round(x*10/10) was a no-op that printed integers
: '';
const idPart = input.sessionId ? ` (checkpoint ${input.sessionId})` : '';
return '\n\n✅ AUTO-CHECKPOINT SAVED BEFORE COMPRESSION (one-shot notice — not a prompt):\n' +
'A machine telemetry record (auto-written; not a manually saved session memory) was stored' + pct + idPart + ' in place of the token-limit warning for this turn — the history that prompt referred to has been compressed.\n' +
"Briefly acknowledge to the user that the auto checkpoint ran before compression; no YES/NO reply or further action is required.";
}
// --- Temporal Awareness Helpers (merged from up_to_date) ---
interface DateTimeCache {
compact: string;
full: string;
}
let cachedDateTimeData: DateTimeCache | null = null;
const CACHE_DURATION_MS = 5 * 60 * 1000; // Refresh every 5 minutes
// ContextGuard integration
let contextGuard: ContextGuard | null = null;
export function setContextGuard(guard: ContextGuard | null): void {
contextGuard = guard;
}
/**
* FIX #21 (09.10, two-tier mid-loop save) — null-safe read accessor for the active ContextGuard instance.
* setContextGuard() stays the ONE-WAY injection point (index.ts constructs once at startup; test harnesses swap
* fakes/reset to null between suites). This getter is the first external READ path: the toolsProvider wrapper
* consumes it to reach generateSessionContinuity()/getThreshold(). Consumers MUST handle null (guard not yet
* constructed or detached) and keep their own typeof method-guards for stub instances — same defensive contract
* as Arc C's `typeof contextGuard.generateSessionContinuity === 'function'` gate.
*/
export function getContextGetter(): ContextGuard | null {
return contextGuard;
}
let cacheTimestamp = 0;
function getCachedDateTime(): DateTimeCache {
const now = Date.now();
if (cachedDateTimeData && (now - cacheTimestamp) < CACHE_DURATION_MS) {
return cachedDateTimeData;
}
const date = new Date();
// Compact format: DD.MM.YYYY, HH:mm
const compact = date.toLocaleString('de-DE', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit'
});
// Full format: Wochentag, DD. MMMM YYYY, HH:mm Uhr
const full = date.toLocaleString('de-DE', {
weekday: 'long',
year: 'numeric',
month: 'long',
day: 'numeric',
hour: '2-digit',
minute: '2-digit'
}) + ' Uhr';
cachedDateTimeData = { compact, full };
cacheTimestamp = now;
return cachedDateTimeData;
}
function getTemporalSuffix(ctl: PromptPreprocessorController): string {
// Option A (02.11): ALL read sites in this file use global scope — settings persist across chats and app restarts.
const config = ctl.getGlobalPluginConfig(configSchematics);
// Use .get() method with proper defaults - more reliable than direct property access
const temporalAwarenessEnabled = config.get('temporalAwareness') ?? true;
if (!temporalAwarenessEnabled) {
return '';
}
const style = config.get('dateFormatStyle') ?? 'standard';
const { compact, full } = getCachedDateTime();
// DEBUG: Uncomment to verify what's being injected
console.log(`[TEMPORAL] Injecting: ${style === 'heuteIst' ? `HEUTE IST ${full}` : `[Zeit: ${compact}]`}`);
if (style === 'heuteIst') {
return `\n\nHEUTE IST ${full}`;
}
return `\n\n[Zeit: ${compact}]`;
}
/** Normalizes a project name for fuzzy matching (hyphens ↔ underscores, lowercase) */
function normalizeProjectName(name: string): string {
return name.toLowerCase().replace(/[-_\s]+/g, '_');
}
/** Generates variants of a project name for fuzzy matching */
function generateNameVariants(name: string): string[] {
const normalized = normalizeProjectName(name);
const variants = new Set<string>();
variants.add(normalized);
variants.add(normalized.replace(/-/g, '_'));
variants.add(normalized.replace(/_/g, '-'));
variants.add(normalized.replace(/[-_\s]/g, ''));
return Array.from(variants);
}
/** Detect if a user message contains a registered project keyword.
* Registry resolution chain: <pluginRoot>/.session_context/project_registry.json (primary)
* → <pluginRoot>/.session_index.json (legacy fallback).
* The registry is loaded ONCE per invocation and reused across all candidate words
* (previously re-read from disk for every candidate word, and the fallback was never consulted — dead in installed envs). */
export function detectProjectKeyword(
text: string,
options?: { pluginRoot?: string },
): { name: string; path: string } | null {
// Extract candidate words/phrases from the message (ignore common verbs/prepositions)
const stopWords = new Set([
'let', 'us', 'work', 'on', 'the', 'a', 'an', 'in', 'at', 'to', 'for', 'with',
'about', 'start', 'begin', 'open', 'switch', 'go', 'back', 'continue', 'resume'
]);
// FIX: length >= 2 (was > 2) — short name components like 'my' in "my cool project" were dropped,
// so multi-word names containing 2-letter parts could never match. Digits excluded to avoid
// matching version numbers / years against numeric name variants; stopword list still filters the rest.
const words = text.toLowerCase().split(/\s+/).filter(w => w.length >= 2 && !stopWords.has(w) && !/^\d+$/.test(w));
if (words.length === 0) return null;
// Load the registry once per invocation (primary → legacy fallback), then match in memory.
const projects = listRegisteredProjects(options?.pluginRoot);
if (projects.length === 0) return null;
// Precompute fuzzy name variants per project to avoid re-computing them for every candidate word
const indexed = projects.map(p => ({ variants: new Set(generateNameVariants(p.name)), project: p }));
// Try matching combinations of 1-3 consecutive words as project name candidates
for (let len = Math.min(3, words.length); len >= 1; len--) {
for (let i = 0; i <= words.length - len; i++) {
const candidate = words.slice(i, i + len).join('_');
for (const entry of indexed) {
if (entry.variants.has(candidate)) {
return { name: entry.project.name, path: entry.project.path };
}
}
}
}
return null;
}
function detectDirectoryPath(text: string): string | null {
// Remove URLs first to avoid false positives like /medium.com from https://medium.com/...
const withoutUrls = text.replace(/https?:\/\/[^\s]+|www\.[^\s]+|file:\/\/[^\s]+/g, '');
// Windows paths: C:\path or D:\folder (must start with drive letter)
const winMatch = withoutUrls.match(/[A-Za-z]:\\[\w\-_. \\]+/);
// ^^^^^^^^^^
// Backslash added ✓
if (winMatch) return winMatch[0].trim();
// Unix absolute paths: /home/user/dir, /var/log, etc.
const unixMatch = withoutUrls.match(/(?:^|\s)(\/[\w\-_. ]{2,})/);
if (unixMatch) {
const path = unixMatch[1].trim();
// Reject paths that look like URLs or fragments (e.g., / Chat files s)
if (!path.startsWith('/ ') && !path.includes(' ')) {
return path;
}
}
// Relative paths: ./folder, ../parent/dir
// Relative paths: ./folder, ../parent/dir
const relMatch = withoutUrls.match(/(?:^|\s)(?:\.\/|\.\\.\/)[\w\-_. ]+/);
if (relMatch) return relMatch[0].trim();
return null;
}
// 🔹 i18n-CONFIRM arc (06.10): option words follow the user's configured language when passed;
// without them the legacy bilingual wording is preserved byte-for-byte for any other caller/test.
// UNIFIED LANGUAGE v2 (step 4, 27.10): banner texts come from the per-turn RuntimeTexts when passed — the box-drawing
// separators and indentation stay OUTSIDE those strings by design; the 'en' render MUST equal the pre-v2 output byte-for-byte.
function injectWorkingDirectoryPrompt(
originalMessage: string,
detectedPath: string,
words?: { yes: string; no: string },
runtimeTexts?: RuntimeTexts,
): string {
const rt = runtimeTexts ?? getRuntimeTexts('en'); // absent → English bundle (byte-identical to the pre-v2 literals)
const replyHint = words
? fillTemplate(rt.workingDirReplyHint, { yes: words.yes, no: words.no })
: "Reply 'yes' or 'ja' to confirm, or 'no'/'nein' to decline."; // legacy bilingual fallback preserved byte-for-byte
const instruction = `
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirDetected}
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirMentionedLine}
${detectedPath}
${rt.workingDirConfirmAsk}
${rt.workingDirExampleLabel}
"${fillTemplate(rt.workingDirExampleText, { path: detectedPath })}
${replyHint}"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirOriginalMessageLabel}
${originalMessage}
`;
return instruction.trim();
}
async function extractPdfText(fileHandle: FileHandle): Promise<string> {
try {
// Typed interface for FileHandle with optional readFile method
type FileHandleWithReadFile = {
name: string;
readFile?: () => Promise<Buffer>;
read?: () => Promise<unknown>;
};
const typedHandle = fileHandle as unknown as FileHandleWithReadFile;
const buffer = typedHandle.readFile
? await typedHandle.readFile()
: Buffer.from(await (typedHandle.read?.() as Promise<string>) ?? '');
const data = await pdfParse(buffer);
return data.text.trim();
} catch (error) {
console.error(`[RAG] Error extracting text from PDF ${fileHandle.name}:`, error);
throw new Error(`Failed to parse PDF: ${fileHandle.name}`);
}
}
function chunkText(text: string, chunkSize: number = 1000, overlap: number = 100): string[] {
const words = text.split(/\s+/);
const chunks: string[] = [];
if (words.length <= chunkSize) {
return [text];
}
let startIndex = 0;
while (startIndex < words.length) {
const endIndex = Math.min(startIndex + chunkSize, words.length);
const chunkText = words.slice(startIndex, endIndex).join(' ');
chunks.push(chunkText);
startIndex = endIndex - overlap;
}
return chunks.filter(c => c.trim().length > 0);
}
function cosineSimilarity(a: number[], b: number[]): number {
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
interface RetrievalResult {
content: string;
score: number;
}
async function retrieveFromPdfs(
ctl: PromptPreprocessorController,
query: string,
pdfFiles: FileHandle[],
): Promise<RetrievalResult[]> {
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const retrievalLimit = pluginConfig.get('retrievalLimit') || 5;
// Lower default threshold to catch more results - was too high at 0.6
const retrievalAffinityThreshold = pluginConfig.get('retrievalAffinityThreshold') ?? 0.3;
console.log(`[RAG] Processing ${pdfFiles.length} PDF file(s)`);
// Extract text from all PDF files
const fileTexts: { file: FileHandle; text: string }[] = [];
for (const file of pdfFiles) {
try {
const text = await extractPdfText(file);
if (text.length > 0) {
console.log(`[RAG] Extracted ${text.length} chars from ${file.name}`);
fileTexts.push({ file, text });
} else {
console.log(`[RAG] No text extracted from ${file.name}`);
}
} catch (error) {
console.error(`[RAG] Skipping PDF ${file.name} due to error:`, error);
}
}
if (fileTexts.length === 0) {
console.log('[RAG] No text extracted from any PDF');
return [];
}
// Chunk the texts
const chunks: { file: FileHandle; chunk: string }[] = [];
for (const { file, text } of fileTexts) {
const fileChunks = chunkText(text);
console.log(`[RAG] ${file.name}: ${text.length} chars → ${fileChunks.length} chunks`);
fileChunks.forEach((chunk) => {
chunks.push({ file, chunk });
});
}
if (chunks.length === 0) return [];
// Generate embeddings for all chunks using LM Studio's embedding model
let model;
try {
console.log('[RAG] Loading embedding model...');
model = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
console.log('[RAG] Embedding model loaded successfully');
} catch (error) {
console.error('[RAG] Failed to load embedding model:', error);
throw new Error(`Embedding model not available: ${error instanceof Error ? error.message : String(error)}`);
}
const batchSize = 32;
const allEmbeddings: number[][] = [];
try {
for (let i = 0; i < chunks.length; i += batchSize) {
console.log(`[RAG] Generating embeddings batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(chunks.length / batchSize)}...`);
const batch = chunks.slice(i, i + batchSize).map(c => c.chunk);
const embeddingsResult = await model.embed(batch);
// Type the embedding result properly
type EmbeddingResult = { embedding: number[] };
allEmbeddings.push(...(embeddingsResult as EmbeddingResult[]).map((e) => e.embedding));
}
} catch (error) {
console.error('[RAG] Error generating embeddings:', error);
throw new Error(`Embedding generation failed: ${(error instanceof Error ? error.message : String(error))}`);
}
// Generate embedding for the query
let queryModel;
try {
queryModel = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
} catch (error) {
console.error('[RAG] Failed to load query embedding model:', error);
throw new Error(`Query embedding failed: ${(error instanceof Error ? error.message : String(error))}`);
}
let queryEmbedding;
try {
const queryResult = await queryModel.embed([query]);
queryEmbedding = queryResult[0].embedding;
} catch (error) {
console.error('[RAG] Error generating query embedding:', error);
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`Query embedding failed: ${errorMessage}`);
}
// Calculate similarities and retrieve top results
const scores: { chunkIndex: number; similarity: number }[] = [];
for (let i = 0; i < chunks.length; i++) {
const similarity = cosineSimilarity(queryEmbedding, allEmbeddings[i]);
scores.push({ chunkIndex: i, similarity });
}
// Sort by similarity descending and filter by threshold
scores.sort((a, b) => b.similarity - a.similarity);
console.log(`[RAG] Found ${scores.length} chunks, filtering with threshold ${retrievalAffinityThreshold}`);
const relevantChunks = scores.filter(
(s) => s.similarity >= retrievalAffinityThreshold && s.chunkIndex < chunks.length,
);
// Limit results
const limitedResults = relevantChunks.slice(0, retrievalLimit);
console.log(`[RAG] Returning ${limitedResults.length} results`);
return limitedResults.map((r) => ({
content: chunks[r.chunkIndex].chunk,
score: r.similarity,
}));
}
// ==================== Step 0.7 Confirm-First Project Switching (Fix A) ====================
/**
* Pending project-switch confirmation state (Fix A). Set when a registered project is detected in
* a user message; consumed one-shot only when the user explicitly replies YES/JA on a subsequent
* message. Cleared on decline, expiry, or successful switch.
*/
let pendingProjectSwitch: { name: string; path: string } | null = null;
/**
* Normalize a multilingual confirmation reply to the canonical FSM input (Fix B, extended by the
* i18n-CONFIRM arc 06.10). Accepts EVERY shipped language's confirm/decline words — union derived
* from src/locales/*.ts at module load (YES/JA/SÍ/确认/確認 → 'YES'; NO/NEIN/取消 → 'NO'); anything
* else → null (not a confirmation reply). Language-agnostic by design: only the user's configured
* language is DISPLAYED, but any shipped word still parses (owner verdict 06.10 ~19:0x).
*/
export function normalizeConfirmationReply(raw: string): 'YES' | 'NO' | null {
const u = raw.trim().toUpperCase();
if (ALL_CONFIRM_WORDS.has(u)) return 'YES';
if (ALL_DECLINE_WORDS.has(u)) return 'NO';
return null;
}
/** Outcome of Step 0.7's confirm-first project-switch gate (Fix A). */
export type ProjectSwitchDecision =
| { kind: 'skip' }
| { kind: 'banner'; match: { name: string; path: string } }
| { kind: 'execute'; match: { name: string; path: string } }
| { kind: 'declined' };
/**
* Pure decision logic for Step 0.7 (Fix A — v1.9.8+ safety rule in the index.ts docblock):
* NEVER switch the working directory on detection alone. The banner asks for confirmation;
* only an explicit YES/JA reply executes the one-shot switch. NO/NEIN or any other reply
* expires the offer (the "next message" contract).
*/
export function decideProjectSwitch(
match: { name: string; path: string } | null,
reply: string,
currentCwd: string,
pending: { name: string; path: string } | null,
): ProjectSwitchDecision {
if (!match) return { kind: 'skip' };
// Already in the detected project's directory → a switch would be a no-op.
if (path.resolve(currentCwd) === path.resolve(match.path)) return { kind: 'skip' };
const normalized = normalizeConfirmationReply(reply);
if (pending && pending.name === match.name && path.resolve(pending.path) === path.resolve(match.path)) {
if (normalized === 'YES') return { kind: 'execute', match }; // one-shot switch on explicit confirmation
return { kind: 'declined' }; // NO/NEIN or any non-confirmation reply → expire the offer
}
// New detection, or a different project than the pending offer → confirm-first banner.
return { kind: 'banner', match };
}
// ==================== Session-Memory Extraction Helpers ====================
/** Coerce a decoded session-summary value into plain text for prompt injection.
* StateManager writes the record as an OBJECT ({task_description, ...}); older/legacy writers used bare strings. */
function summaryValueToText(value: unknown): string | undefined {
if (typeof value === 'string' && value.length > 0) return value;
if (value && typeof value === 'object') {
const o = value as Record<string, unknown>;
if (typeof o.task_description === 'string' && o.task_description.length > 0) return o.task_description;
try { return JSON.stringify(value); } catch { return undefined; }
}
return undefined;
}
/** CONTAMINATION-FIX follow-up C / Gap-2b (01.10): pull the latest session summary out of a decoded store,
* understanding BOTH writer shapes:
* - CURRENT: StateManager record array [{ key, value, timestamp }] with key 'session_summary_latest'
* (value is an object or string) — see stateManager.ts / contextManagementTools.ts saveSessionSummary;
* - LEGACY: plain object map { latest_session_summary | session_summary_latest: string }.
* Pre-fix the caller only understood the legacy map shape, so real StateManager stores were decoded but
* never yielded a summary ("no summary found" despite a populated file). Returns undefined when no usable
* summary is present (not an error). */
function extractLatestSessionSummary(decoded: unknown): string | undefined {
if (!decoded || typeof decoded !== 'object') return undefined;
// CURRENT writer shape: array of {key, value, timestamp} records
if (Array.isArray(decoded)) {
for (let i = decoded.length - 1; i >= 0; i--) {
const r = (decoded as Array<Record<string, unknown>>)[i];
if (!r || typeof r !== 'object') continue;
if (typeof r.key === 'string' && r.key.includes('session_summary')) return summaryValueToText(r.value);
}
return undefined;
}
// LEGACY writer shape: object map with string values
const d = decoded as Record<string, unknown>;
for (const k of ['latest_session_summary', 'session_summary_latest']) {
// 🔹 LINT FIX (01.10): the typeof guard already narrows d[k] to string — the two "as string" assertions were no-ops tripping @typescript-eslint/no-unnecessary-type-assertion.
if (typeof d[k] === 'string' && d[k].length > 0) return d[k];
}
const objVal = d.session_summary_latest ?? d.latestSummary;
return summaryValueToText(objVal);
}
/**
* Execute a confirmed project CWD switch (Step 0.7, Fix A): canonical persistent state change via
* setWorkingDir() plus best-effort process.chdir(), then load and inject the target project's
* session memory (msgpack with plain-JSON fallback). Returns the fully composed prompt for this turn.
*/
// CONTAMINATION-FIX follow-up C (01.10): exported for the hermetic session-memory read-guard tests
// (tests/sessionMemoryReadGuards.test.ts) — the confirm-first flow in preprocess() stays the only production caller.
// UNIFIED LANGUAGE v2 (step 4): optional per-turn RuntimeTexts for the session-memory banners below — absent →
// English bundle via getRuntimeTexts('en') (byte-identical to the pre-v2 literals; keeps non-preprocess callers/tests untouched).
export async function executeProjectSwitch(
newCwd: string,
userPrompt: string,
attachmentNotice: string,
checkpointSuffix: string,
ctl: PromptPreprocessorController,
runtimeTexts?: RuntimeTexts,
): Promise<string> {
console.log(`[ProjectAutoDetect] Switching working directory to: ${newCwd}`);
try {
// Canonical CWD change — persistent working-dir state (authoritative for all getWorkingDir() consumers)
// plus best-effort process.chdir. Previously only process.chdir was called, so tool-level CWD never changed.
if (!applyProjectCwdSwitch(newCwd)) {
console.warn(`[ProjectAutoDetect] CWD switch rejected for '${newCwd}' — proceeding without switch`);
return `${userPrompt}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
// Read session memory from the new working directory (if exists)
try {
// CONTAMINATION-FIX follow-up C / Gap-2 (01.10): candidate store files — per-project FIRST, with the
// EXACT naming StateManager uses for its own file (`resolveProjectName()` derivation over this project's
// basename), legacy fixed-name .ai_toolbox_memory.msgpack as fallback. At this point applyProjectCwdSwitch()
// has already re-bound the working dir to newCwd, so deriving from newCwd's basename is byte-identical to
// StateManager.getMemoryFilePath()'s live derivation — no cross-project reads (RESEARCH doc §4).
const memoryCandidates = [
path.join(newCwd, '.session_context', `.${path.basename(newCwd).toLowerCase().replace(/[^a-z0-9]/g, '_')}_memory.msgpack`),
path.join(newCwd, '.session_context', '.ai_toolbox_memory.msgpack'),
];
let msgpackPath: string | null = null;
for (const candidate of memoryCandidates) {
if (fs.existsSync(candidate)) {
msgpackPath = candidate;
break;
}
}
if (msgpackPath !== null) {
console.log(`[ProjectAutoDetect] Found session memory at ${msgpackPath}`);
// Load and decode the store file via the static @msgpack/msgpack import (see module header — replaces
// the former per-call `await import('@msgpack/msgpack')`, fragile under CJS).
try {
const rawBytes = fs.readFileSync(msgpackPath);
if (rawBytes.length > 0) {
// CONTAMINATION-FIX follow-up C / Gap-2b (01.10): understand BOTH writer shapes — the CURRENT
// StateManager record array [{key,value,timestamp}] and LEGACY object maps — via extractLatestSessionSummary
// (above). Pre-fix this path only understood the legacy map shape, so real stores decoded fine but
// NEVER yielded a summary.
const latestSummary: string | undefined = extractLatestSessionSummary(decodeMsgpack(rawBytes));
// If found, inject session summary context into the prompt
if (latestSummary && latestSummary.length > 0) {
console.log(`[ProjectAutoDetect] Loaded session memory from ${newCwd}`);
// UNIFIED LANGUAGE v2 (step 4): banner title/label from the per-turn RuntimeTexts — 📋 + box-drawing structure stays literal here.
const memRt = runtimeTexts ?? getRuntimeTexts('en');
const sessionContext = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${memRt.sessionMemoryLoadedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${memRt.previousSessionLabel}\n${latestSummary}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`;
return `${userPrompt}${sessionContext}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
} else {
console.log(`[ProjectAutoDetect] Session memory exists but no summary found`);
}
}
} catch (decodeError) {
console.warn(`[ProjectAutoDetect] Failed to decode msgpack from ${msgpackPath}:`, decodeError instanceof Error ? decodeError.message : String(decodeError));
// Fallback: try reading as plain JSON if msgpack fails — the shared helper understands both record
// arrays and object maps, so no duplicated shape-check code is needed here.
try {
const rawJson = fs.readFileSync(msgpackPath, 'utf-8');
const summaryText: string | undefined = extractLatestSessionSummary(JSON.parse(rawJson));
if (summaryText && summaryText.length > 0) {
console.log(`[ProjectAutoDetect] Loaded session memory from ${newCwd} (JSON fallback)`);
// UNIFIED LANGUAGE v2 (step 4): same RuntimeTexts-driven banner as the msgpack path above.
const memRt = runtimeTexts ?? getRuntimeTexts('en');
const sessionContext = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${memRt.sessionMemoryLoadedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${memRt.previousSessionLabel}\n${summaryText}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`;
return `${userPrompt}${sessionContext}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
} catch (jsonError) {
// Defensive net: a malformed JSON fallback (JSON.parse throw) must never abort the switch.
console.warn(`[ProjectAutoDetect] JSON fallback also failed:`, jsonError instanceof Error ? jsonError.message : String(jsonError));
}
}
} else {
console.log(`[ProjectAutoDetect] No session memory found under ${newCwd} (no per-project or legacy store file present)`);
}
} catch (e) {
console.warn(`[ProjectAutoDetect] Failed to load session memory from ${newCwd}:`, e instanceof Error ? e.message : String(e));
}
} catch (e) {
console.error(`[ProjectAutoDetect] Error detecting/switching project:`, e);
}
// If automatic switch failed or no session memory found, return original message with context
return `${userPrompt}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
/** Apply the project CWD switch (Step 0.7).
* Canonical change = persistent working-dir state via setWorkingDir() — authoritative for all getWorkingDir() consumers;
* plus a best-effort process.chdir so raw process.cwd() consumers (git/image tools) follow along too.
* Previously only process.chdir was called, so the tool-level CWD (state file) never actually changed.
* Returns false if the switch was rejected (path invalid). */
export function applyProjectCwdSwitch(newCwd: string): boolean {
// CONTAMINATION-FIX Part A (01.10): capture the pre-switch working dir — required to flush the previous
// project's pending RAM into its OWN file when rebinding StateManager identity below.
const prevWd = getWorkingDir();
if (!setWorkingDir(newCwd)) return false;
try {
const oldCwd = process.cwd();
if (oldCwd !== newCwd && typeof process.chdir === 'function') {
process.chdir(newCwd);
console.log(`[ProjectAutoDetect] Working directory changed: ${oldCwd} → ${newCwd}`);
}
} catch (chdirError) {
// Non-fatal — persistent state is already set; only raw process.cwd() consumers stay behind.
console.warn(`[ProjectAutoDetect] process.chdir failed (non-fatal): ${chdirError instanceof Error ? chdirError.message : String(chdirError)}`);
}
// ✅ CONTAMINATION-FIX Part A (01.10): fire-and-forget identity rebind — this function is synchronous by its
// contract, so the async flush/reload runs after return. Errors are contained inside refreshProject; the
// catch here covers only unforeseen throws (see stateManager.ts for the ordering guarantees).
void getStateManager()
?.refreshProject(prevWd)
.catch((err: unknown) => {
console.warn(`[ProjectAutoDetect] StateManager identity rebind after CWD switch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
});
return true;
}
export async function preprocess(
ctl: PromptPreprocessorController,
userMessage: ChatMessage
): Promise<string | ChatMessage> {
let userPrompt = '';
try {
const rawText = await Promise.resolve(userMessage.getText());
userPrompt = typeof rawText === 'string' ? rawText : String(rawText ?? '');
} catch {}
// UNIFIED LANGUAGE v2 (step 4, 27.10): resolve THIS turn's injected-text language ONCE at the top of the message layer —
// chain = concretely configured value → LM Studio host preference → OS locale first tag → English fallback
// (see resolveTurnLanguage()). MUST sit before any consumer below: the S5 autoTracker call and the answer-language line
// live INSIDE the ContextGuard try-block, whose finally runs well before Step 0.6+. One extra config fetch vs. a later
// site — negligible per turn (config reads are fresh per call elsewhere in this file too).
const turnLang = resolveTurnLanguage(ctl.getGlobalPluginConfig(configSchematics).get('language'));
const rt = getRuntimeTexts(turnLang);
// UNIFIED LANGUAGE v2 step 5 (27.10): keep the boot-bake state file in sync with THIS session's effective language —
// after any reload, panel labels and prompt texts render in the same locale. persistConcretePick() is change-aware
// (no write while the value is unchanged) and non-throwing by contract; i18n persistence must never break a turn.
try {
persistConcretePick(turnLang);
} catch { /* non-fatal by design — the boot chain re-resolves on next load */ }
// ✅ DIAGNOSTIC: Verify preprocessor is actually being called by LM Studio
console.log(`✅ [Preprocessor] Called. Message length: ${userPrompt.length}`);
if (!contextGuard) { console.warn('[ContextGuard] contextGuard is NULL!'); }
// 🔹 C compaction family (24.09) SERIALIZATION GUARD: a new user message can arrive while the previous
// turn's agentic tool loop is still executing in this process. Drain with a BOUNDED wait before any
// turn-level state mutation below — otherwise in-flight results would be recorded against an already-
// cleared baseline (resetMidLoopDelta) and auto-compression could replace history under live tools
// (DeepSeek harness item C: "serialize user-triggered operations against active loops"). Bounded by
// design: on timeout the drain logs FAIL-LOUD and proceeds; a user message must never deadlock.
try {
await drainActiveToolTurns();
} catch (drainErr) {
console.warn('[C-compact] Drain of active tool turn failed (non-fatal, proceeding):', drainErr);
}
// 🔹 FIX #20 (A1): reset the per-turn tool-payload delta. By the time this runs, last turn's tool
// results are part of pullHistory() and get counted natively below — keeping them in the delta
// would double-count them on the next mid-loop guard evaluation.
TokenStatsManager.resetMidLoopDelta();
// 🔹 Declare variables at top scope so they're accessible across all steps
let pendingWarning: string | undefined;
// 🔹 PART B (re-applied 21.08 after restore lost it): set when the pre-compression checkpoint did
// NOT save but a YES/NO prompt was still pending — a last-chance note is appended to this turn's output.
let lastChanceNote = false;
// 🔹 Fix A/B coordination: set when Step 0.6 consumes a checkpoint reply this turn —
// prevents the same reply from being re-interpreted as a project-switch confirmation in Step 0.7.
let checkpointConsumedThisTurn = false;
let messageCount = 0;
let tokenCount = 0;
// 🔹 FIX #3 (04.10, CTX-FOOTER arc part 3): function-scoped so the finally-publish below can see it even
// when a throw lands before the assignment further down — hoisted from its former const inside the try body.
let maxTokens = 0;
// 🔹 FIX #21 (09.10, two-tier mid-loop save): function-scoped like the others above so the finally-publish can
// see it even when a throw lands before any assignment below — stays null then, i.e. "no list published for this
// turn" (the wrapper's trigger gate treats null as no-op, same safe direction as baseline === 0).
let publishedTurnMessages: { role?: string; content?: unknown; [key: string]: unknown }[] | null = null;
let historyTextLength = 0; // ✅ Moved outside if-block to fix scoping
// Step 0.5: ContextGuard auto-compression & token tracking
if (contextGuard) {
try {
console.log('[ContextGuard DEBUG] Step 1: pullHistory()...');
const history = await ctl.pullHistory();
console.log('[ContextGuard DEBUG] Step 2: checking history object...');
if (!history) { console.warn('[ContextGuard] pullHistory() returned undefined, skipping'); return userPrompt; }
history.append(userMessage);
console.log('[ContextGuard DEBUG] Step 3: getMessagesArray()...');
const messages = history.getMessagesArray() as unknown as { role?: string; content?: unknown; [key: string]: unknown }[] | undefined;
console.log('[ContextGuard DEBUG] Step 4: messages type:', typeof messages, 'isArray:', Array.isArray(messages));
// Safely extract files/images, handling both arrays and single objects
let toolCallCount = 0;
let imageCount = 0;
if (messages && Array.isArray(messages)) {
for (const msg of messages) {
const typedMsg = msg as ExtendedMessage;
if (typedMsg.toolCalls || typedMsg.tool_calls) {
toolCallCount += typedMsg.toolCalls?.length || typedMsg.tool_calls?.length || 0;
}
const fileObj = typedMsg.files || typedMsg.images;
if (fileObj && Array.isArray(fileObj) && fileObj.length > 0) {
imageCount += fileObj.length;
} else if (fileObj && typeof fileObj === 'object') {
// Handle single object case
imageCount += 1;
}
}
// ✅ Use LM Studio's native history API instead of .content casting (matches vibe-lm approach)
historyTextLength = 0;
try {
const msgCount = history.getLength();
for (let i = 0; i < msgCount; i++) {
const msg = history.at(i);
let msgText = '';
if (msg.getText) msgText += msg.getText() || '';
if (msg.getToolCallRequests) msgText += JSON.stringify(msg.getToolCallRequests()) || '';
if (msg.getToolCallResults) msgText += JSON.stringify(msg.getToolCallResults()) || '';
historyTextLength += msgText.length;
}
} catch (e) {
console.warn('[TokenDebug] Failed to iterate native history:', e);
}
// ✅ v2.x: Include serialized tool definitions in the counted span — LM Studio's sidebar counts them too;
// previously only message history was counted here (root cause of "sidebar ~265k vs plugin count much lower").
const toolOverheadChars = getToolOverheadChars();
if (toolOverheadChars > 0) {
console.log(`[TokenDebug] Tool-definition overhead: +${toolOverheadChars} chars added to counted span`);
historyTextLength += toolOverheadChars;
}
const estimatedHistoryTokens = estimateTokensFromChars(historyTextLength); // shared ratio - see tokenStatsManager
// 15.10: label no longer hardcodes a ratio — the density is user-editable (contextGuardAgenticDensity) and live in tokenStatsManager.
console.log(`[TokenDebug] History Text Length: ${historyTextLength} chars | Est. Tokens (shared agentic density, live — see tokenStatsManager): ${estimatedHistoryTokens}`);
}
console.log(`[TokenDebug] Total Tool calls: ${toolCallCount}, Total Files/Images: ${imageCount}`);
// ✅ FIX: Dynamically resolve actual model context length before counting tokens
console.log('[ContextGuard DEBUG] Step 5: getGlobalPluginConfig()...');
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
// Inject LM Studio client from controller for dynamic model queries (SDK v1.x)
if (ctl.client && contextGuard.setLMClient) {
contextGuard.setLMClient(ctl.client);
}
// Re-seed the user's configured compression trigger PERCENTAGE every turn (07.10). ContextGuard is
// constructed in index.ts WITHOUT config access, so without this reseed a menu change would only take
// effect after a plugin reload. The effective WINDOW still comes from setTokenLimitFromModel() below
// (model-reported window; built-in 30k fallback when unavailable) — the percent applies on top of it.
const compressionPercent = pluginConfig.get('contextGuardCompressionPercent');
if (typeof compressionPercent === 'number' && Number.isFinite(compressionPercent) && compressionPercent > 0) {
contextGuard.updateConfig({ compressionPercent });
}
// Re-seed the user's configured agentic chars-per-token DENSITY every turn (15.10 owner request). estimateTokensFromChars()
// is a module-level function in tokenStatsManager WITHOUT config access, so — exactly like the percent above — a menu change
// only takes effect via this per-turn reseed. Stub/plugin configs returning undefined keep the built-in default 3.6 (raised from 3.4 on 15.10 owner instruction).
const agenticDensity = pluginConfig.get('contextGuardAgenticDensity');
if (typeof agenticDensity === 'number' && Number.isFinite(agenticDensity) && agenticDensity > 0) {
setAgenticCharsPerToken(agenticDensity);
}
// Auto-detect the currently active model and get its ID for accurate token counting
let activeModelId = '';
try {
const llmClient = ctl.client?.llm;
if (llmClient && typeof llmClient.model === 'function') {
// Get the active model to extract its ID
const activeModel = await llmClient.model();
if (activeModel) {
const typedModel = activeModel as LLMModelWithOptionalFields;
activeModelId = typedModel.modelKey || typedModel.id || '';
console.log(`[ContextGuard] ✅ Auto-detected active model: ${activeModelId}`);
// ACTIVE-CHAT-MODEL FALLBACK (09.11, owner GO): publish the auto-detected chat-model id so the wrapper-side
// mid-loop saves (TIER-2 / M3 in toolsProvider) can fall back to it when contextGuardSummaryModel is empty —
// their ToolCallContext exposes no model hop. Publish ONLY here: the configured-summaryModel branches below
// resolve inside generateSessionContinuity() itself, and an undetected turn must not erase a previous id.
TokenStatsManager.setActiveChatModelId(activeModelId);
// Update token limit from SDK
await contextGuard.setTokenLimitFromModel(activeModelId);
} else {
const configuredSummaryModel = pluginConfig.get('contextGuardSummaryModel') || '';
if (configuredSummaryModel) {
activeModelId = configuredSummaryModel;
await contextGuard.setTokenLimitFromModel(activeModelId);
console.log(`[ContextGuard] ✅ Using configured summaryModel: ${activeModelId}`);
} else {
console.warn('[ContextGuard] No active model detected. Using default tokenLimit.');
}
}
}
} catch {
const configuredSummaryModel = pluginConfig.get('contextGuardSummaryModel') || '';
if (configuredSummaryModel) {
activeModelId = configuredSummaryModel;
console.warn(`[ContextGuard] ⚠️ Auto-detection failed, using summaryModel: ${activeModelId}`);
} else {
console.warn('[ContextGuard] ⚠️ No model detected. Using default tokenLimit.');
}
}
// Extract System Prompt from history (usually index 0) to ensure accurate budget tracking
let extractedSystemPrompt: string | undefined;
if (Array.isArray(messages) && messages.length > 0 && typeof messages[0] === 'object') {
const firstMsg = messages[0] as ExtendedMessage & { content?: unknown };
if ((firstMsg.role === 'system' || firstMsg.role === 'system_prompt') && firstMsg.content) {
let sysContent: string | undefined;
if (typeof firstMsg.content === 'string') {
sysContent = firstMsg.content;
} else if (Array.isArray(firstMsg.content)) {
const contentArr = firstMsg.content as Array<{ type?: string; text?: string }>;
sysContent = contentArr.filter((c) => c.type === 'text').map((c) => c.text ?? '').join(' ');
}
if (sysContent && sysContent.trim().length > 0) extractedSystemPrompt = sysContent;
}
}
// Calculate tokens for threshold check using actual model limit & the shared agentic-density estimate (estimateTokensFromChars)
const safeMessages = messages ?? [];
// 🔹 FIX #21 (09.10): publish this turn's live message list with the token baseline — re-published below when
// a same-turn compression replaces it, so the finally point always sees the post-compression array.
publishedTurnMessages = safeMessages;
console.log('[ContextGuard DEBUG] Step 6: calling countTokens()...');
try {
tokenCount = await contextGuard.countTokens(safeMessages, imageCount, activeModelId || undefined, extractedSystemPrompt, historyTextLength);
console.log('[ContextGuard DEBUG] Step 7: countTokens() succeeded, getting limits...');
} catch (countError) {
console.error('[ContextGuard DEBUG] STEP 6 FAILED:', countError);
throw countError; // Re-throw to see full stack trace
}
// 🔹 FIX #3 (04.10): assignment into the function-scoped `maxTokens` declared at the top — the
// finally-publish must read a value that survives throws landing before this line (it stays 0 then,
// i.e. the same safe no-op state as never publishing).
maxTokens = contextGuard.getTokenLimit();
const threshold = contextGuard.getThreshold();
// ✅ DIAGNOSTIC: Log accurate token counts against real model limits
console.log(`✅ [TokenCheck] Model limit: ${maxTokens} | Tokens used: ${tokenCount} | Threshold: ${Math.round(threshold)}`);
console.log('[ContextGuard DEBUG] Step 8: checking auto-tracker config...');
// 🔹 WIRE UP AUTO-TRACKER THRESHOLD PROMPT — Step 0.5b (uses correct maxTokens)
const autoTrackingConfig = ctl.getGlobalPluginConfig(configSchematics);
if ((autoTrackingConfig.get('autoTrackingEnabled') ?? true) && maxTokens > 0) {
console.log('[ContextGuard DEBUG] Step 9: calling checkAndGeneratePrompt()...');
try {
// i18n-CONFIRM arc (06.10): the checkpoint question is DISPLAYED in the user's configured
// language ("🌐 Language" menu); reply parsing stays multilingual inside Step 0.6 below.
// UNIFIED LANGUAGE v2 (step 4): S5 runtime texts ride along so the threshold warning renders in this turn's language.
const promptResult = autoTracker.checkAndGeneratePrompt(
tokenCount,
maxTokens,
getConfirmationWords(autoTrackingConfig.get('language')),
rt,
);
if (promptResult.triggered && promptResult.warning) {
pendingWarning = promptResult.warning;
console.log(`[AutoTracker] ✅ THRESHOLD PROMPT GENERATED — user will be asked to save session memory`);
}
} catch (autoTrackError) {
console.error('[ContextGuard DEBUG] STEP 9 FAILED:', autoTrackError);
throw autoTrackError;
}
}
// Capture message count for later use in Step 0.6
messageCount = history?.getLength() ?? 0;
// 🔹 F1 (17.09 G-B fix): one-shot notice text produced inside PART B below; appended to userPrompt after the
// compression block (see there). Lives only in this preprocess() run → never re-injected on later turns.
let checkpointSavedNotice = '';
if (tokenCount > threshold) {
console.log(`[ContextGuard] Token count ${tokenCount} exceeds compression threshold ${threshold}, compressing...`);
// 🔹 C compaction family (24.09) STEP 3 — PRE-SUMMARIZATION PRUNING: prune oversized tool payloads
// from the message array BEFORE any summarization work starts. Why before, not after: (a)
// compressHistory() ships every pre-keepLast message verbatim to the summary model — one giant
// read_file/web_fetch payload can push THAT prompt past the summary model's own context window and
// force the documented fallback path ("Original content unavailable") that loses all detail; (b)
// keepLast messages survive verbatim in the rebuilt history, so an unpruned giant result would sit
// in live context forever and re-fire this threshold almost immediately. Nothing is lost: every
// pruned payload is stored verbatim under its SHA-256 digest in .ai_toolbox/compaction/, and the
// preview embeds a retrieval hint + opaque locator (C step 2). STORE-BEFORE-PRUNE ordering: if any
// store fails, pruning for this turn is skipped entirely — history keeps full payloads exactly as
// it does today (safe degradation; orphaned already-stored files are idempotency-safe).
if (pluginConfig.get('compactionEnabled') !== false) { // default ON; explicit false opts out entirely
try {
const cfgMaxBytes = pluginConfig.get('compactionMaxResultBytes');
const maxBytes = typeof cfgMaxBytes === 'number' && Number.isFinite(cfgMaxBytes) && cfgMaxBytes > 0
? Math.floor(cfgMaxBytes)
: DEFAULT_MAX_BYTES_PER_RESULT;
const refs = collectPrunedPayloadRefs(safeMessages, maxBytes);
if (refs.length > 0) {
for (const ref of refs) {
// Exclusive 'wx' create + 0o600: never overwrites, refuses symlink targets at the path.
await storePrunedPayload(getWorkingDir(), ref);
}
const pruneOutcome = pruneOversizedToolPayloads(
safeMessages,
{ maxBytesPerResult: maxBytes, digests: refs.map((r) => r.digestHex) },
);
if (pruneOutcome.prunedCount > 0) {
console.log(
`[C-compact] Pruned ${pruneOutcome.prunedCount} oversized tool payload(s) before summarization — ` +
`~${(pruneOutcome.bytesSaved / 1024).toFixed(1)} KiB out of context; verbatim copies in .ai_toolbox/compaction/`,
);
} else {
console.warn(`[C-compact] FAIL-LOUD: ${refs.length} payload(s) collected but NONE pruned (skipped=${pruneOutcome.skippedCount}) — history kept verbatim this turn`);
}
}
} catch (cErr) {
// Any store failure aborts pruning for THIS TURN (the prune pass only runs after the full store loop).
console.error(`[C-compact] Prune-before-summarization aborted (non-fatal — compressing with verbatim history):`, cErr);
}
}
// 🔹 PART B — PRE-COMPRESSION CHECKPOINT (awaited BEFORE history is destroyed). Non-fatal by design:
// any failure only logs; compression must still run (parity with the Part A / YES-reply save paths).
let snapshotSaved = false;
let snapshotSessionId: string | undefined; // 🔹 F1 — id for the one-shot user-visible notice below
try {
const saveResult = await autoTracker.autoSaveSessionMemory(tokenCount, maxTokens, messageCount);
snapshotSaved = !!saveResult?.saved;
if (snapshotSaved) {
snapshotSessionId = saveResult.sessionId; // 🔹 F1
console.log(`[AutoTracker] ✅ Pre-compression checkpoint saved: ${saveResult.sessionId}`);
} else {
console.warn('[AutoTracker] ⚠️ Pre-compression checkpoint NOT saved — compressing without a clean checkpoint');
}
} catch (e) {
console.warn(`[AutoTracker] ⚠️ Pre-compression snapshot failed (non-fatal): ${e instanceof Error ? e.message : String(e)}`);
}
// 🔹 F1 (17.09 G-B fix): the checkpoint SAVED while a warning was live in this same run — the old flow
// consumed that warning below with zero user-visible acknowledgement. Queue exactly one notice for this
// turn's model input (consumed by PART B-2 right after, so hasPendingWarning() here is the pre-consume state).
const f1Notice = buildCheckpointSavedNotice({
snapshotSaved,
warningWasLive: pendingWarning !== undefined || autoTracker.hasPendingWarning(),
usagePercent: maxTokens > 0 ? (tokenCount / maxTokens) * 100 : null,
sessionId: snapshotSessionId,
});
if (f1Notice) {
checkpointSavedNotice = f1Notice;
}
// 🔹 PART B-2: settle any pending YES/NO prompt — the context it referred to is about to be replaced, so a
// re-injected <SYSTEM_INSTRUCTION> would demand a reply (and a second save) for history that no longer exists.
if (!snapshotSaved && autoTracker.hasPendingWarning()) {
lastChanceNote = true;
}
if (autoTracker.hasPendingWarning()) {
autoTracker.consumePendingConfirmation(); // clear the tracker-side prompt; FSM no longer reports it pending
pendingWarning = undefined; // drop this turn's locally-captured copy too — the suffix builder must NOT re-inject a YES/NO request about history that is being replaced now
}
// 💾 Arc C (07.10) — FORCED STRUCTURED SESSION-MEMORY SAVE at the compression trigger, awaited BEFORE
// compressHistory() destroys history (same ordering guarantee as PART B's telemetry snapshot above).
// STRICT `=== true` gate on purpose: production ParsedConfig applies the zod default(true), while pinned
// harness stubs return undefined for unknown keys → the arc-C path is fully skipped there. The typeof guard
// additionally protects legacy/stub ContextGuard objects lacking generateSessionContinuity; and
// safeMessages.length > 10 guarantees a non-empty summarizable span past the keepLast=10 exclusion below.
// Non-fatal by design: any failure only logs — compression must still run and PART B's telemetry snapshot
// + F1 notice stay byte-identical (pinned by tests/precompressSnapshot.test.ts, which never reach this block).
if (pluginConfig.get('contextGuardForceSummaryOnCompress') === true
&& typeof contextGuard.generateSessionContinuity === 'function'
&& safeMessages.length > 10) {
try {
// Flush buffered auto-tracked actions FIRST: they land in the persistent store before the
// structured summary is generated from this turn's message span.
await autoTracker.flushActionsToMemory();
// keepLast=10 exclusion on the CALLER side — mirrors compressHistory's slice(0, -keepLast): the last
// 10 messages survive compression verbatim and are deliberately NOT summarized here.
const toSummarize = safeMessages.slice(0, safeMessages.length - 10);
// contextGuard is narrowed to the concrete class here (the enclosing `if (contextGuard)` scope);
// generateSessionContinuity is a regular class method — no cast/assertion needed. The typeof check in
// the gate above remains the runtime guard for legacy/stub instances lacking the method.
const summary = await contextGuard.generateSessionContinuity(toSummarize, {
fallbackModelId: activeModelId || undefined,
abortSignal: ctl.abortSignal,
});
if (summary) {
const persistOutcome = await persistGeneratedSessionSummary(summary);
if (persistOutcome.saved) {
console.log('[Arc C] ✅ Structured session-continuity summary persisted before compression');
} else {
// Telemetry fallback — deterministic, non-fatal: the summary was generated but NOT durable.
console.warn(`[Arc C] ⚠️ Continuity summary generated but persistence failed (${persistOutcome.error ?? 'unknown'}) — telemetry only`);
}
} else {
// No parsable model output (or no usable model) — deterministic telemetry fallback, compression proceeds.
// 15.10 owner report: warn→log reroute so this by-design notice stops rendering as an ERROR-level line in
// the host log file (stderr=error-level mapping; byte-identical text, ⚠️ marker stays in-text). Pairs with
// the matching C2a gate-skip notice in contextGuard.generateSessionContinuity().
console.log('[Arc C] ⚠️ No structured continuity summary available — telemetry fallback applies');
}
} catch (arcCErr) {
console.warn(`[Arc C] ⚠️ Forced session-memory save failed (non-fatal): ${arcCErr instanceof Error ? arcCErr.message : String(arcCErr)}`);
}
}
// Forward the host abort signal so an in-flight summarization prediction stops when the user
// cancels this turn (same ctl.abortSignal already used for embedding/retrieval below).
const compressedMessages = await contextGuard.compressHistory(safeMessages, ctl.abortSignal) as unknown as ChatMessage[];
// Clear history by popping all messages
while ((history?.getLength() ?? 0) > 0) {
history.pop();
}
compressedMessages.forEach((msg: ChatMessage) => history.append(msg));
contextGuard.resetTokenCache();
// 🔹 FIX #21 (09.10): re-publish the POST-compression array — the wrapper-side forced save must never see
// messages that compression just destroyed (the pre-compression safeMessages is dead at this point).
publishedTurnMessages = compressedMessages as unknown as { role?: string; content?: unknown; [key: string]: unknown }[];
// 🔹 FIX-1A (29.10, owner GO — supersedes FIX #20 A2): the pop/append above mutates ONLY the pullHistory()
// COPY — SDK contract (ProcessingController.pullHistory docblock): "The returned history is a copy, so
// mutating it will NOT affect the actual history." Counting the compressed array here therefore published a
// baseline for what is NOT on the wire: 29.10 incident L13057 logged "Post-compression count: 1440 tokens"
// while LM Studio rejected the real 167,060-token request in the same second (L13078) — the mid-loop guard
// then measured tool-payload growth against ~1.4k and could never arm inside that turn. FIX-1A re-pulls a
// FRESH copy of the ACTUAL host history and counts it with the SAME shared agentic-density estimator used at
// turn start (Step 0.5 pattern: native text + serialized tool-call requests/results char sums, plus the
// tool-definition overhead that rides every request) → the published baseline reflects reality again, so the
// mid-loop guard (FIX B, 95% pre-emptive gate) evaluates true usage after compression.
try {
const freshHistory = await ctl.pullHistory();
let freshChars = 0;
if (freshHistory) {
const freshLen = freshHistory.getLength();
for (let i = 0; i < freshLen; i++) {
// Structural cast to the exact surface this code touches (29.10 gate feedback: no-unnecessary-type-assertion +
// no-explicit-any + unsafe-* warnings on any-casts) — one typed view, zero `any`.
const fmView = freshHistory.at(i) as unknown as { getText?: () => string; getToolCallRequests?: () => unknown; getToolCallResults?: () => unknown };
let fText = '';
if (typeof fmView.getText === 'function') fText += (fmView.getText() || '');
try { if (typeof fmView.getToolCallRequests === 'function') fText += JSON.stringify(fmView.getToolCallRequests()) || ''; } catch { /* non-fatal — count what we have */ }
try { if (typeof fmView.getToolCallResults === 'function') fText += JSON.stringify(fmView.getToolCallResults()) || ''; } catch { /* non-fatal — count what we have */ }
freshChars += fText.length;
}
}
const freshOverhead = getToolOverheadChars(); // parity with Step 0.5: serialized tool defs ride on every request
if (freshOverhead > 0) freshChars += freshOverhead;
// Image parity with Step 0.5's primary path (countTokens adds imageCount*500 when historyTextLength is set).
// 29.10 gate feedback (publishHardening 'healthy turn' case): a fresh recount with ZERO counted chars is UNUSABLE —
// that happens only in hermetic stubs (empty fake history + mocked countTokens, e.g. tests/publishHardening.test.ts),
// never in production (compression fires above threshold ⇒ the host history has real messages). Overwriting a valid
// STEP-6 baseline with 0 would break the pinned publish contract AND blind the mid-loop guard — so adopt the fresh
// count only when it measured something, otherwise keep the pre-compression baseline (same conservative direction as the catch path).
const freshCount = Math.round(estimateTokensFromChars(freshChars)) + imageCount * 500;
if (freshChars > 0) {
tokenCount = freshCount; // also makes the Step 0.6 YES-reply checkpoint report accurate numbers
console.log(`[ContextGuard] [FIX-1A] Post-compression recount (FRESH pullHistory — host history is immutable to plugins): ${freshChars.toLocaleString('en-US')} chars → ${tokenCount.toLocaleString('en-US')} tokens (TRUE baseline for the mid-loop guard; compressed-copy count intentionally NOT used)`);
} else {
console.log(`[ContextGuard] [FIX-1A] Fresh post-compression recount unusable (0 chars counted — hermetic/empty history) — keeping pre-compression baseline ${tokenCount.toLocaleString('en-US')} tokens`);
}
} catch (countErr) {
// Non-fatal: keep the PRE-compression tokenCount — stale-HIGH is the conservative direction (the guard may
// snapshot early), whereas the old compressed-copy fallback was stale-LOW and blinded the guard entirely.
console.warn('[FIX-1A] Fresh post-compression recount failed; keeping pre-compression baseline (conservative):', countErr);
}
// 🔹 FIX-1B (29.10, owner GO — truthful unrecoverable-window notice): if the FRESH host-history recount is
// still at/above the compression threshold, in-band recovery is IMPOSSIBLE — plugins cannot mutate LM
// Studio's conversation history (SDK pullHistory() returns a copy; no rewrite endpoint exists in any of the
// 36 port RPCs; verified identical in SDK 2.1.0). Retrying would re-send the same oversized payload
// (29.10: ~15 consecutive send_error rejections, payload growing per retry) — so state that fact ONCE to
// the user instead of implying a reset happened. F1 pattern: appended straight onto userPrompt, one-shot
// (dies with this preprocess() run), independent of checkpointSuffix's hasPendingWarning()-gated paths.
if (tokenCount >= contextGuard.getThreshold()) {
const unrecoverableNotice =
'\n\n⚠️ CONTEXT WINDOW UNRECOVERABLE THIS CHAT: compression ran, but the host conversation history cannot be ' +
`modified from a plugin — LM Studio is still sending ~${Math.round(tokenCount / 1000)}k tokens against the ${Math.round(contextGuard.getTokenLimit() / 1000)}k window. ` +
'Starting a NEW chat is the only way to reset the context; everything important was checkpointed to session memory before compression.';
userPrompt += unrecoverableNotice;
console.log(`[ContextGuard] [FIX-1B] Unrecoverable-window notice appended (post-compression recount ${tokenCount.toLocaleString('en-US')} ≥ threshold ${Math.round(contextGuard.getThreshold()).toLocaleString('en-US')}) — one-shot`);
// 🔹 LATCH SUB-ITEM (30.10, owner GO — FIX #1 Part A guard): the fresh recount just PROVED this chat's window
// unrecoverable in-band — arm the TokenStatsManager latch so the wrapper-side mid-loop gates (TIER-2 forced
// save + FIX B pre-emptive compression) STOP re-firing on every model step of this and later turns. Without it,
// each per-step toolsProvider() entry resets both wrapper latches and re-runs prune → snapshot → Arc-C
// regeneration (up-to-60s summarization prediction) → compressHistory over a history that cannot shrink — pure
// no-op spend per step. Idempotent: later boundary recounts in the same chat re-affirm; a below-threshold
// recount clears it (else branch below); chat reset (TokenStatsManager.clear()) is the other clear path.
TokenStatsManager.setUnrecoverableWindow(tokenCount, contextGuard.getTokenLimit());
} else {
// 🔹 LATCH SUB-ITEM (30.10): post-compression fresh recount landed BELOW threshold — in-band recovery IS now
// possible (only legitimate source: a larger effective window, e.g. model switch between boundary runs). Clear
// the latch so the mid-loop gates re-arm for this turn; no-op with no log when already clear.
TokenStatsManager.clearUnrecoverableWindow();
}
}
// 🔹 F1 (17.09 G-B fix): surface the saved checkpoint to this turn's model input. Deliberately appended
// straight onto userPrompt rather than routed through checkpointSuffix — several return paths gate that suffix
// on hasPendingWarning() (false here by construction: PART B-2 consumed it above), which would silently drop
// this notice exactly like the original G-B bug. One-shot: the variable dies with this preprocess() run.
if (checkpointSavedNotice) {
userPrompt += checkpointSavedNotice;
console.log("[AutoTracker] ✅ Checkpoint-saved notice appended to this turn's model input (F1 one-shot)");
}
// 🔹 FIX-A (09.10, owner GO): surface a mid-loop threshold crossing from a PREVIOUS turn the same way — F1-style
// straight onto userPrompt (NOT via checkpointSuffix: return paths gate that suffix on hasPendingWarning(), which
// is false here by construction whenever this notice exists — guardMidLoopThreshold deliberately does not touch the
// FSM). takePendingMidLoopNotice() is one-shot INSIDE autoTracker; a throw between its render and the append below
// consumes exactly like the F1 variable (dies with this run) — same loss semantics as pre-#20, no double-injection.
// Independent of checkpointSavedNotice: both can be live in the same run (a crossing armed earlier mid-loop plus a
// checkpoint saved at compression now), and the model gets one short acknowledgement instruction per fact.
const midLoopNotice = autoTracker.takePendingMidLoopNotice();
if (midLoopNotice) {
userPrompt += midLoopNotice;
console.log("[AutoTracker] ✅ Mid-loop crossing notice appended to this turn's model input (FIX-A one-shot)");
}
// 🔹 UNIFIED LANGUAGE v2 (step 4, owner GO): per-turn answer-language instruction — F1-style straight onto
// userPrompt (NOT via checkpointSuffix: its hasPendingWarning()-gated return paths drop notices). Appended once
// per turn after the FIX-A block; rt is guaranteed non-empty for every shipped locale.
const answerLine = fillTemplate(rt.answerLanguageLine, { language: LANGUAGE_DISPLAY_NAMES[turnLang] });
if (answerLine) {
userPrompt += '\n\n' + answerLine;
console.log('[UL-v2] Answer-language line appended (' + turnLang + ')');
}
// 🔹 UPDATE-ANNOUNCE (19.10, owner GO): users with OS/desktop notifications disabled never see the toast — every turn still passes
// through preprocess(), so a pending update is announced IN CHAT here instead: the model tells them and gives them the clickable link.
// F1-style straight onto userPrompt (the checkpointSuffix route would be silently dropped by its hasPendingWarning()-gated return
// paths). getPendingUpdateAnnouncement() latches once per process lifetime per version inside updateChecker → at most one short line
// (~25 tokens) until plugin reload or a newer release; dismissed versions are honored here too (zero network — cached state only).
// SPEC A XOR (21.10): pass the LIVE panel toggle so channel selection reaches the accessor — an explicit OFF makes in-chat THE
// channel (announces even a previously-toasted version, once per process); undefined/ON keeps pre-fix mutual exclusion byte-exactly.
const updateNotice = getPendingUpdateAnnouncement(pluginConfig.get('notificationsEnabled'));
if (updateNotice) {
userPrompt += `\n[AI Toolbox] New version available: v${updateNotice.version} (installed ${LOCAL_VERSION}). Mention this to the user once and give them this link: ${updateNotice.url ?? 'https://github.com/crunch3r/ai_toolbox/releases'}`;
console.log(`[AI Toolbox] [UPDATE-CHECK] In-chat announcement appended to this turn's model input (once per process for v${updateNotice.version})`);
}
} catch (e) {
console.error('[ContextGuard] Auto-compression failed:', e);
} finally {
// 🔹 FIX #20 (A2): single publish point — AFTER any same-turn compression. The tool wrapper's mid-loop
// guard measures growth against this turn-start baseline + model limit until the next preprocess().
// 🔹 FIX #3 (04.10, CTX-FOOTER arc part 3 — publish hardening): moved into FINALLY so a throw between
// resetMidLoopDelta() above and here can no longer skip it: STEP 6 countTokens re-throw, STEP 9
// auto-tracker re-throw, prune/checkpoint/compress/recount failures. Before this fix such a turn ran
// its ENTIRE tool loop with baseline === 0 — mid-loop guard + every ctx_footer silently no-op (3.10
// incident: zero [CTX-FOOTER] lines; the wrapper's one-shot suppression warn names that window).
// finally also covers the early return at "pullHistory() returned undefined". Zero compression behavior
// change: this publishes exactly what the try body computed, and when counting itself failed both values
// are 0 — byte-equivalent state to the old skip (documented in resetMidLoopDelta()).
TokenStatsManager.setTurnEvaluation(tokenCount, maxTokens);
// 🔹 FIX #21 (09.10): same single publish point — the turn's live message list for the wrapper-side forced
// 90% save. null on any throw before its first assignment = safe no-op for that turn (documented above).
TokenStatsManager.setTurnMessages(publishedTurnMessages);
}
}
// 🔹 UNIFIED CHECKPOINT WARNING INJECTION (Fixes missing prompt when RAG disabled or no files found)
// CRITICAL FIX: Previous approach appended plain text trailing content — LLMs reliably ignored it.
// New format uses explicit <SYSTEM_INSTRUCTION> XML-style markers + imperative directives that LLMs process as role-level instructions.
let checkpointSuffix = '';
if (pendingWarning) {
// UNIFIED LANGUAGE v2 (step 4, 27.10): wrapper wording from the per-turn RuntimeTexts (en render byte-identical);
// `<SYSTEM_INSTRUCTION>` markers + '---' separator stay literal here by design — they are structure, not text.
checkpointSuffix = `\n\n<SYSTEM_INSTRUCTION>\n` +
`${rt.checkpointWarningHeader}\n` +
`${pendingWarning}\n` +
`---\n` +
`${rt.checkpointActionRequiredLead}\n` +
// i18n-CONFIRM arc (06.10): wording is language-agnostic on purpose — the parser accepts every shipped
// confirmation word, so reference the options above instead of duplicating one language's words here.
`${rt.checkpointIfConfirmedLine}\n` +
`${rt.checkpointIfDeclinedLine}\n` +
`${rt.checkpointDoNotProceedLine}\n` +
`</SYSTEM_INSTRUCTION>`;
pendingWarning = undefined; // Clear to prevent duplication in Step 1/2
}
// 🔹 PART B-2 (last chance): the pre-compression snapshot did NOT save, but a YES/NO prompt was still live —
// tell the user this turn is the last clean opportunity before history is compressed. (One-shot: consumed now.)
if (lastChanceNote) {
checkpointSuffix += '\n\n⚠️ LAST chance before history is compressed — no session-memory checkpoint could be saved automatically this turn.';
lastChanceNote = false;
}
// Step 0.6: Auto-tracking analysis + checkpoint reply handling
try {
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const autoTrackingEnabled = pluginConfig.get('autoTrackingEnabled') ?? true; // Default to ON
if (autoTrackingEnabled) {
// Update tracker config from plugin settings (also includes token threshold)
autoTracker.updateConfig({
autoTrackingEnabled: true,
autoTrackDecisions: pluginConfig.get('autoTrackDecisions') ?? true,
autoTrackCompletions: pluginConfig.get('autoTrackCompletions') ?? true,
autoTrackErrors: pluginConfig.get('autoTrackErrors') ?? true,
autoSummaryInterval: pluginConfig.get('autoSummaryInterval') ?? 50,
autoTrackTokenThreshold: (pluginConfig.get('autoTrackTokenThreshold')) ?? 75,
});
// 🔹 CHECK FOR YES/NO (or German JA/NEIN) REPLY TO PENDING CHECKPOINT PROMPT — Fix B
if (autoTracker.hasPendingWarning()) {
const replyRaw = userPrompt.trim().toUpperCase();
// Normalize JA/NEIN onto the canonical 'YES' | 'NO' FSM inputs; null = not a confirmation reply.
const replyMatch = normalizeConfirmationReply(userPrompt);
if (replyMatch === 'YES' || replyMatch === 'NO') {
checkpointConsumedThisTurn = true; // One reply serves one prompt — Step 0.7 must not reuse it as a switch confirmation.
console.log(`[AutoTracker] ✅ User replied '${replyRaw}' to checkpoint prompt — processing...`);
autoTracker.processUserReply(replyMatch); // Canonical 'YES' | 'NO' (JA/NEIN normalized above)
// If YES → flush buffered actions + save session memory now
if (replyMatch === 'YES') {
const maxTokens = contextGuard?.getTokenLimit() ?? 0;
// 🔹 FIX: Use direct checkpoint path instead of checkAndSaveTokenThreshold()
// which is designed for independent threshold triggers, not post-confirmation flow.
console.log(`[AutoTracker DEBUG] Direct save path triggered after user confirmation`);
// Flush buffered actions first
const flushedCount = await autoTracker.flushActionsToMemory();
if (flushedCount > 0) {
console.log(`[AutoTracker] ✅ Flushed ${flushedCount} buffered action(s)`);
}
// Then save session memory directly
const saveResult = await autoTracker.autoSaveSessionMemory(
tokenCount,
maxTokens,
messageCount
);
if (saveResult.saved) {
console.log(`[AutoTracker] ✅ Session memory checkpoint saved: ${saveResult.sessionId}`);
} else {
console.error('[AutoTracker] ❌ Checkpoint save failed');
}
}
console.log(`[AutoTracker] ✅ Checkpoint reply consumed — FSM state: ${autoTracker.getState()}`);
}
}
// Analyze user message for tracking triggers (silent background)
const actions = autoTracker.analyzeMessage?.(userPrompt) ?? [];
if (actions.length > 0) {
console.log(`[Auto-Track] Detected ${actions.length} event(s):`, actions.map(a => `${a.type} (${a.confidence.toFixed(2)})`).join(', '));
// 🔹 Actions are now buffered in-memory and will be flushed to persistent storage
// when the token threshold checkpoint fires (with user confirmation)
}
} else {
// Ensure tracker is disabled if config says so
autoTracker.updateConfig({
autoTrackingEnabled: false,
});
}
} catch (e) {
console.error('[Auto-Track] Analysis failed:', e);
}
// Step 0: Always register attachments so tools can access them by name
const allFiles = userMessage.getFiles?.(ctl.client) ?? [];
setAttachments(allFiles);
// Build attachment notice to inject into prompt
let attachmentNotice = '';
if (allFiles.length > 0) {
// UNIFIED LANGUAGE v2 (step 4): title + lead-in from the per-turn RuntimeTexts (en render byte-identical).
const fileNames = listAttachments();
attachmentNotice = `\n\n${rt.attachedFilesTitle}\n${rt.attachedFilesLead}\n${fileNames.map(name => `- ${name}`).join('\n')}`;
}
// Step 0.7: Project keyword detection — check registered projects before path detection
const projectMatch = detectProjectKeyword(userPrompt);
if (projectMatch) {
console.log(`[ProjectAutoDetect] Found registered project "${projectMatch.name}" at ${projectMatch.path}`);
// 🔹 Fix A (v1.9.8+ safety rule in index.ts): NEVER auto-switch on detection alone —
// confirm-first banner; the CWD changes only after an explicit confirmation reply in a later message.
// i18n-CONFIRM arc (06.10): options DISPLAYED in the user's configured language; parsing accepts all shipped words.
const confWords = getConfirmationWords(ctl.getGlobalPluginConfig(configSchematics).get('language'));
const decision = decideProjectSwitch(projectMatch, userPrompt, getWorkingDir(), pendingProjectSwitch);
if (decision.kind === 'skip') {
// Already in the detected project's directory — switching would be a no-op.
if (pendingProjectSwitch && pendingProjectSwitch.name === projectMatch.name) {
console.log(`[ProjectAutoDetect] Already in "${projectMatch.name}" — dropping stale switch offer`);
pendingProjectSwitch = null;
} else {
console.log('[ProjectAutoDetect] Already in target working directory — no switch needed');
}
} else if (decision.kind === 'banner') {
// New detection → inject the confirm-first banner; do NOT change the working directory this turn.
pendingProjectSwitch = { name: projectMatch.name, path: projectMatch.path };
console.log(`[ProjectAutoDetect] Confirmation requested — switch will run only on an explicit YES/JA reply`);
// Include the checkpoint warning only if it is still live after Step 0.6 ran this turn (avoids re-asking a consumed prompt).
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
// UNIFIED LANGUAGE v2 (step 4): banner wording from the per-turn RuntimeTexts; box-drawing separators stay
// literal in this template by design — they sit outside the dict strings.
const projectInjectPrompt = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${rt.projectDetectedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${rt.projectFoundLead}\n\n${fillTemplate(rt.projectNameLabel, { name: projectMatch.name })}\n${fillTemplate(rt.projectPathLabel, { path: projectMatch.path })}\n\n${rt.projectSwitchQuestion}\nReply '${confWords.yes}' to confirm, or '${confWords.no}' to decline.\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\nUser's original message:\n${userPrompt}${attachmentNotice}\n`;
return `${projectInjectPrompt.trim()}${activeSuffix}`.trim() + getTemporalSuffix(ctl);
} else if (decision.kind === 'declined') {
console.log(`[ProjectAutoDetect] Switch to "${projectMatch.name}" declined — keeping current working directory`);
pendingProjectSwitch = null;
// No early return — continue with the normal flow below (Step 1/2).
} else {
// decision.kind === 'execute' → explicit YES/JA confirmation: run the one-shot switch now.
if (checkpointConsumedThisTurn) {
console.log(`[ProjectAutoDetect] Reply already consumed as checkpoint confirmation — keeping pending offer for "${projectMatch.name}"`);
// Continue with the normal flow below; re-offer on a later message.
} else {
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
pendingProjectSwitch = null;
console.log(`[ProjectAutoDetect] User confirmed — executing one-shot switch to: ${projectMatch.path}`);
return await executeProjectSwitch(projectMatch.path, userPrompt, attachmentNotice, activeSuffix, ctl, rt);
}
}
} else if (pendingProjectSwitch) {
// No fresh keyword match this turn — a pending offer can only be answered by an explicit reply.
const normalized = normalizeConfirmationReply(userPrompt);
if (checkpointConsumedThisTurn && (normalized === 'YES' || normalized === 'NO')) {
console.log(`[ProjectAutoDetect] Reply already consumed as checkpoint confirmation — keeping pending offer for "${pendingProjectSwitch.name}"`);
// Continue with the normal flow below; re-offer on a later message.
} else if (normalized === 'YES') {
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
const target = pendingProjectSwitch;
pendingProjectSwitch = null;
console.log(`[ProjectAutoDetect] User confirmed — executing one-shot switch to: ${target.path}`);
return await executeProjectSwitch(target.path, userPrompt, attachmentNotice, activeSuffix, ctl, rt);
} else if (normalized === 'NO' || normalized === 'NEIN') {
console.log(`[ProjectAutoDetect] Switch to "${pendingProjectSwitch.name}" declined — keeping current working directory`);
pendingProjectSwitch = null;
// Continue with the normal flow below.
} else {
// Non-confirmation reply while an offer is pending → expire it (the "next message" contract).
console.log(`[ProjectAutoDetect] Non-confirmation reply — expiring switch offer for "${pendingProjectSwitch.name}"`);
pendingProjectSwitch = null;
}
}
// Step 1: Directory detection (highest priority)
const detectedPath = detectDirectoryPath(userPrompt);
if (detectedPath) {
// i18n-CONFIRM arc (06.10): the directory banner's reply options also follow the configured language.
const dirWords = getConfirmationWords(ctl.getGlobalPluginConfig(configSchematics).get('language'));
let base = injectWorkingDirectoryPrompt(userPrompt + attachmentNotice, detectedPath, dirWords, rt) + checkpointSuffix; // UL-v2 step-4 completion (28.10): per-turn RuntimeTexts — without rt the helper fell back to getRuntimeTexts('en') and this banner stayed English in all other locales; EN render byte-identical (rt === en bundle when turnLang='en', same object as the old fallback)
return base + getTemporalSuffix(ctl);
}
// Step 2: Document RAG processing (if enabled)
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const documentRAGEnabled = pluginConfig.get('documentRAG');
console.log(`[RAG] documentRAG enabled: ${documentRAGEnabled}`);
if (!documentRAGEnabled) {
// If RAG is disabled, just return the message with attachment notice and checkpoint warning
const base = userPrompt + attachmentNotice + checkpointSuffix;
return base + getTemporalSuffix(ctl);
}
const newFiles = allFiles.filter(f => f.type !== 'image');
console.log(`[RAG] Found ${newFiles.length} non-image files`);
if (newFiles.length === 0) {
const base = userPrompt + attachmentNotice + checkpointSuffix;
return base + getTemporalSuffix(ctl);
}
// Separate PDF files from other file types
const pdfFiles = newFiles.filter(f => f.name.toLowerCase().endsWith('.pdf'));
const otherFiles = newFiles.filter(f => !f.name.toLowerCase().endsWith('.pdf'));
console.log(`[RAG] PDFs: ${pdfFiles.length}, Other: ${otherFiles.length}`);
let allResults: RetrievalResult[] = [];
// Process PDFs with custom local pipeline (more reliable for complex layouts)
if (pdfFiles.length > 0) {
try {
const pdfResults = await retrieveFromPdfs(ctl, userPrompt, pdfFiles);
console.log(`[RAG] PDF retrieval returned ${pdfResults.length} results`);
allResults.push(...pdfResults);
} catch (error) {
console.error('[RAG] Error processing PDFs:', error);
}
}
// Process other files with LM Studio's native retrieval API (handles .txt, .md, etc. natively)
if (otherFiles.length > 0) {
try {
const model = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
const result = await ctl.client.files.retrieve(userPrompt, otherFiles, {
embeddingModel: model,
limit: pluginConfig.get('retrievalLimit') || 5,
signal: ctl.abortSignal,
});
// Convert high-level API results to our format
const filteredEntries = (result?.entries || []).filter(
entry => entry.score > (pluginConfig.get('retrievalAffinityThreshold') ?? 0.3)
);
console.log(`[RAG] Native retrieval returned ${filteredEntries.length} results`);
allResults.push(...filteredEntries.map(e => ({ content: e.content, score: e.score })));
} catch (error) {
console.error('[RAG] Error retrieving from other files:', error);
}
}
// Sort and limit results
allResults.sort((a, b) => b.score - a.score);
const retrievalLimit = pluginConfig.get('retrievalLimit') || 5;
allResults = allResults.slice(0, retrievalLimit);
console.log(`[RAG] Total results after sorting: ${allResults.length}`);
// 🔹 Inject checkpoint confirmation prompt if triggered/pending from Step 0.5
let finalMessage = userPrompt + attachmentNotice + checkpointSuffix;
// Inject context if results found
if (allResults.length > 0) {
let contextInjection = '';
for (const result of allResults) {
contextInjection += `\n${result.content}\n---\n`;
}
// UNIFIED LANGUAGE v2 (step 4): RAG section header from the per-turn RuntimeTexts (en render byte-identical).
return `${finalMessage}\n\n${rt.ragContextHeader}\n${contextInjection.trim()}` + getTemporalSuffix(ctl);
}
// If no results found, return original message with attachment notice
console.log('[RAG] No relevant results found');
const base = finalMessage;
return base + getTemporalSuffix(ctl);
}
/**
* Document RAG Prompt Preprocessor + Working Directory Detection + Temporal Awareness
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { type ChatMessage, type FileHandle, type PromptPreprocessorController } from '@lmstudio/sdk';
import { configSchematics } from './config';
import pdfParse from 'pdf-parse';
import type { ContextGuard } from './contextGuard';
import { setAttachments, listAttachments } from './attachmentManager';
// i18n-CONFIRM arc (06.10): confirmation prompts follow the user's configured language ("🌐 Language" menu);
// acceptance stays multilingual via the derived union sets — single source of truth in src/locales/.
import { getConfirmationWords, ALL_CONFIRM_WORDS, ALL_DECLINE_WORDS, getRuntimeTexts, LANGUAGE_DISPLAY_NAMES, fillTemplate } from './locales';
// UNIFIED LANGUAGE v2 (step 4, 27.10): per-turn host/OS language fallback chain — the existing '^\.\/hostLanguage\.js$'
// jest mapper entry resolves this; no new config needed (verified 26.10). Option B: the turn resolver below consumes it here ('message layer').
import { getHostLocale, getOsLocale } from './hostLanguage.js';
// UNIFIED LANGUAGE v2 step 5 (27.10): per-turn state-file sync — extensionless on purpose: ts-jest resolves it via
// moduleFileExtensions and tsup/tsc accept the form ('./locales' precedent), so NO jest mapper entry is required.
import { persistConcretePick } from './uiLabelBake';
import type { LanguageCode, RuntimeTexts } from './locales/types.js'; // type-only — erased at runtime, never jest-resolved (RC#4 class avoided)
/** Shipped locale IDs keyed by their lowercase config form — the concrete values resolveTurnLanguage() accepts. */
const SHIPPED_LANGUAGE_CODES: Readonly<Record<string, LanguageCode>> = {
en: 'en',
de: 'de',
es: 'es',
'zh-cn': 'zh-CN',
'zh-tw': 'zh-TW',
};
/**
* UNIFIED LANGUAGE v2 (step 4): resolve the language for THIS turn's injected runtime texts.
* Chain: (a) a concretely configured language (trimmed/lowercased, matched against the shipped set — ANY
* non-matching value falls through by construction; step 5 will add 'auto', which must keep falling
* through); (b) the LM Studio host preference (getHostLocale() !== ''); (c) the OS locale's first tag
* ('de-DE' → 'de') matched against the same set; (d) English. Pure function of its inputs — exported so the
* fallback chain stays jest-testable without SDK mocks.
*/
export function resolveTurnLanguage(configured?: string | null): LanguageCode {
if (typeof configured === 'string') {
const code = SHIPPED_LANGUAGE_CODES[configured.trim().toLowerCase()];
if (code) return code; // concrete pick wins — anything else (incl. future 'auto') falls through by design
}
const hostLocale = getHostLocale();
if (hostLocale !== '') return hostLocale;
const osTag = getOsLocale().split('-')[0];
return SHIPPED_LANGUAGE_CODES[osTag] ?? 'en';
}
import { autoTracker } from './autoTracker';
import { TokenStatsManager, drainActiveToolTurns, estimateTokensFromChars, setAgenticCharsPerToken } from './tokenStatsManager';
// C compaction family (24.09): pre-summarization pruning of oversized tool payloads (DeepSeek harness item C)
import { DEFAULT_MAX_BYTES_PER_RESULT, pruneOversizedToolPayloads } from './utils/toolPayloadCompaction.js';
import { collectPrunedPayloadRefs, storePrunedPayload } from './utils/toolPayloadStorage.js';
import { getToolOverheadChars } from './toolOverhead.js';
import { getWorkingDir, setWorkingDir, listRegisteredProjects } from './workingDir.js';
// CONTAMINATION-FIX Part A (01.10): identity rebind after Step 0.7 CWD switches. No import cycle: only index.ts
// imports both modules; toolsProvider does not reference promptPreprocessor.
import { getStateManager } from './toolsProvider.js';
// UPDATE-ANNOUNCE (19.10, owner GO): one-shot in-chat update announcement accessor — direct import of the pure state+fetch module is safe here
// (no cycle; toolsProvider already imports it). Jest resolution: this specifier matches the existing '^\.\/updateChecker\.js$' mapper entry (Phase-1), so no new config.
import { getPendingUpdateAnnouncement, LOCAL_VERSION } from './updateChecker.js';
// Arc C (07.10): forced structured session-memory save at the compression trigger — persists via the canonical
// save_session_summary writer path (see src/sessionSummaryPersist.ts); cycle-free edge set (only index.ts imports both sides).
import { persistGeneratedSessionSummary } from './sessionSummaryPersist.js';
// CONTAMINATION-FIX follow-up C / Gap-2 (01.10): static decode import for the post-switch session-memory read in
// executeProjectSwitch(). Replaces a per-call `await import('@msgpack/msgpack')` — dynamic native-import under a
// CJS build is fragile, and this aligns with how every other module in src/ consumes @msgpack/msgpack.
import { decode as decodeMsgpack } from '@msgpack/msgpack';
interface ExtendedMessage {
role?: string;
content?: unknown;
toolCalls?: unknown[];
tool_calls?: unknown[];
files?: unknown[];
images?: unknown[];
}
/** Minimal interface for LM Studio LLM model objects with optional metadata */
interface LLMModelWithOptionalFields {
modelKey?: string;
id?: string;
}
// 🔹 F1 (17.09 G-B fix): PURE decision/formatting helper for the one-shot "auto-checkpoint saved before
// compression" notice. Extracted out of preprocess() PART B so it is jest-testable without SDK mocks.
// Origin: 17.09 log verdict (memory_1789659390790) — the threshold prompt generated at 17:16:31 was consumed
// by consumePendingConfirmation() in the same run with ZERO user-visible acknowledgement.
export interface CheckpointNoticeInput {
/** The pre-compression snapshot save succeeded this turn. */
snapshotSaved: boolean;
/** A checkpoint warning was live during the same preprocess run (generated here, or pending from a prior turn). */
warningWasLive: boolean;
/** Pre-compression usage in percent of the model context window (tokenCount/maxTokens*100); null when unknown. */
usagePercent: number | null;
/** Saved snapshot id, when available. */
sessionId?: string;
}
export function buildCheckpointSavedNotice(input: CheckpointNoticeInput): string | null {
if (!input.snapshotSaved || !input.warningWasLive) {
return null; // warning path (PART B-2 last-chance note) or plain compression — unchanged behavior
}
const pct = typeof input.usagePercent === 'number' && Number.isFinite(input.usagePercent)
? ` at ~${(Math.round(input.usagePercent * 10) / 10)}% of context` // one-decimal rounding (matches toFixed(1) percent convention elsewhere); outer Math.round(x*10/10) was a no-op that printed integers
: '';
const idPart = input.sessionId ? ` (checkpoint ${input.sessionId})` : '';
return '\n\n✅ AUTO-CHECKPOINT SAVED BEFORE COMPRESSION (one-shot notice — not a prompt):\n' +
'A machine telemetry record (auto-written; not a manually saved session memory) was stored' + pct + idPart + ' in place of the token-limit warning for this turn — the history that prompt referred to has been compressed.\n' +
"Briefly acknowledge to the user that the auto checkpoint ran before compression; no YES/NO reply or further action is required.";
}
// --- Temporal Awareness Helpers (merged from up_to_date) ---
interface DateTimeCache {
compact: string;
full: string;
}
let cachedDateTimeData: DateTimeCache | null = null;
const CACHE_DURATION_MS = 5 * 60 * 1000; // Refresh every 5 minutes
// ContextGuard integration
let contextGuard: ContextGuard | null = null;
export function setContextGuard(guard: ContextGuard | null): void {
contextGuard = guard;
}
/**
* FIX #21 (09.10, two-tier mid-loop save) — null-safe read accessor for the active ContextGuard instance.
* setContextGuard() stays the ONE-WAY injection point (index.ts constructs once at startup; test harnesses swap
* fakes/reset to null between suites). This getter is the first external READ path: the toolsProvider wrapper
* consumes it to reach generateSessionContinuity()/getThreshold(). Consumers MUST handle null (guard not yet
* constructed or detached) and keep their own typeof method-guards for stub instances — same defensive contract
* as Arc C's `typeof contextGuard.generateSessionContinuity === 'function'` gate.
*/
export function getContextGetter(): ContextGuard | null {
return contextGuard;
}
let cacheTimestamp = 0;
function getCachedDateTime(): DateTimeCache {
const now = Date.now();
if (cachedDateTimeData && (now - cacheTimestamp) < CACHE_DURATION_MS) {
return cachedDateTimeData;
}
const date = new Date();
// Compact format: DD.MM.YYYY, HH:mm
const compact = date.toLocaleString('de-DE', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit'
});
// Full format: Wochentag, DD. MMMM YYYY, HH:mm Uhr
const full = date.toLocaleString('de-DE', {
weekday: 'long',
year: 'numeric',
month: 'long',
day: 'numeric',
hour: '2-digit',
minute: '2-digit'
}) + ' Uhr';
cachedDateTimeData = { compact, full };
cacheTimestamp = now;
return cachedDateTimeData;
}
function getTemporalSuffix(ctl: PromptPreprocessorController): string {
// Option A (02.11): ALL read sites in this file use global scope — settings persist across chats and app restarts.
const config = ctl.getGlobalPluginConfig(configSchematics);
// Use .get() method with proper defaults - more reliable than direct property access
const temporalAwarenessEnabled = config.get('temporalAwareness') ?? true;
if (!temporalAwarenessEnabled) {
return '';
}
const style = config.get('dateFormatStyle') ?? 'standard';
const { compact, full } = getCachedDateTime();
// DEBUG: Uncomment to verify what's being injected
console.log(`[TEMPORAL] Injecting: ${style === 'heuteIst' ? `HEUTE IST ${full}` : `[Zeit: ${compact}]`}`);
if (style === 'heuteIst') {
return `\n\nHEUTE IST ${full}`;
}
return `\n\n[Zeit: ${compact}]`;
}
/** Normalizes a project name for fuzzy matching (hyphens ↔ underscores, lowercase) */
function normalizeProjectName(name: string): string {
return name.toLowerCase().replace(/[-_\s]+/g, '_');
}
/** Generates variants of a project name for fuzzy matching */
function generateNameVariants(name: string): string[] {
const normalized = normalizeProjectName(name);
const variants = new Set<string>();
variants.add(normalized);
variants.add(normalized.replace(/-/g, '_'));
variants.add(normalized.replace(/_/g, '-'));
variants.add(normalized.replace(/[-_\s]/g, ''));
return Array.from(variants);
}
/** Detect if a user message contains a registered project keyword.
* Registry resolution chain: <pluginRoot>/.session_context/project_registry.json (primary)
* → <pluginRoot>/.session_index.json (legacy fallback).
* The registry is loaded ONCE per invocation and reused across all candidate words
* (previously re-read from disk for every candidate word, and the fallback was never consulted — dead in installed envs). */
export function detectProjectKeyword(
text: string,
options?: { pluginRoot?: string },
): { name: string; path: string } | null {
// Extract candidate words/phrases from the message (ignore common verbs/prepositions)
const stopWords = new Set([
'let', 'us', 'work', 'on', 'the', 'a', 'an', 'in', 'at', 'to', 'for', 'with',
'about', 'start', 'begin', 'open', 'switch', 'go', 'back', 'continue', 'resume'
]);
// FIX: length >= 2 (was > 2) — short name components like 'my' in "my cool project" were dropped,
// so multi-word names containing 2-letter parts could never match. Digits excluded to avoid
// matching version numbers / years against numeric name variants; stopword list still filters the rest.
const words = text.toLowerCase().split(/\s+/).filter(w => w.length >= 2 && !stopWords.has(w) && !/^\d+$/.test(w));
if (words.length === 0) return null;
// Load the registry once per invocation (primary → legacy fallback), then match in memory.
const projects = listRegisteredProjects(options?.pluginRoot);
if (projects.length === 0) return null;
// Precompute fuzzy name variants per project to avoid re-computing them for every candidate word
const indexed = projects.map(p => ({ variants: new Set(generateNameVariants(p.name)), project: p }));
// Try matching combinations of 1-3 consecutive words as project name candidates
for (let len = Math.min(3, words.length); len >= 1; len--) {
for (let i = 0; i <= words.length - len; i++) {
const candidate = words.slice(i, i + len).join('_');
for (const entry of indexed) {
if (entry.variants.has(candidate)) {
return { name: entry.project.name, path: entry.project.path };
}
}
}
}
return null;
}
function detectDirectoryPath(text: string): string | null {
// Remove URLs first to avoid false positives like /medium.com from https://medium.com/...
const withoutUrls = text.replace(/https?:\/\/[^\s]+|www\.[^\s]+|file:\/\/[^\s]+/g, '');
// Windows paths: C:\path or D:\folder (must start with drive letter)
const winMatch = withoutUrls.match(/[A-Za-z]:\\[\w\-_. \\]+/);
// ^^^^^^^^^^
// Backslash added ✓
if (winMatch) return winMatch[0].trim();
// Unix absolute paths: /home/user/dir, /var/log, etc.
const unixMatch = withoutUrls.match(/(?:^|\s)(\/[\w\-_. ]{2,})/);
if (unixMatch) {
const path = unixMatch[1].trim();
// Reject paths that look like URLs or fragments (e.g., / Chat files s)
if (!path.startsWith('/ ') && !path.includes(' ')) {
return path;
}
}
// Relative paths: ./folder, ../parent/dir
// Relative paths: ./folder, ../parent/dir
const relMatch = withoutUrls.match(/(?:^|\s)(?:\.\/|\.\\.\/)[\w\-_. ]+/);
if (relMatch) return relMatch[0].trim();
return null;
}
// 🔹 i18n-CONFIRM arc (06.10): option words follow the user's configured language when passed;
// without them the legacy bilingual wording is preserved byte-for-byte for any other caller/test.
// UNIFIED LANGUAGE v2 (step 4, 27.10): banner texts come from the per-turn RuntimeTexts when passed — the box-drawing
// separators and indentation stay OUTSIDE those strings by design; the 'en' render MUST equal the pre-v2 output byte-for-byte.
function injectWorkingDirectoryPrompt(
originalMessage: string,
detectedPath: string,
words?: { yes: string; no: string },
runtimeTexts?: RuntimeTexts,
): string {
const rt = runtimeTexts ?? getRuntimeTexts('en'); // absent → English bundle (byte-identical to the pre-v2 literals)
const replyHint = words
? fillTemplate(rt.workingDirReplyHint, { yes: words.yes, no: words.no })
: "Reply 'yes' or 'ja' to confirm, or 'no'/'nein' to decline."; // legacy bilingual fallback preserved byte-for-byte
const instruction = `
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirDetected}
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirMentionedLine}
${detectedPath}
${rt.workingDirConfirmAsk}
${rt.workingDirExampleLabel}
"${fillTemplate(rt.workingDirExampleText, { path: detectedPath })}
${replyHint}"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${rt.workingDirOriginalMessageLabel}
${originalMessage}
`;
return instruction.trim();
}
async function extractPdfText(fileHandle: FileHandle): Promise<string> {
try {
// Typed interface for FileHandle with optional readFile method
type FileHandleWithReadFile = {
name: string;
readFile?: () => Promise<Buffer>;
read?: () => Promise<unknown>;
};
const typedHandle = fileHandle as unknown as FileHandleWithReadFile;
const buffer = typedHandle.readFile
? await typedHandle.readFile()
: Buffer.from(await (typedHandle.read?.() as Promise<string>) ?? '');
const data = await pdfParse(buffer);
return data.text.trim();
} catch (error) {
console.error(`[RAG] Error extracting text from PDF ${fileHandle.name}:`, error);
throw new Error(`Failed to parse PDF: ${fileHandle.name}`);
}
}
function chunkText(text: string, chunkSize: number = 1000, overlap: number = 100): string[] {
const words = text.split(/\s+/);
const chunks: string[] = [];
if (words.length <= chunkSize) {
return [text];
}
let startIndex = 0;
while (startIndex < words.length) {
const endIndex = Math.min(startIndex + chunkSize, words.length);
const chunkText = words.slice(startIndex, endIndex).join(' ');
chunks.push(chunkText);
startIndex = endIndex - overlap;
}
return chunks.filter(c => c.trim().length > 0);
}
function cosineSimilarity(a: number[], b: number[]): number {
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
interface RetrievalResult {
content: string;
score: number;
}
async function retrieveFromPdfs(
ctl: PromptPreprocessorController,
query: string,
pdfFiles: FileHandle[],
): Promise<RetrievalResult[]> {
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const retrievalLimit = pluginConfig.get('retrievalLimit') || 5;
// Lower default threshold to catch more results - was too high at 0.6
const retrievalAffinityThreshold = pluginConfig.get('retrievalAffinityThreshold') ?? 0.3;
console.log(`[RAG] Processing ${pdfFiles.length} PDF file(s)`);
// Extract text from all PDF files
const fileTexts: { file: FileHandle; text: string }[] = [];
for (const file of pdfFiles) {
try {
const text = await extractPdfText(file);
if (text.length > 0) {
console.log(`[RAG] Extracted ${text.length} chars from ${file.name}`);
fileTexts.push({ file, text });
} else {
console.log(`[RAG] No text extracted from ${file.name}`);
}
} catch (error) {
console.error(`[RAG] Skipping PDF ${file.name} due to error:`, error);
}
}
if (fileTexts.length === 0) {
console.log('[RAG] No text extracted from any PDF');
return [];
}
// Chunk the texts
const chunks: { file: FileHandle; chunk: string }[] = [];
for (const { file, text } of fileTexts) {
const fileChunks = chunkText(text);
console.log(`[RAG] ${file.name}: ${text.length} chars → ${fileChunks.length} chunks`);
fileChunks.forEach((chunk) => {
chunks.push({ file, chunk });
});
}
if (chunks.length === 0) return [];
// Generate embeddings for all chunks using LM Studio's embedding model
let model;
try {
console.log('[RAG] Loading embedding model...');
model = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
console.log('[RAG] Embedding model loaded successfully');
} catch (error) {
console.error('[RAG] Failed to load embedding model:', error);
throw new Error(`Embedding model not available: ${error instanceof Error ? error.message : String(error)}`);
}
const batchSize = 32;
const allEmbeddings: number[][] = [];
try {
for (let i = 0; i < chunks.length; i += batchSize) {
console.log(`[RAG] Generating embeddings batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(chunks.length / batchSize)}...`);
const batch = chunks.slice(i, i + batchSize).map(c => c.chunk);
const embeddingsResult = await model.embed(batch);
// Type the embedding result properly
type EmbeddingResult = { embedding: number[] };
allEmbeddings.push(...(embeddingsResult as EmbeddingResult[]).map((e) => e.embedding));
}
} catch (error) {
console.error('[RAG] Error generating embeddings:', error);
throw new Error(`Embedding generation failed: ${(error instanceof Error ? error.message : String(error))}`);
}
// Generate embedding for the query
let queryModel;
try {
queryModel = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
} catch (error) {
console.error('[RAG] Failed to load query embedding model:', error);
throw new Error(`Query embedding failed: ${(error instanceof Error ? error.message : String(error))}`);
}
let queryEmbedding;
try {
const queryResult = await queryModel.embed([query]);
queryEmbedding = queryResult[0].embedding;
} catch (error) {
console.error('[RAG] Error generating query embedding:', error);
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`Query embedding failed: ${errorMessage}`);
}
// Calculate similarities and retrieve top results
const scores: { chunkIndex: number; similarity: number }[] = [];
for (let i = 0; i < chunks.length; i++) {
const similarity = cosineSimilarity(queryEmbedding, allEmbeddings[i]);
scores.push({ chunkIndex: i, similarity });
}
// Sort by similarity descending and filter by threshold
scores.sort((a, b) => b.similarity - a.similarity);
console.log(`[RAG] Found ${scores.length} chunks, filtering with threshold ${retrievalAffinityThreshold}`);
const relevantChunks = scores.filter(
(s) => s.similarity >= retrievalAffinityThreshold && s.chunkIndex < chunks.length,
);
// Limit results
const limitedResults = relevantChunks.slice(0, retrievalLimit);
console.log(`[RAG] Returning ${limitedResults.length} results`);
return limitedResults.map((r) => ({
content: chunks[r.chunkIndex].chunk,
score: r.similarity,
}));
}
// ==================== Step 0.7 Confirm-First Project Switching (Fix A) ====================
/**
* Pending project-switch confirmation state (Fix A). Set when a registered project is detected in
* a user message; consumed one-shot only when the user explicitly replies YES/JA on a subsequent
* message. Cleared on decline, expiry, or successful switch.
*/
let pendingProjectSwitch: { name: string; path: string } | null = null;
/**
* Normalize a multilingual confirmation reply to the canonical FSM input (Fix B, extended by the
* i18n-CONFIRM arc 06.10). Accepts EVERY shipped language's confirm/decline words — union derived
* from src/locales/*.ts at module load (YES/JA/SÍ/确认/確認 → 'YES'; NO/NEIN/取消 → 'NO'); anything
* else → null (not a confirmation reply). Language-agnostic by design: only the user's configured
* language is DISPLAYED, but any shipped word still parses (owner verdict 06.10 ~19:0x).
*/
export function normalizeConfirmationReply(raw: string): 'YES' | 'NO' | null {
const u = raw.trim().toUpperCase();
if (ALL_CONFIRM_WORDS.has(u)) return 'YES';
if (ALL_DECLINE_WORDS.has(u)) return 'NO';
return null;
}
/** Outcome of Step 0.7's confirm-first project-switch gate (Fix A). */
export type ProjectSwitchDecision =
| { kind: 'skip' }
| { kind: 'banner'; match: { name: string; path: string } }
| { kind: 'execute'; match: { name: string; path: string } }
| { kind: 'declined' };
/**
* Pure decision logic for Step 0.7 (Fix A — v1.9.8+ safety rule in the index.ts docblock):
* NEVER switch the working directory on detection alone. The banner asks for confirmation;
* only an explicit YES/JA reply executes the one-shot switch. NO/NEIN or any other reply
* expires the offer (the "next message" contract).
*/
export function decideProjectSwitch(
match: { name: string; path: string } | null,
reply: string,
currentCwd: string,
pending: { name: string; path: string } | null,
): ProjectSwitchDecision {
if (!match) return { kind: 'skip' };
// Already in the detected project's directory → a switch would be a no-op.
if (path.resolve(currentCwd) === path.resolve(match.path)) return { kind: 'skip' };
const normalized = normalizeConfirmationReply(reply);
if (pending && pending.name === match.name && path.resolve(pending.path) === path.resolve(match.path)) {
if (normalized === 'YES') return { kind: 'execute', match }; // one-shot switch on explicit confirmation
return { kind: 'declined' }; // NO/NEIN or any non-confirmation reply → expire the offer
}
// New detection, or a different project than the pending offer → confirm-first banner.
return { kind: 'banner', match };
}
// ==================== Session-Memory Extraction Helpers ====================
/** Coerce a decoded session-summary value into plain text for prompt injection.
* StateManager writes the record as an OBJECT ({task_description, ...}); older/legacy writers used bare strings. */
function summaryValueToText(value: unknown): string | undefined {
if (typeof value === 'string' && value.length > 0) return value;
if (value && typeof value === 'object') {
const o = value as Record<string, unknown>;
if (typeof o.task_description === 'string' && o.task_description.length > 0) return o.task_description;
try { return JSON.stringify(value); } catch { return undefined; }
}
return undefined;
}
/** CONTAMINATION-FIX follow-up C / Gap-2b (01.10): pull the latest session summary out of a decoded store,
* understanding BOTH writer shapes:
* - CURRENT: StateManager record array [{ key, value, timestamp }] with key 'session_summary_latest'
* (value is an object or string) — see stateManager.ts / contextManagementTools.ts saveSessionSummary;
* - LEGACY: plain object map { latest_session_summary | session_summary_latest: string }.
* Pre-fix the caller only understood the legacy map shape, so real StateManager stores were decoded but
* never yielded a summary ("no summary found" despite a populated file). Returns undefined when no usable
* summary is present (not an error). */
function extractLatestSessionSummary(decoded: unknown): string | undefined {
if (!decoded || typeof decoded !== 'object') return undefined;
// CURRENT writer shape: array of {key, value, timestamp} records
if (Array.isArray(decoded)) {
for (let i = decoded.length - 1; i >= 0; i--) {
const r = (decoded as Array<Record<string, unknown>>)[i];
if (!r || typeof r !== 'object') continue;
if (typeof r.key === 'string' && r.key.includes('session_summary')) return summaryValueToText(r.value);
}
return undefined;
}
// LEGACY writer shape: object map with string values
const d = decoded as Record<string, unknown>;
for (const k of ['latest_session_summary', 'session_summary_latest']) {
// 🔹 LINT FIX (01.10): the typeof guard already narrows d[k] to string — the two "as string" assertions were no-ops tripping @typescript-eslint/no-unnecessary-type-assertion.
if (typeof d[k] === 'string' && d[k].length > 0) return d[k];
}
const objVal = d.session_summary_latest ?? d.latestSummary;
return summaryValueToText(objVal);
}
/**
* Execute a confirmed project CWD switch (Step 0.7, Fix A): canonical persistent state change via
* setWorkingDir() plus best-effort process.chdir(), then load and inject the target project's
* session memory (msgpack with plain-JSON fallback). Returns the fully composed prompt for this turn.
*/
// CONTAMINATION-FIX follow-up C (01.10): exported for the hermetic session-memory read-guard tests
// (tests/sessionMemoryReadGuards.test.ts) — the confirm-first flow in preprocess() stays the only production caller.
// UNIFIED LANGUAGE v2 (step 4): optional per-turn RuntimeTexts for the session-memory banners below — absent →
// English bundle via getRuntimeTexts('en') (byte-identical to the pre-v2 literals; keeps non-preprocess callers/tests untouched).
export async function executeProjectSwitch(
newCwd: string,
userPrompt: string,
attachmentNotice: string,
checkpointSuffix: string,
ctl: PromptPreprocessorController,
runtimeTexts?: RuntimeTexts,
): Promise<string> {
console.log(`[ProjectAutoDetect] Switching working directory to: ${newCwd}`);
try {
// Canonical CWD change — persistent working-dir state (authoritative for all getWorkingDir() consumers)
// plus best-effort process.chdir. Previously only process.chdir was called, so tool-level CWD never changed.
if (!applyProjectCwdSwitch(newCwd)) {
console.warn(`[ProjectAutoDetect] CWD switch rejected for '${newCwd}' — proceeding without switch`);
return `${userPrompt}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
// Read session memory from the new working directory (if exists)
try {
// CONTAMINATION-FIX follow-up C / Gap-2 (01.10): candidate store files — per-project FIRST, with the
// EXACT naming StateManager uses for its own file (`resolveProjectName()` derivation over this project's
// basename), legacy fixed-name .ai_toolbox_memory.msgpack as fallback. At this point applyProjectCwdSwitch()
// has already re-bound the working dir to newCwd, so deriving from newCwd's basename is byte-identical to
// StateManager.getMemoryFilePath()'s live derivation — no cross-project reads (RESEARCH doc §4).
const memoryCandidates = [
path.join(newCwd, '.session_context', `.${path.basename(newCwd).toLowerCase().replace(/[^a-z0-9]/g, '_')}_memory.msgpack`),
path.join(newCwd, '.session_context', '.ai_toolbox_memory.msgpack'),
];
let msgpackPath: string | null = null;
for (const candidate of memoryCandidates) {
if (fs.existsSync(candidate)) {
msgpackPath = candidate;
break;
}
}
if (msgpackPath !== null) {
console.log(`[ProjectAutoDetect] Found session memory at ${msgpackPath}`);
// Load and decode the store file via the static @msgpack/msgpack import (see module header — replaces
// the former per-call `await import('@msgpack/msgpack')`, fragile under CJS).
try {
const rawBytes = fs.readFileSync(msgpackPath);
if (rawBytes.length > 0) {
// CONTAMINATION-FIX follow-up C / Gap-2b (01.10): understand BOTH writer shapes — the CURRENT
// StateManager record array [{key,value,timestamp}] and LEGACY object maps — via extractLatestSessionSummary
// (above). Pre-fix this path only understood the legacy map shape, so real stores decoded fine but
// NEVER yielded a summary.
const latestSummary: string | undefined = extractLatestSessionSummary(decodeMsgpack(rawBytes));
// If found, inject session summary context into the prompt
if (latestSummary && latestSummary.length > 0) {
console.log(`[ProjectAutoDetect] Loaded session memory from ${newCwd}`);
// UNIFIED LANGUAGE v2 (step 4): banner title/label from the per-turn RuntimeTexts — 📋 + box-drawing structure stays literal here.
const memRt = runtimeTexts ?? getRuntimeTexts('en');
const sessionContext = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${memRt.sessionMemoryLoadedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${memRt.previousSessionLabel}\n${latestSummary}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`;
return `${userPrompt}${sessionContext}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
} else {
console.log(`[ProjectAutoDetect] Session memory exists but no summary found`);
}
}
} catch (decodeError) {
console.warn(`[ProjectAutoDetect] Failed to decode msgpack from ${msgpackPath}:`, decodeError instanceof Error ? decodeError.message : String(decodeError));
// Fallback: try reading as plain JSON if msgpack fails — the shared helper understands both record
// arrays and object maps, so no duplicated shape-check code is needed here.
try {
const rawJson = fs.readFileSync(msgpackPath, 'utf-8');
const summaryText: string | undefined = extractLatestSessionSummary(JSON.parse(rawJson));
if (summaryText && summaryText.length > 0) {
console.log(`[ProjectAutoDetect] Loaded session memory from ${newCwd} (JSON fallback)`);
// UNIFIED LANGUAGE v2 (step 4): same RuntimeTexts-driven banner as the msgpack path above.
const memRt = runtimeTexts ?? getRuntimeTexts('en');
const sessionContext = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${memRt.sessionMemoryLoadedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${memRt.previousSessionLabel}\n${summaryText}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`;
return `${userPrompt}${sessionContext}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
} catch (jsonError) {
// Defensive net: a malformed JSON fallback (JSON.parse throw) must never abort the switch.
console.warn(`[ProjectAutoDetect] JSON fallback also failed:`, jsonError instanceof Error ? jsonError.message : String(jsonError));
}
}
} else {
console.log(`[ProjectAutoDetect] No session memory found under ${newCwd} (no per-project or legacy store file present)`);
}
} catch (e) {
console.warn(`[ProjectAutoDetect] Failed to load session memory from ${newCwd}:`, e instanceof Error ? e.message : String(e));
}
} catch (e) {
console.error(`[ProjectAutoDetect] Error detecting/switching project:`, e);
}
// If automatic switch failed or no session memory found, return original message with context
return `${userPrompt}${attachmentNotice}${checkpointSuffix}`.trim() + getTemporalSuffix(ctl);
}
/** Apply the project CWD switch (Step 0.7).
* Canonical change = persistent working-dir state via setWorkingDir() — authoritative for all getWorkingDir() consumers;
* plus a best-effort process.chdir so raw process.cwd() consumers (git/image tools) follow along too.
* Previously only process.chdir was called, so the tool-level CWD (state file) never actually changed.
* Returns false if the switch was rejected (path invalid). */
export function applyProjectCwdSwitch(newCwd: string): boolean {
// CONTAMINATION-FIX Part A (01.10): capture the pre-switch working dir — required to flush the previous
// project's pending RAM into its OWN file when rebinding StateManager identity below.
const prevWd = getWorkingDir();
if (!setWorkingDir(newCwd)) return false;
try {
const oldCwd = process.cwd();
if (oldCwd !== newCwd && typeof process.chdir === 'function') {
process.chdir(newCwd);
console.log(`[ProjectAutoDetect] Working directory changed: ${oldCwd} → ${newCwd}`);
}
} catch (chdirError) {
// Non-fatal — persistent state is already set; only raw process.cwd() consumers stay behind.
console.warn(`[ProjectAutoDetect] process.chdir failed (non-fatal): ${chdirError instanceof Error ? chdirError.message : String(chdirError)}`);
}
// ✅ CONTAMINATION-FIX Part A (01.10): fire-and-forget identity rebind — this function is synchronous by its
// contract, so the async flush/reload runs after return. Errors are contained inside refreshProject; the
// catch here covers only unforeseen throws (see stateManager.ts for the ordering guarantees).
void getStateManager()
?.refreshProject(prevWd)
.catch((err: unknown) => {
console.warn(`[ProjectAutoDetect] StateManager identity rebind after CWD switch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
});
return true;
}
export async function preprocess(
ctl: PromptPreprocessorController,
userMessage: ChatMessage
): Promise<string | ChatMessage> {
let userPrompt = '';
try {
const rawText = await Promise.resolve(userMessage.getText());
userPrompt = typeof rawText === 'string' ? rawText : String(rawText ?? '');
} catch {}
// UNIFIED LANGUAGE v2 (step 4, 27.10): resolve THIS turn's injected-text language ONCE at the top of the message layer —
// chain = concretely configured value → LM Studio host preference → OS locale first tag → English fallback
// (see resolveTurnLanguage()). MUST sit before any consumer below: the S5 autoTracker call and the answer-language line
// live INSIDE the ContextGuard try-block, whose finally runs well before Step 0.6+. One extra config fetch vs. a later
// site — negligible per turn (config reads are fresh per call elsewhere in this file too).
const turnLang = resolveTurnLanguage(ctl.getGlobalPluginConfig(configSchematics).get('language'));
const rt = getRuntimeTexts(turnLang);
// UNIFIED LANGUAGE v2 step 5 (27.10): keep the boot-bake state file in sync with THIS session's effective language —
// after any reload, panel labels and prompt texts render in the same locale. persistConcretePick() is change-aware
// (no write while the value is unchanged) and non-throwing by contract; i18n persistence must never break a turn.
try {
persistConcretePick(turnLang);
} catch { /* non-fatal by design — the boot chain re-resolves on next load */ }
// ✅ DIAGNOSTIC: Verify preprocessor is actually being called by LM Studio
console.log(`✅ [Preprocessor] Called. Message length: ${userPrompt.length}`);
if (!contextGuard) { console.warn('[ContextGuard] contextGuard is NULL!'); }
// 🔹 C compaction family (24.09) SERIALIZATION GUARD: a new user message can arrive while the previous
// turn's agentic tool loop is still executing in this process. Drain with a BOUNDED wait before any
// turn-level state mutation below — otherwise in-flight results would be recorded against an already-
// cleared baseline (resetMidLoopDelta) and auto-compression could replace history under live tools
// (DeepSeek harness item C: "serialize user-triggered operations against active loops"). Bounded by
// design: on timeout the drain logs FAIL-LOUD and proceeds; a user message must never deadlock.
try {
await drainActiveToolTurns();
} catch (drainErr) {
console.warn('[C-compact] Drain of active tool turn failed (non-fatal, proceeding):', drainErr);
}
// 🔹 FIX #20 (A1): reset the per-turn tool-payload delta. By the time this runs, last turn's tool
// results are part of pullHistory() and get counted natively below — keeping them in the delta
// would double-count them on the next mid-loop guard evaluation.
TokenStatsManager.resetMidLoopDelta();
// 🔹 Declare variables at top scope so they're accessible across all steps
let pendingWarning: string | undefined;
// 🔹 PART B (re-applied 21.08 after restore lost it): set when the pre-compression checkpoint did
// NOT save but a YES/NO prompt was still pending — a last-chance note is appended to this turn's output.
let lastChanceNote = false;
// 🔹 Fix A/B coordination: set when Step 0.6 consumes a checkpoint reply this turn —
// prevents the same reply from being re-interpreted as a project-switch confirmation in Step 0.7.
let checkpointConsumedThisTurn = false;
let messageCount = 0;
let tokenCount = 0;
// 🔹 FIX #3 (04.10, CTX-FOOTER arc part 3): function-scoped so the finally-publish below can see it even
// when a throw lands before the assignment further down — hoisted from its former const inside the try body.
let maxTokens = 0;
// 🔹 FIX #21 (09.10, two-tier mid-loop save): function-scoped like the others above so the finally-publish can
// see it even when a throw lands before any assignment below — stays null then, i.e. "no list published for this
// turn" (the wrapper's trigger gate treats null as no-op, same safe direction as baseline === 0).
let publishedTurnMessages: { role?: string; content?: unknown; [key: string]: unknown }[] | null = null;
let historyTextLength = 0; // ✅ Moved outside if-block to fix scoping
// Step 0.5: ContextGuard auto-compression & token tracking
if (contextGuard) {
try {
console.log('[ContextGuard DEBUG] Step 1: pullHistory()...');
const history = await ctl.pullHistory();
console.log('[ContextGuard DEBUG] Step 2: checking history object...');
if (!history) { console.warn('[ContextGuard] pullHistory() returned undefined, skipping'); return userPrompt; }
history.append(userMessage);
console.log('[ContextGuard DEBUG] Step 3: getMessagesArray()...');
const messages = history.getMessagesArray() as unknown as { role?: string; content?: unknown; [key: string]: unknown }[] | undefined;
console.log('[ContextGuard DEBUG] Step 4: messages type:', typeof messages, 'isArray:', Array.isArray(messages));
// Safely extract files/images, handling both arrays and single objects
let toolCallCount = 0;
let imageCount = 0;
if (messages && Array.isArray(messages)) {
for (const msg of messages) {
const typedMsg = msg as ExtendedMessage;
if (typedMsg.toolCalls || typedMsg.tool_calls) {
toolCallCount += typedMsg.toolCalls?.length || typedMsg.tool_calls?.length || 0;
}
const fileObj = typedMsg.files || typedMsg.images;
if (fileObj && Array.isArray(fileObj) && fileObj.length > 0) {
imageCount += fileObj.length;
} else if (fileObj && typeof fileObj === 'object') {
// Handle single object case
imageCount += 1;
}
}
// ✅ Use LM Studio's native history API instead of .content casting (matches vibe-lm approach)
historyTextLength = 0;
try {
const msgCount = history.getLength();
for (let i = 0; i < msgCount; i++) {
const msg = history.at(i);
let msgText = '';
if (msg.getText) msgText += msg.getText() || '';
if (msg.getToolCallRequests) msgText += JSON.stringify(msg.getToolCallRequests()) || '';
if (msg.getToolCallResults) msgText += JSON.stringify(msg.getToolCallResults()) || '';
historyTextLength += msgText.length;
}
} catch (e) {
console.warn('[TokenDebug] Failed to iterate native history:', e);
}
// ✅ v2.x: Include serialized tool definitions in the counted span — LM Studio's sidebar counts them too;
// previously only message history was counted here (root cause of "sidebar ~265k vs plugin count much lower").
const toolOverheadChars = getToolOverheadChars();
if (toolOverheadChars > 0) {
console.log(`[TokenDebug] Tool-definition overhead: +${toolOverheadChars} chars added to counted span`);
historyTextLength += toolOverheadChars;
}
const estimatedHistoryTokens = estimateTokensFromChars(historyTextLength); // shared ratio - see tokenStatsManager
// 15.10: label no longer hardcodes a ratio — the density is user-editable (contextGuardAgenticDensity) and live in tokenStatsManager.
console.log(`[TokenDebug] History Text Length: ${historyTextLength} chars | Est. Tokens (shared agentic density, live — see tokenStatsManager): ${estimatedHistoryTokens}`);
}
console.log(`[TokenDebug] Total Tool calls: ${toolCallCount}, Total Files/Images: ${imageCount}`);
// ✅ FIX: Dynamically resolve actual model context length before counting tokens
console.log('[ContextGuard DEBUG] Step 5: getGlobalPluginConfig()...');
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
// Inject LM Studio client from controller for dynamic model queries (SDK v1.x)
if (ctl.client && contextGuard.setLMClient) {
contextGuard.setLMClient(ctl.client);
}
// Re-seed the user's configured compression trigger PERCENTAGE every turn (07.10). ContextGuard is
// constructed in index.ts WITHOUT config access, so without this reseed a menu change would only take
// effect after a plugin reload. The effective WINDOW still comes from setTokenLimitFromModel() below
// (model-reported window; built-in 30k fallback when unavailable) — the percent applies on top of it.
const compressionPercent = pluginConfig.get('contextGuardCompressionPercent');
if (typeof compressionPercent === 'number' && Number.isFinite(compressionPercent) && compressionPercent > 0) {
contextGuard.updateConfig({ compressionPercent });
}
// Re-seed the user's configured agentic chars-per-token DENSITY every turn (15.10 owner request). estimateTokensFromChars()
// is a module-level function in tokenStatsManager WITHOUT config access, so — exactly like the percent above — a menu change
// only takes effect via this per-turn reseed. Stub/plugin configs returning undefined keep the built-in default 3.6 (raised from 3.4 on 15.10 owner instruction).
const agenticDensity = pluginConfig.get('contextGuardAgenticDensity');
if (typeof agenticDensity === 'number' && Number.isFinite(agenticDensity) && agenticDensity > 0) {
setAgenticCharsPerToken(agenticDensity);
}
// Auto-detect the currently active model and get its ID for accurate token counting
let activeModelId = '';
try {
const llmClient = ctl.client?.llm;
if (llmClient && typeof llmClient.model === 'function') {
// Get the active model to extract its ID
const activeModel = await llmClient.model();
if (activeModel) {
const typedModel = activeModel as LLMModelWithOptionalFields;
activeModelId = typedModel.modelKey || typedModel.id || '';
console.log(`[ContextGuard] ✅ Auto-detected active model: ${activeModelId}`);
// ACTIVE-CHAT-MODEL FALLBACK (09.11, owner GO): publish the auto-detected chat-model id so the wrapper-side
// mid-loop saves (TIER-2 / M3 in toolsProvider) can fall back to it when contextGuardSummaryModel is empty —
// their ToolCallContext exposes no model hop. Publish ONLY here: the configured-summaryModel branches below
// resolve inside generateSessionContinuity() itself, and an undetected turn must not erase a previous id.
TokenStatsManager.setActiveChatModelId(activeModelId);
// Update token limit from SDK
await contextGuard.setTokenLimitFromModel(activeModelId);
} else {
const configuredSummaryModel = pluginConfig.get('contextGuardSummaryModel') || '';
if (configuredSummaryModel) {
activeModelId = configuredSummaryModel;
await contextGuard.setTokenLimitFromModel(activeModelId);
console.log(`[ContextGuard] ✅ Using configured summaryModel: ${activeModelId}`);
} else {
console.warn('[ContextGuard] No active model detected. Using default tokenLimit.');
}
}
}
} catch {
const configuredSummaryModel = pluginConfig.get('contextGuardSummaryModel') || '';
if (configuredSummaryModel) {
activeModelId = configuredSummaryModel;
console.warn(`[ContextGuard] ⚠️ Auto-detection failed, using summaryModel: ${activeModelId}`);
} else {
console.warn('[ContextGuard] ⚠️ No model detected. Using default tokenLimit.');
}
}
// Extract System Prompt from history (usually index 0) to ensure accurate budget tracking
let extractedSystemPrompt: string | undefined;
if (Array.isArray(messages) && messages.length > 0 && typeof messages[0] === 'object') {
const firstMsg = messages[0] as ExtendedMessage & { content?: unknown };
if ((firstMsg.role === 'system' || firstMsg.role === 'system_prompt') && firstMsg.content) {
let sysContent: string | undefined;
if (typeof firstMsg.content === 'string') {
sysContent = firstMsg.content;
} else if (Array.isArray(firstMsg.content)) {
const contentArr = firstMsg.content as Array<{ type?: string; text?: string }>;
sysContent = contentArr.filter((c) => c.type === 'text').map((c) => c.text ?? '').join(' ');
}
if (sysContent && sysContent.trim().length > 0) extractedSystemPrompt = sysContent;
}
}
// Calculate tokens for threshold check using actual model limit & the shared agentic-density estimate (estimateTokensFromChars)
const safeMessages = messages ?? [];
// 🔹 FIX #21 (09.10): publish this turn's live message list with the token baseline — re-published below when
// a same-turn compression replaces it, so the finally point always sees the post-compression array.
publishedTurnMessages = safeMessages;
console.log('[ContextGuard DEBUG] Step 6: calling countTokens()...');
try {
tokenCount = await contextGuard.countTokens(safeMessages, imageCount, activeModelId || undefined, extractedSystemPrompt, historyTextLength);
console.log('[ContextGuard DEBUG] Step 7: countTokens() succeeded, getting limits...');
} catch (countError) {
console.error('[ContextGuard DEBUG] STEP 6 FAILED:', countError);
throw countError; // Re-throw to see full stack trace
}
// 🔹 FIX #3 (04.10): assignment into the function-scoped `maxTokens` declared at the top — the
// finally-publish must read a value that survives throws landing before this line (it stays 0 then,
// i.e. the same safe no-op state as never publishing).
maxTokens = contextGuard.getTokenLimit();
const threshold = contextGuard.getThreshold();
// ✅ DIAGNOSTIC: Log accurate token counts against real model limits
console.log(`✅ [TokenCheck] Model limit: ${maxTokens} | Tokens used: ${tokenCount} | Threshold: ${Math.round(threshold)}`);
console.log('[ContextGuard DEBUG] Step 8: checking auto-tracker config...');
// 🔹 WIRE UP AUTO-TRACKER THRESHOLD PROMPT — Step 0.5b (uses correct maxTokens)
const autoTrackingConfig = ctl.getGlobalPluginConfig(configSchematics);
if ((autoTrackingConfig.get('autoTrackingEnabled') ?? true) && maxTokens > 0) {
console.log('[ContextGuard DEBUG] Step 9: calling checkAndGeneratePrompt()...');
try {
// i18n-CONFIRM arc (06.10): the checkpoint question is DISPLAYED in the user's configured
// language ("🌐 Language" menu); reply parsing stays multilingual inside Step 0.6 below.
// UNIFIED LANGUAGE v2 (step 4): S5 runtime texts ride along so the threshold warning renders in this turn's language.
const promptResult = autoTracker.checkAndGeneratePrompt(
tokenCount,
maxTokens,
getConfirmationWords(autoTrackingConfig.get('language')),
rt,
);
if (promptResult.triggered && promptResult.warning) {
pendingWarning = promptResult.warning;
console.log(`[AutoTracker] ✅ THRESHOLD PROMPT GENERATED — user will be asked to save session memory`);
}
} catch (autoTrackError) {
console.error('[ContextGuard DEBUG] STEP 9 FAILED:', autoTrackError);
throw autoTrackError;
}
}
// Capture message count for later use in Step 0.6
messageCount = history?.getLength() ?? 0;
// 🔹 F1 (17.09 G-B fix): one-shot notice text produced inside PART B below; appended to userPrompt after the
// compression block (see there). Lives only in this preprocess() run → never re-injected on later turns.
let checkpointSavedNotice = '';
if (tokenCount > threshold) {
console.log(`[ContextGuard] Token count ${tokenCount} exceeds compression threshold ${threshold}, compressing...`);
// 🔹 C compaction family (24.09) STEP 3 — PRE-SUMMARIZATION PRUNING: prune oversized tool payloads
// from the message array BEFORE any summarization work starts. Why before, not after: (a)
// compressHistory() ships every pre-keepLast message verbatim to the summary model — one giant
// read_file/web_fetch payload can push THAT prompt past the summary model's own context window and
// force the documented fallback path ("Original content unavailable") that loses all detail; (b)
// keepLast messages survive verbatim in the rebuilt history, so an unpruned giant result would sit
// in live context forever and re-fire this threshold almost immediately. Nothing is lost: every
// pruned payload is stored verbatim under its SHA-256 digest in .ai_toolbox/compaction/, and the
// preview embeds a retrieval hint + opaque locator (C step 2). STORE-BEFORE-PRUNE ordering: if any
// store fails, pruning for this turn is skipped entirely — history keeps full payloads exactly as
// it does today (safe degradation; orphaned already-stored files are idempotency-safe).
if (pluginConfig.get('compactionEnabled') !== false) { // default ON; explicit false opts out entirely
try {
const cfgMaxBytes = pluginConfig.get('compactionMaxResultBytes');
const maxBytes = typeof cfgMaxBytes === 'number' && Number.isFinite(cfgMaxBytes) && cfgMaxBytes > 0
? Math.floor(cfgMaxBytes)
: DEFAULT_MAX_BYTES_PER_RESULT;
const refs = collectPrunedPayloadRefs(safeMessages, maxBytes);
if (refs.length > 0) {
for (const ref of refs) {
// Exclusive 'wx' create + 0o600: never overwrites, refuses symlink targets at the path.
await storePrunedPayload(getWorkingDir(), ref);
}
const pruneOutcome = pruneOversizedToolPayloads(
safeMessages,
{ maxBytesPerResult: maxBytes, digests: refs.map((r) => r.digestHex) },
);
if (pruneOutcome.prunedCount > 0) {
console.log(
`[C-compact] Pruned ${pruneOutcome.prunedCount} oversized tool payload(s) before summarization — ` +
`~${(pruneOutcome.bytesSaved / 1024).toFixed(1)} KiB out of context; verbatim copies in .ai_toolbox/compaction/`,
);
} else {
console.warn(`[C-compact] FAIL-LOUD: ${refs.length} payload(s) collected but NONE pruned (skipped=${pruneOutcome.skippedCount}) — history kept verbatim this turn`);
}
}
} catch (cErr) {
// Any store failure aborts pruning for THIS TURN (the prune pass only runs after the full store loop).
console.error(`[C-compact] Prune-before-summarization aborted (non-fatal — compressing with verbatim history):`, cErr);
}
}
// 🔹 PART B — PRE-COMPRESSION CHECKPOINT (awaited BEFORE history is destroyed). Non-fatal by design:
// any failure only logs; compression must still run (parity with the Part A / YES-reply save paths).
let snapshotSaved = false;
let snapshotSessionId: string | undefined; // 🔹 F1 — id for the one-shot user-visible notice below
try {
const saveResult = await autoTracker.autoSaveSessionMemory(tokenCount, maxTokens, messageCount);
snapshotSaved = !!saveResult?.saved;
if (snapshotSaved) {
snapshotSessionId = saveResult.sessionId; // 🔹 F1
console.log(`[AutoTracker] ✅ Pre-compression checkpoint saved: ${saveResult.sessionId}`);
} else {
console.warn('[AutoTracker] ⚠️ Pre-compression checkpoint NOT saved — compressing without a clean checkpoint');
}
} catch (e) {
console.warn(`[AutoTracker] ⚠️ Pre-compression snapshot failed (non-fatal): ${e instanceof Error ? e.message : String(e)}`);
}
// 🔹 F1 (17.09 G-B fix): the checkpoint SAVED while a warning was live in this same run — the old flow
// consumed that warning below with zero user-visible acknowledgement. Queue exactly one notice for this
// turn's model input (consumed by PART B-2 right after, so hasPendingWarning() here is the pre-consume state).
const f1Notice = buildCheckpointSavedNotice({
snapshotSaved,
warningWasLive: pendingWarning !== undefined || autoTracker.hasPendingWarning(),
usagePercent: maxTokens > 0 ? (tokenCount / maxTokens) * 100 : null,
sessionId: snapshotSessionId,
});
if (f1Notice) {
checkpointSavedNotice = f1Notice;
}
// 🔹 PART B-2: settle any pending YES/NO prompt — the context it referred to is about to be replaced, so a
// re-injected <SYSTEM_INSTRUCTION> would demand a reply (and a second save) for history that no longer exists.
if (!snapshotSaved && autoTracker.hasPendingWarning()) {
lastChanceNote = true;
}
if (autoTracker.hasPendingWarning()) {
autoTracker.consumePendingConfirmation(); // clear the tracker-side prompt; FSM no longer reports it pending
pendingWarning = undefined; // drop this turn's locally-captured copy too — the suffix builder must NOT re-inject a YES/NO request about history that is being replaced now
}
// 💾 Arc C (07.10) — FORCED STRUCTURED SESSION-MEMORY SAVE at the compression trigger, awaited BEFORE
// compressHistory() destroys history (same ordering guarantee as PART B's telemetry snapshot above).
// STRICT `=== true` gate on purpose: production ParsedConfig applies the zod default(true), while pinned
// harness stubs return undefined for unknown keys → the arc-C path is fully skipped there. The typeof guard
// additionally protects legacy/stub ContextGuard objects lacking generateSessionContinuity; and
// safeMessages.length > 10 guarantees a non-empty summarizable span past the keepLast=10 exclusion below.
// Non-fatal by design: any failure only logs — compression must still run and PART B's telemetry snapshot
// + F1 notice stay byte-identical (pinned by tests/precompressSnapshot.test.ts, which never reach this block).
if (pluginConfig.get('contextGuardForceSummaryOnCompress') === true
&& typeof contextGuard.generateSessionContinuity === 'function'
&& safeMessages.length > 10) {
try {
// Flush buffered auto-tracked actions FIRST: they land in the persistent store before the
// structured summary is generated from this turn's message span.
await autoTracker.flushActionsToMemory();
// keepLast=10 exclusion on the CALLER side — mirrors compressHistory's slice(0, -keepLast): the last
// 10 messages survive compression verbatim and are deliberately NOT summarized here.
const toSummarize = safeMessages.slice(0, safeMessages.length - 10);
// contextGuard is narrowed to the concrete class here (the enclosing `if (contextGuard)` scope);
// generateSessionContinuity is a regular class method — no cast/assertion needed. The typeof check in
// the gate above remains the runtime guard for legacy/stub instances lacking the method.
const summary = await contextGuard.generateSessionContinuity(toSummarize, {
fallbackModelId: activeModelId || undefined,
abortSignal: ctl.abortSignal,
});
if (summary) {
const persistOutcome = await persistGeneratedSessionSummary(summary);
if (persistOutcome.saved) {
console.log('[Arc C] ✅ Structured session-continuity summary persisted before compression');
} else {
// Telemetry fallback — deterministic, non-fatal: the summary was generated but NOT durable.
console.warn(`[Arc C] ⚠️ Continuity summary generated but persistence failed (${persistOutcome.error ?? 'unknown'}) — telemetry only`);
}
} else {
// No parsable model output (or no usable model) — deterministic telemetry fallback, compression proceeds.
// 15.10 owner report: warn→log reroute so this by-design notice stops rendering as an ERROR-level line in
// the host log file (stderr=error-level mapping; byte-identical text, ⚠️ marker stays in-text). Pairs with
// the matching C2a gate-skip notice in contextGuard.generateSessionContinuity().
console.log('[Arc C] ⚠️ No structured continuity summary available — telemetry fallback applies');
}
} catch (arcCErr) {
console.warn(`[Arc C] ⚠️ Forced session-memory save failed (non-fatal): ${arcCErr instanceof Error ? arcCErr.message : String(arcCErr)}`);
}
}
// Forward the host abort signal so an in-flight summarization prediction stops when the user
// cancels this turn (same ctl.abortSignal already used for embedding/retrieval below).
const compressedMessages = await contextGuard.compressHistory(safeMessages, ctl.abortSignal) as unknown as ChatMessage[];
// Clear history by popping all messages
while ((history?.getLength() ?? 0) > 0) {
history.pop();
}
compressedMessages.forEach((msg: ChatMessage) => history.append(msg));
contextGuard.resetTokenCache();
// 🔹 FIX #21 (09.10): re-publish the POST-compression array — the wrapper-side forced save must never see
// messages that compression just destroyed (the pre-compression safeMessages is dead at this point).
publishedTurnMessages = compressedMessages as unknown as { role?: string; content?: unknown; [key: string]: unknown }[];
// 🔹 FIX-1A (29.10, owner GO — supersedes FIX #20 A2): the pop/append above mutates ONLY the pullHistory()
// COPY — SDK contract (ProcessingController.pullHistory docblock): "The returned history is a copy, so
// mutating it will NOT affect the actual history." Counting the compressed array here therefore published a
// baseline for what is NOT on the wire: 29.10 incident L13057 logged "Post-compression count: 1440 tokens"
// while LM Studio rejected the real 167,060-token request in the same second (L13078) — the mid-loop guard
// then measured tool-payload growth against ~1.4k and could never arm inside that turn. FIX-1A re-pulls a
// FRESH copy of the ACTUAL host history and counts it with the SAME shared agentic-density estimator used at
// turn start (Step 0.5 pattern: native text + serialized tool-call requests/results char sums, plus the
// tool-definition overhead that rides every request) → the published baseline reflects reality again, so the
// mid-loop guard (FIX B, 95% pre-emptive gate) evaluates true usage after compression.
try {
const freshHistory = await ctl.pullHistory();
let freshChars = 0;
if (freshHistory) {
const freshLen = freshHistory.getLength();
for (let i = 0; i < freshLen; i++) {
// Structural cast to the exact surface this code touches (29.10 gate feedback: no-unnecessary-type-assertion +
// no-explicit-any + unsafe-* warnings on any-casts) — one typed view, zero `any`.
const fmView = freshHistory.at(i) as unknown as { getText?: () => string; getToolCallRequests?: () => unknown; getToolCallResults?: () => unknown };
let fText = '';
if (typeof fmView.getText === 'function') fText += (fmView.getText() || '');
try { if (typeof fmView.getToolCallRequests === 'function') fText += JSON.stringify(fmView.getToolCallRequests()) || ''; } catch { /* non-fatal — count what we have */ }
try { if (typeof fmView.getToolCallResults === 'function') fText += JSON.stringify(fmView.getToolCallResults()) || ''; } catch { /* non-fatal — count what we have */ }
freshChars += fText.length;
}
}
const freshOverhead = getToolOverheadChars(); // parity with Step 0.5: serialized tool defs ride on every request
if (freshOverhead > 0) freshChars += freshOverhead;
// Image parity with Step 0.5's primary path (countTokens adds imageCount*500 when historyTextLength is set).
// 29.10 gate feedback (publishHardening 'healthy turn' case): a fresh recount with ZERO counted chars is UNUSABLE —
// that happens only in hermetic stubs (empty fake history + mocked countTokens, e.g. tests/publishHardening.test.ts),
// never in production (compression fires above threshold ⇒ the host history has real messages). Overwriting a valid
// STEP-6 baseline with 0 would break the pinned publish contract AND blind the mid-loop guard — so adopt the fresh
// count only when it measured something, otherwise keep the pre-compression baseline (same conservative direction as the catch path).
const freshCount = Math.round(estimateTokensFromChars(freshChars)) + imageCount * 500;
if (freshChars > 0) {
tokenCount = freshCount; // also makes the Step 0.6 YES-reply checkpoint report accurate numbers
console.log(`[ContextGuard] [FIX-1A] Post-compression recount (FRESH pullHistory — host history is immutable to plugins): ${freshChars.toLocaleString('en-US')} chars → ${tokenCount.toLocaleString('en-US')} tokens (TRUE baseline for the mid-loop guard; compressed-copy count intentionally NOT used)`);
} else {
console.log(`[ContextGuard] [FIX-1A] Fresh post-compression recount unusable (0 chars counted — hermetic/empty history) — keeping pre-compression baseline ${tokenCount.toLocaleString('en-US')} tokens`);
}
} catch (countErr) {
// Non-fatal: keep the PRE-compression tokenCount — stale-HIGH is the conservative direction (the guard may
// snapshot early), whereas the old compressed-copy fallback was stale-LOW and blinded the guard entirely.
console.warn('[FIX-1A] Fresh post-compression recount failed; keeping pre-compression baseline (conservative):', countErr);
}
// 🔹 FIX-1B (29.10, owner GO — truthful unrecoverable-window notice): if the FRESH host-history recount is
// still at/above the compression threshold, in-band recovery is IMPOSSIBLE — plugins cannot mutate LM
// Studio's conversation history (SDK pullHistory() returns a copy; no rewrite endpoint exists in any of the
// 36 port RPCs; verified identical in SDK 2.1.0). Retrying would re-send the same oversized payload
// (29.10: ~15 consecutive send_error rejections, payload growing per retry) — so state that fact ONCE to
// the user instead of implying a reset happened. F1 pattern: appended straight onto userPrompt, one-shot
// (dies with this preprocess() run), independent of checkpointSuffix's hasPendingWarning()-gated paths.
if (tokenCount >= contextGuard.getThreshold()) {
const unrecoverableNotice =
'\n\n⚠️ CONTEXT WINDOW UNRECOVERABLE THIS CHAT: compression ran, but the host conversation history cannot be ' +
`modified from a plugin — LM Studio is still sending ~${Math.round(tokenCount / 1000)}k tokens against the ${Math.round(contextGuard.getTokenLimit() / 1000)}k window. ` +
'Starting a NEW chat is the only way to reset the context; everything important was checkpointed to session memory before compression.';
userPrompt += unrecoverableNotice;
console.log(`[ContextGuard] [FIX-1B] Unrecoverable-window notice appended (post-compression recount ${tokenCount.toLocaleString('en-US')} ≥ threshold ${Math.round(contextGuard.getThreshold()).toLocaleString('en-US')}) — one-shot`);
// 🔹 LATCH SUB-ITEM (30.10, owner GO — FIX #1 Part A guard): the fresh recount just PROVED this chat's window
// unrecoverable in-band — arm the TokenStatsManager latch so the wrapper-side mid-loop gates (TIER-2 forced
// save + FIX B pre-emptive compression) STOP re-firing on every model step of this and later turns. Without it,
// each per-step toolsProvider() entry resets both wrapper latches and re-runs prune → snapshot → Arc-C
// regeneration (up-to-60s summarization prediction) → compressHistory over a history that cannot shrink — pure
// no-op spend per step. Idempotent: later boundary recounts in the same chat re-affirm; a below-threshold
// recount clears it (else branch below); chat reset (TokenStatsManager.clear()) is the other clear path.
TokenStatsManager.setUnrecoverableWindow(tokenCount, contextGuard.getTokenLimit());
} else {
// 🔹 LATCH SUB-ITEM (30.10): post-compression fresh recount landed BELOW threshold — in-band recovery IS now
// possible (only legitimate source: a larger effective window, e.g. model switch between boundary runs). Clear
// the latch so the mid-loop gates re-arm for this turn; no-op with no log when already clear.
TokenStatsManager.clearUnrecoverableWindow();
}
}
// 🔹 F1 (17.09 G-B fix): surface the saved checkpoint to this turn's model input. Deliberately appended
// straight onto userPrompt rather than routed through checkpointSuffix — several return paths gate that suffix
// on hasPendingWarning() (false here by construction: PART B-2 consumed it above), which would silently drop
// this notice exactly like the original G-B bug. One-shot: the variable dies with this preprocess() run.
if (checkpointSavedNotice) {
userPrompt += checkpointSavedNotice;
console.log("[AutoTracker] ✅ Checkpoint-saved notice appended to this turn's model input (F1 one-shot)");
}
// 🔹 FIX-A (09.10, owner GO): surface a mid-loop threshold crossing from a PREVIOUS turn the same way — F1-style
// straight onto userPrompt (NOT via checkpointSuffix: return paths gate that suffix on hasPendingWarning(), which
// is false here by construction whenever this notice exists — guardMidLoopThreshold deliberately does not touch the
// FSM). takePendingMidLoopNotice() is one-shot INSIDE autoTracker; a throw between its render and the append below
// consumes exactly like the F1 variable (dies with this run) — same loss semantics as pre-#20, no double-injection.
// Independent of checkpointSavedNotice: both can be live in the same run (a crossing armed earlier mid-loop plus a
// checkpoint saved at compression now), and the model gets one short acknowledgement instruction per fact.
const midLoopNotice = autoTracker.takePendingMidLoopNotice();
if (midLoopNotice) {
userPrompt += midLoopNotice;
console.log("[AutoTracker] ✅ Mid-loop crossing notice appended to this turn's model input (FIX-A one-shot)");
}
// 🔹 UNIFIED LANGUAGE v2 (step 4, owner GO): per-turn answer-language instruction — F1-style straight onto
// userPrompt (NOT via checkpointSuffix: its hasPendingWarning()-gated return paths drop notices). Appended once
// per turn after the FIX-A block; rt is guaranteed non-empty for every shipped locale.
const answerLine = fillTemplate(rt.answerLanguageLine, { language: LANGUAGE_DISPLAY_NAMES[turnLang] });
if (answerLine) {
userPrompt += '\n\n' + answerLine;
console.log('[UL-v2] Answer-language line appended (' + turnLang + ')');
}
// 🔹 UPDATE-ANNOUNCE (19.10, owner GO): users with OS/desktop notifications disabled never see the toast — every turn still passes
// through preprocess(), so a pending update is announced IN CHAT here instead: the model tells them and gives them the clickable link.
// F1-style straight onto userPrompt (the checkpointSuffix route would be silently dropped by its hasPendingWarning()-gated return
// paths). getPendingUpdateAnnouncement() latches once per process lifetime per version inside updateChecker → at most one short line
// (~25 tokens) until plugin reload or a newer release; dismissed versions are honored here too (zero network — cached state only).
// SPEC A XOR (21.10): pass the LIVE panel toggle so channel selection reaches the accessor — an explicit OFF makes in-chat THE
// channel (announces even a previously-toasted version, once per process); undefined/ON keeps pre-fix mutual exclusion byte-exactly.
const updateNotice = getPendingUpdateAnnouncement(pluginConfig.get('notificationsEnabled'));
if (updateNotice) {
userPrompt += `\n[AI Toolbox] New version available: v${updateNotice.version} (installed ${LOCAL_VERSION}). Mention this to the user once and give them this link: ${updateNotice.url ?? 'https://github.com/crunch3r/ai_toolbox/releases'}`;
console.log(`[AI Toolbox] [UPDATE-CHECK] In-chat announcement appended to this turn's model input (once per process for v${updateNotice.version})`);
}
} catch (e) {
console.error('[ContextGuard] Auto-compression failed:', e);
} finally {
// 🔹 FIX #20 (A2): single publish point — AFTER any same-turn compression. The tool wrapper's mid-loop
// guard measures growth against this turn-start baseline + model limit until the next preprocess().
// 🔹 FIX #3 (04.10, CTX-FOOTER arc part 3 — publish hardening): moved into FINALLY so a throw between
// resetMidLoopDelta() above and here can no longer skip it: STEP 6 countTokens re-throw, STEP 9
// auto-tracker re-throw, prune/checkpoint/compress/recount failures. Before this fix such a turn ran
// its ENTIRE tool loop with baseline === 0 — mid-loop guard + every ctx_footer silently no-op (3.10
// incident: zero [CTX-FOOTER] lines; the wrapper's one-shot suppression warn names that window).
// finally also covers the early return at "pullHistory() returned undefined". Zero compression behavior
// change: this publishes exactly what the try body computed, and when counting itself failed both values
// are 0 — byte-equivalent state to the old skip (documented in resetMidLoopDelta()).
TokenStatsManager.setTurnEvaluation(tokenCount, maxTokens);
// 🔹 FIX #21 (09.10): same single publish point — the turn's live message list for the wrapper-side forced
// 90% save. null on any throw before its first assignment = safe no-op for that turn (documented above).
TokenStatsManager.setTurnMessages(publishedTurnMessages);
}
}
// 🔹 UNIFIED CHECKPOINT WARNING INJECTION (Fixes missing prompt when RAG disabled or no files found)
// CRITICAL FIX: Previous approach appended plain text trailing content — LLMs reliably ignored it.
// New format uses explicit <SYSTEM_INSTRUCTION> XML-style markers + imperative directives that LLMs process as role-level instructions.
let checkpointSuffix = '';
if (pendingWarning) {
// UNIFIED LANGUAGE v2 (step 4, 27.10): wrapper wording from the per-turn RuntimeTexts (en render byte-identical);
// `<SYSTEM_INSTRUCTION>` markers + '---' separator stay literal here by design — they are structure, not text.
checkpointSuffix = `\n\n<SYSTEM_INSTRUCTION>\n` +
`${rt.checkpointWarningHeader}\n` +
`${pendingWarning}\n` +
`---\n` +
`${rt.checkpointActionRequiredLead}\n` +
// i18n-CONFIRM arc (06.10): wording is language-agnostic on purpose — the parser accepts every shipped
// confirmation word, so reference the options above instead of duplicating one language's words here.
`${rt.checkpointIfConfirmedLine}\n` +
`${rt.checkpointIfDeclinedLine}\n` +
`${rt.checkpointDoNotProceedLine}\n` +
`</SYSTEM_INSTRUCTION>`;
pendingWarning = undefined; // Clear to prevent duplication in Step 1/2
}
// 🔹 PART B-2 (last chance): the pre-compression snapshot did NOT save, but a YES/NO prompt was still live —
// tell the user this turn is the last clean opportunity before history is compressed. (One-shot: consumed now.)
if (lastChanceNote) {
checkpointSuffix += '\n\n⚠️ LAST chance before history is compressed — no session-memory checkpoint could be saved automatically this turn.';
lastChanceNote = false;
}
// Step 0.6: Auto-tracking analysis + checkpoint reply handling
try {
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const autoTrackingEnabled = pluginConfig.get('autoTrackingEnabled') ?? true; // Default to ON
if (autoTrackingEnabled) {
// Update tracker config from plugin settings (also includes token threshold)
autoTracker.updateConfig({
autoTrackingEnabled: true,
autoTrackDecisions: pluginConfig.get('autoTrackDecisions') ?? true,
autoTrackCompletions: pluginConfig.get('autoTrackCompletions') ?? true,
autoTrackErrors: pluginConfig.get('autoTrackErrors') ?? true,
autoSummaryInterval: pluginConfig.get('autoSummaryInterval') ?? 50,
autoTrackTokenThreshold: (pluginConfig.get('autoTrackTokenThreshold')) ?? 75,
});
// 🔹 CHECK FOR YES/NO (or German JA/NEIN) REPLY TO PENDING CHECKPOINT PROMPT — Fix B
if (autoTracker.hasPendingWarning()) {
const replyRaw = userPrompt.trim().toUpperCase();
// Normalize JA/NEIN onto the canonical 'YES' | 'NO' FSM inputs; null = not a confirmation reply.
const replyMatch = normalizeConfirmationReply(userPrompt);
if (replyMatch === 'YES' || replyMatch === 'NO') {
checkpointConsumedThisTurn = true; // One reply serves one prompt — Step 0.7 must not reuse it as a switch confirmation.
console.log(`[AutoTracker] ✅ User replied '${replyRaw}' to checkpoint prompt — processing...`);
autoTracker.processUserReply(replyMatch); // Canonical 'YES' | 'NO' (JA/NEIN normalized above)
// If YES → flush buffered actions + save session memory now
if (replyMatch === 'YES') {
const maxTokens = contextGuard?.getTokenLimit() ?? 0;
// 🔹 FIX: Use direct checkpoint path instead of checkAndSaveTokenThreshold()
// which is designed for independent threshold triggers, not post-confirmation flow.
console.log(`[AutoTracker DEBUG] Direct save path triggered after user confirmation`);
// Flush buffered actions first
const flushedCount = await autoTracker.flushActionsToMemory();
if (flushedCount > 0) {
console.log(`[AutoTracker] ✅ Flushed ${flushedCount} buffered action(s)`);
}
// Then save session memory directly
const saveResult = await autoTracker.autoSaveSessionMemory(
tokenCount,
maxTokens,
messageCount
);
if (saveResult.saved) {
console.log(`[AutoTracker] ✅ Session memory checkpoint saved: ${saveResult.sessionId}`);
} else {
console.error('[AutoTracker] ❌ Checkpoint save failed');
}
}
console.log(`[AutoTracker] ✅ Checkpoint reply consumed — FSM state: ${autoTracker.getState()}`);
}
}
// Analyze user message for tracking triggers (silent background)
const actions = autoTracker.analyzeMessage?.(userPrompt) ?? [];
if (actions.length > 0) {
console.log(`[Auto-Track] Detected ${actions.length} event(s):`, actions.map(a => `${a.type} (${a.confidence.toFixed(2)})`).join(', '));
// 🔹 Actions are now buffered in-memory and will be flushed to persistent storage
// when the token threshold checkpoint fires (with user confirmation)
}
} else {
// Ensure tracker is disabled if config says so
autoTracker.updateConfig({
autoTrackingEnabled: false,
});
}
} catch (e) {
console.error('[Auto-Track] Analysis failed:', e);
}
// Step 0: Always register attachments so tools can access them by name
const allFiles = userMessage.getFiles?.(ctl.client) ?? [];
setAttachments(allFiles);
// Build attachment notice to inject into prompt
let attachmentNotice = '';
if (allFiles.length > 0) {
// UNIFIED LANGUAGE v2 (step 4): title + lead-in from the per-turn RuntimeTexts (en render byte-identical).
const fileNames = listAttachments();
attachmentNotice = `\n\n${rt.attachedFilesTitle}\n${rt.attachedFilesLead}\n${fileNames.map(name => `- ${name}`).join('\n')}`;
}
// Step 0.7: Project keyword detection — check registered projects before path detection
const projectMatch = detectProjectKeyword(userPrompt);
if (projectMatch) {
console.log(`[ProjectAutoDetect] Found registered project "${projectMatch.name}" at ${projectMatch.path}`);
// 🔹 Fix A (v1.9.8+ safety rule in index.ts): NEVER auto-switch on detection alone —
// confirm-first banner; the CWD changes only after an explicit confirmation reply in a later message.
// i18n-CONFIRM arc (06.10): options DISPLAYED in the user's configured language; parsing accepts all shipped words.
const confWords = getConfirmationWords(ctl.getGlobalPluginConfig(configSchematics).get('language'));
const decision = decideProjectSwitch(projectMatch, userPrompt, getWorkingDir(), pendingProjectSwitch);
if (decision.kind === 'skip') {
// Already in the detected project's directory — switching would be a no-op.
if (pendingProjectSwitch && pendingProjectSwitch.name === projectMatch.name) {
console.log(`[ProjectAutoDetect] Already in "${projectMatch.name}" — dropping stale switch offer`);
pendingProjectSwitch = null;
} else {
console.log('[ProjectAutoDetect] Already in target working directory — no switch needed');
}
} else if (decision.kind === 'banner') {
// New detection → inject the confirm-first banner; do NOT change the working directory this turn.
pendingProjectSwitch = { name: projectMatch.name, path: projectMatch.path };
console.log(`[ProjectAutoDetect] Confirmation requested — switch will run only on an explicit YES/JA reply`);
// Include the checkpoint warning only if it is still live after Step 0.6 ran this turn (avoids re-asking a consumed prompt).
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
// UNIFIED LANGUAGE v2 (step 4): banner wording from the per-turn RuntimeTexts; box-drawing separators stay
// literal in this template by design — they sit outside the dict strings.
const projectInjectPrompt = `\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n${rt.projectDetectedTitle}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n${rt.projectFoundLead}\n\n${fillTemplate(rt.projectNameLabel, { name: projectMatch.name })}\n${fillTemplate(rt.projectPathLabel, { path: projectMatch.path })}\n\n${rt.projectSwitchQuestion}\nReply '${confWords.yes}' to confirm, or '${confWords.no}' to decline.\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\nUser's original message:\n${userPrompt}${attachmentNotice}\n`;
return `${projectInjectPrompt.trim()}${activeSuffix}`.trim() + getTemporalSuffix(ctl);
} else if (decision.kind === 'declined') {
console.log(`[ProjectAutoDetect] Switch to "${projectMatch.name}" declined — keeping current working directory`);
pendingProjectSwitch = null;
// No early return — continue with the normal flow below (Step 1/2).
} else {
// decision.kind === 'execute' → explicit YES/JA confirmation: run the one-shot switch now.
if (checkpointConsumedThisTurn) {
console.log(`[ProjectAutoDetect] Reply already consumed as checkpoint confirmation — keeping pending offer for "${projectMatch.name}"`);
// Continue with the normal flow below; re-offer on a later message.
} else {
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
pendingProjectSwitch = null;
console.log(`[ProjectAutoDetect] User confirmed — executing one-shot switch to: ${projectMatch.path}`);
return await executeProjectSwitch(projectMatch.path, userPrompt, attachmentNotice, activeSuffix, ctl, rt);
}
}
} else if (pendingProjectSwitch) {
// No fresh keyword match this turn — a pending offer can only be answered by an explicit reply.
const normalized = normalizeConfirmationReply(userPrompt);
if (checkpointConsumedThisTurn && (normalized === 'YES' || normalized === 'NO')) {
console.log(`[ProjectAutoDetect] Reply already consumed as checkpoint confirmation — keeping pending offer for "${pendingProjectSwitch.name}"`);
// Continue with the normal flow below; re-offer on a later message.
} else if (normalized === 'YES') {
const activeSuffix = autoTracker.hasPendingWarning() ? checkpointSuffix : '';
const target = pendingProjectSwitch;
pendingProjectSwitch = null;
console.log(`[ProjectAutoDetect] User confirmed — executing one-shot switch to: ${target.path}`);
return await executeProjectSwitch(target.path, userPrompt, attachmentNotice, activeSuffix, ctl, rt);
} else if (normalized === 'NO' || normalized === 'NEIN') {
console.log(`[ProjectAutoDetect] Switch to "${pendingProjectSwitch.name}" declined — keeping current working directory`);
pendingProjectSwitch = null;
// Continue with the normal flow below.
} else {
// Non-confirmation reply while an offer is pending → expire it (the "next message" contract).
console.log(`[ProjectAutoDetect] Non-confirmation reply — expiring switch offer for "${pendingProjectSwitch.name}"`);
pendingProjectSwitch = null;
}
}
// Step 1: Directory detection (highest priority)
const detectedPath = detectDirectoryPath(userPrompt);
if (detectedPath) {
// i18n-CONFIRM arc (06.10): the directory banner's reply options also follow the configured language.
const dirWords = getConfirmationWords(ctl.getGlobalPluginConfig(configSchematics).get('language'));
let base = injectWorkingDirectoryPrompt(userPrompt + attachmentNotice, detectedPath, dirWords, rt) + checkpointSuffix; // UL-v2 step-4 completion (28.10): per-turn RuntimeTexts — without rt the helper fell back to getRuntimeTexts('en') and this banner stayed English in all other locales; EN render byte-identical (rt === en bundle when turnLang='en', same object as the old fallback)
return base + getTemporalSuffix(ctl);
}
// Step 2: Document RAG processing (if enabled)
const pluginConfig = ctl.getGlobalPluginConfig(configSchematics);
const documentRAGEnabled = pluginConfig.get('documentRAG');
console.log(`[RAG] documentRAG enabled: ${documentRAGEnabled}`);
if (!documentRAGEnabled) {
// If RAG is disabled, just return the message with attachment notice and checkpoint warning
const base = userPrompt + attachmentNotice + checkpointSuffix;
return base + getTemporalSuffix(ctl);
}
const newFiles = allFiles.filter(f => f.type !== 'image');
console.log(`[RAG] Found ${newFiles.length} non-image files`);
if (newFiles.length === 0) {
const base = userPrompt + attachmentNotice + checkpointSuffix;
return base + getTemporalSuffix(ctl);
}
// Separate PDF files from other file types
const pdfFiles = newFiles.filter(f => f.name.toLowerCase().endsWith('.pdf'));
const otherFiles = newFiles.filter(f => !f.name.toLowerCase().endsWith('.pdf'));
console.log(`[RAG] PDFs: ${pdfFiles.length}, Other: ${otherFiles.length}`);
let allResults: RetrievalResult[] = [];
// Process PDFs with custom local pipeline (more reliable for complex layouts)
if (pdfFiles.length > 0) {
try {
const pdfResults = await retrieveFromPdfs(ctl, userPrompt, pdfFiles);
console.log(`[RAG] PDF retrieval returned ${pdfResults.length} results`);
allResults.push(...pdfResults);
} catch (error) {
console.error('[RAG] Error processing PDFs:', error);
}
}
// Process other files with LM Studio's native retrieval API (handles .txt, .md, etc. natively)
if (otherFiles.length > 0) {
try {
const model = await ctl.client.embedding.model('nomic-ai/nomic-embed-text-v1.5-GGUF', {
signal: ctl.abortSignal,
});
const result = await ctl.client.files.retrieve(userPrompt, otherFiles, {
embeddingModel: model,
limit: pluginConfig.get('retrievalLimit') || 5,
signal: ctl.abortSignal,
});
// Convert high-level API results to our format
const filteredEntries = (result?.entries || []).filter(
entry => entry.score > (pluginConfig.get('retrievalAffinityThreshold') ?? 0.3)
);
console.log(`[RAG] Native retrieval returned ${filteredEntries.length} results`);
allResults.push(...filteredEntries.map(e => ({ content: e.content, score: e.score })));
} catch (error) {
console.error('[RAG] Error retrieving from other files:', error);
}
}
// Sort and limit results
allResults.sort((a, b) => b.score - a.score);
const retrievalLimit = pluginConfig.get('retrievalLimit') || 5;
allResults = allResults.slice(0, retrievalLimit);
console.log(`[RAG] Total results after sorting: ${allResults.length}`);
// 🔹 Inject checkpoint confirmation prompt if triggered/pending from Step 0.5
let finalMessage = userPrompt + attachmentNotice + checkpointSuffix;
// Inject context if results found
if (allResults.length > 0) {
let contextInjection = '';
for (const result of allResults) {
contextInjection += `\n${result.content}\n---\n`;
}
// UNIFIED LANGUAGE v2 (step 4): RAG section header from the per-turn RuntimeTexts (en render byte-identical).
return `${finalMessage}\n\n${rt.ragContextHeader}\n${contextInjection.trim()}` + getTemporalSuffix(ctl);
}
// If no results found, return original message with attachment notice
console.log('[RAG] No relevant results found');
const base = finalMessage;
return base + getTemporalSuffix(ctl);
}