Project Files
dist / index.js
'use strict';
var sdk = require('@lmstudio/sdk');
var generativeAi = require('@google/generative-ai');
var fs = require('fs');
var path = require('path');
var Jimp = require('jimp');
var os = require('os');
var crypto = require('crypto');
var child_process = require('child_process');
var util = require('util');
async function toolsProvider(ctl) {
const tools = [];
return tools;
}
// This file contains the definition of configuration schematics for your plugin.
const configSchematics = sdk.createConfigSchematics()
// 1) Model
.field("model", "select", {
displayName: "Model",
subtitle: "Select the agent model to use for generation.",
options: [
{ value: "gemini-3-pro-image-preview", displayName: "Gemini 3 Pro Image (Nano Banana Pro)" },
{ value: "gemini-3.1-pro-preview", displayName: "Gemini 3.1 Pro" },
],
}, "gemini-3-pro-image-preview")
// 1b) Thinking Level
.field("thinkingLevel", "select", {
displayName: "Thinking Level",
subtitle: "Controls the reasoning depth for Gemini 3 Pro Preview. Currently defaults to 'low'.",
options: [
{ value: "low", displayName: "Low" },
{ value: "medium", displayName: "Medium" },
{ value: "high", displayName: "High" },
],
}, "low")
// 1c) Show Only Last Image Variant
.field("showOnlyLastImageVariant", "boolean", {
displayName: "Show Only Last Image Variant",
subtitle: "For Gemini 3 Pro Image: Hides intermediate reasoning images in chat and promotion, showing/using only the final variant.",
}, true)
// 4) Use Files for Vision
.field("useFilesApiForVision", "boolean", {
displayName: "Use Files for Vision",
subtitle: "Use the Files API for vision tasks (more efficient).",
}, true)
// 5) Debug: Log Chunks
.field("debugChunks", "boolean", {
displayName: "Debug: Log Chunks",
subtitle: "Prints chunk processing and tool-call events to the console.",
engineDoesNotSupport: true,
}, false)
// 6) Debug: Log requests/response
.field("logRequests", "boolean", {
displayName: "Debug: Log requests/response",
subtitle: "Logs full request/response JSON; may include sensitive data.",
engineDoesNotSupport: true,
}, false)
.build();
const globalConfigSchematics = sdk.createConfigSchematics()
// 7) Vision Promotion Mode
.field("visionPromotionPersistent", "boolean", {
displayName: "Vision Promotion: Persistent",
subtitle: "ON: promote attachments and variants every turn. OFF: promote only when new (default).",
}, false)
// 8) API Key
.field("apiKey", "string", {
displayName: "Google AI Studio API Key",
isProtected: true,
placeholder: "AIzaSy...",
}, "")
.build();
// Centralized model capability detection and Files API eligibility
// Policy (current):
// - Strict separation of capabilities per model.
// - "Model Families" are ignored in favor of explicit configuration.
const CAPABILITY_MAP = {
"gemini-3.1-pro-preview": {
supportsTools: true,
supportsVision: true,
supportsImage: false, // Text-only model, no image generation
supportsThinking: true,
thinking: {
levels: ["low", "medium", "high"],
defaultLevel: "low",
},
supportsStreaming: true,
// Text-only: default 2 attachments for vision context
maxPromotedAttachments: 2,
},
"gemini-3-pro-image-preview": {
supportsTools: true, // LM Studio local tools work via function declarations
supportsVision: true,
supportsImage: true,
imageGeneration: {
numberOfImages: 4,
numberOfImagesDefault: 1,
imageSize: ["1K", "2K", "4K"], // "Nano Banana Pro" supports higher res
imageSizeDefault: "1K",
aspectRatio: ["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"],
aspectRatioDefault: "1:1",
// Google Docs: "up to 14 reference images" (6 objects + 5 humans + others)
maxInputImages: 14,
},
supportsThinking: true,
thinking: {
levels: [], // Does not support adjustable thinking levels
defaultLevel: "",
},
supportsStreaming: true,
responseModalities: ["TEXT", "IMAGE"],
// Native image generation: promote ALL attachments (up to 14)
maxPromotedAttachments: 4,
},
};
const DEFAULT_CAPABILITIES = {
supportsTools: false,
supportsVision: true,
supportsImage: false,
supportsThinking: false,
supportsStreaming: true,
maxPromotedAttachments: 2, // Default: only last 2 attachments visually promoted
};
/**
* Detect capabilities for this plugin based on strict model ID lookup.
*/
function detectCapabilities(modelId) {
const m = (modelId || "").toLowerCase();
// Direct lookup first
if (CAPABILITY_MAP[modelId]) {
return CAPABILITY_MAP[modelId];
}
// Fallback lookup (case-insensitive)
const key = Object.keys(CAPABILITY_MAP).find(k => k.toLowerCase() === m);
if (key) {
return CAPABILITY_MAP[key];
}
console.warn(`[Capabilities] Unknown model ID "${modelId}", using defaults.`);
return DEFAULT_CAPABILITIES;
}
/**
* Get the maximum number of attachments to visually promote for a model.
*
* - Image-capable models (Nano Banana, Nano Banana Pro): higher limits (10-14)
* because user may reference any [aN] for image2image operations
* - Text-only or tool-based image models: default 2 (last two attachments)
*/
function getMaxPromotedAttachments(modelId) {
const caps = detectCapabilities(modelId);
return caps.maxPromotedAttachments ?? 2;
}
/**
* Decide if Files API should be used for the given model under current policy.
* - Requires user toggle ON.
* - Kein separates Flash-Gating mehr: alle Modelle folgen dem Toggle.
*/
function shouldUseFilesApiForModel(modelId, userToggle) {
if (!userToggle)
return false;
// Aktuell: Files API für alle Modelle, sobald das Toggle aktiv ist.
return true;
}
// Tool-related helpers extracted from generator.ts to keep behavior unchanged.
// Includes schema conversion, tool gating, name sanitization, and response sanitization.
function convertJsonSchemaToGemini(schema) {
if (!schema || typeof schema !== "object")
return { type: "OBJECT" };
const t = (schema.type || "object").toString().toLowerCase();
switch (t) {
case "string": {
const ds = { type: "STRING" };
if (schema.description)
ds.description = schema.description;
if (Array.isArray(schema.enum))
ds.enum = schema.enum;
return ds;
}
case "number": {
const ds = { type: "NUMBER" };
if (schema.description)
ds.description = schema.description;
return ds;
}
case "integer": {
const ds = { type: "INTEGER" };
if (schema.description)
ds.description = schema.description;
return ds;
}
case "boolean": {
const ds = { type: "BOOLEAN" };
if (schema.description)
ds.description = schema.description;
return ds;
}
case "array": {
const ds = { type: "ARRAY" };
if (schema.description)
ds.description = schema.description;
if (schema.items)
ds.items = convertJsonSchemaToGemini(schema.items);
return ds;
}
case "object":
default: {
const ds = { type: "OBJECT" };
if (schema.description)
ds.description = schema.description;
const props = {};
const required = Array.isArray(schema.required) ? [...schema.required] : [];
if (schema.properties && typeof schema.properties === "object") {
for (const [k, v] of Object.entries(schema.properties)) {
props[k] = convertJsonSchemaToGemini(v);
}
}
if (Object.keys(props).length)
ds.properties = props;
if (required.length)
ds.required = required;
return ds;
}
}
}
function sanitizeToolName(name) {
// Gemini: must start with a letter, then letters/digits/_; max length 64
let n = name || "tool";
n = n.replace(/[^A-Za-z0-9_]/g, "_");
if (!/^[A-Za-z]/.test(n))
n = "t_" + n;
if (n.length > 64)
n = n.slice(0, 64);
return n;
}
function pickRelevantToolDefs(defs, userText) {
if (!Array.isArray(defs) || !userText)
return [];
const t = userText.toLowerCase();
const picked = [];
for (const d of defs) {
const name = String(d?.name ?? d?.toolName ?? "").toLowerCase();
const desc = String(d?.description ?? "").toLowerCase();
const hay = `${name} ${desc}`;
// Simple conservative gating heuristics
const img = /(image|bild|picture|photo|render|zeichnen|draw|paint|image2image)/i;
const search = /(search|find|lookup|look\s*up|suche)/i;
const http = /(fetch|download|http|https|request|url)/i;
const translate = /(translate|übersetz|uebersetz)/i;
const file = /(file|read|write|save|fs|filesystem)/i;
let match = false;
if (img.test(t) && /image|img|draw|render/.test(hay))
match = true;
else if (search.test(t) && /search|find/.test(hay))
match = true;
else if (http.test(t) && /fetch|http|request|download|url/.test(hay))
match = true;
else if (translate.test(t) && /translat|uebersetz|übersetz/.test(hay))
match = true;
else if (file.test(t) && /file|fs|filesystem|read|write/.test(hay))
match = true;
if (match)
picked.push(d);
}
return picked;
}
function buildGeminiTools(ctl, userText) {
const defs = ctl.getToolDefinitions?.();
let gated = pickRelevantToolDefs(defs, userText);
// Fallback: if gating found nothing, include all tool definitions
if ((!gated || gated.length === 0) && Array.isArray(defs) && defs.length) {
gated = defs;
}
const functionDecls = [];
const originalToSafe = new Map();
const safeToOriginal = new Map();
const safeNames = [];
if (Array.isArray(gated) && gated.length) {
for (const d of gated) {
const origName = String(d?.function?.name ?? d?.name ?? d?.toolName ?? "tool");
let safeName = sanitizeToolName(origName);
// ensure uniqueness
let i = 1;
while (safeToOriginal.has(safeName) && safeToOriginal.get(safeName) !== origName) {
const base = safeName.slice(0, Math.max(0, 60));
safeName = `${base}_${++i}`;
}
originalToSafe.set(origName, safeName);
safeToOriginal.set(safeName, origName);
const description = d?.function?.description ?? d?.description ?? "";
const parametersSchema = convertJsonSchemaToGemini(d?.function?.parameters ?? d?.parameters ?? d?.schema ?? { type: "object", properties: {} });
functionDecls.push({
name: safeName,
description,
parameters: parametersSchema,
});
safeNames.push(safeName);
}
}
const tools = functionDecls.length ? [{ functionDeclarations: functionDecls }] : undefined;
return { tools, originalToSafe, safeToOriginal, safeNames };
}
/**
* Pass-through helper for tool results.
* IMPORTANT:
* - If we inject image markdown into the chat ourselves, we must prevent double-rendering.
* In that case we remove tool-returned image objects (incl. $hint/markdown) from the
* payload that is fed back into the model context.
* - We also strip purely logistical Preview/Original path lines.
*/
function sanitizeToolResponseForModel(payload, mode, chatWd) {
if (payload === null || payload === undefined)
return {};
// Robust sanitizer: tool payloads come in many shapes (array, object with content/result/output arrays, nested).
// We remove explicit tool image objects (incl. $hint/markdown) and strip purely logistical Preview/Original lines.
const stripLogisticalText = (item) => {
if (!item || typeof item !== 'object')
return false;
if (item.type !== 'text')
return false;
if (typeof item.text !== 'string')
return false;
const t = item.text.trim();
// Strip "Original vN:" and "Preview vN:" lines (contain file:// paths)
if (/^Original v\d+:/i.test(t) || /^Preview v\d+:/i.test(t))
return true;
// Strip JSON metadata objects containing local paths or sensitive fields
if (t.startsWith('{')) {
try {
const parsed = JSON.parse(t);
const stringified = JSON.stringify(parsed);
// Check for path patterns that leak local filesystem info
if (/lmstudio_attachment:|file:\/\/\/|\/Users\/|\/home\/|C:\\Users\\|\.lmstudio\//i.test(stringified)) {
return true;
}
// Check for known metadata fields that contain local paths
if (parsed.original_png_url || parsed.analysis_preview_url || parsed.source?.includes?.('/Users/')) {
return true;
}
}
catch {
// Not valid JSON - check raw string for path patterns
if (/file:\/\/\/|\/Users\/|\/home\/|C:\\Users\\/i.test(t)) {
return true;
}
}
}
return false;
};
const sanitizeArray = (arr) => {
const hasToolImageObjects = arr.some((entry) => entry && typeof entry === 'object' && entry.type === 'image');
return arr
.filter((entry) => {
if (!entry || typeof entry !== 'object')
return true;
if (hasToolImageObjects && entry.type === 'image')
return false;
if (stripLogisticalText(entry))
return false;
return true;
})
.map(sanitizeAny);
};
const sanitizeObject = (obj) => {
const out = Array.isArray(obj) ? [] : {};
for (const [k, v] of Object.entries(obj)) {
out[k] = sanitizeAny(v);
}
return out;
};
const sanitizeAny = (v) => {
if (v === null || v === undefined)
return v;
if (Array.isArray(v))
return sanitizeArray(v);
if (typeof v === 'object')
return sanitizeObject(v);
// If it's a JSON-encoded string containing an array/object, try to sanitize it too.
if (typeof v === 'string') {
const s = v.trim();
if ((s.startsWith('[') && s.endsWith(']')) || (s.startsWith('{') && s.endsWith('}'))) {
try {
const parsed = JSON.parse(s);
const sanitized = sanitizeAny(parsed);
return sanitized;
}
catch {
return v;
}
}
}
return v;
};
try {
return sanitizeAny(payload);
}
catch {
return payload;
}
}
function toIsoLikeTimestamp(d) {
// Compact form: 20251028T164247509Z (no dashes/colons)
const iso = d.toISOString(); // 2025-10-28T16:42:47.509Z
// yyyy-mm-ddThh:MM:ss.sssZ -> yyyymmddThhMMsssssZ
const [datePart, timePartWithZ] = iso.split("T");
const date = datePart.replace(/-/g, ""); // yyyymmdd
const timePart = timePartWithZ.replace("Z", ""); // hh:MM:ss.sss
const time = timePart.replace(/:/g, "").replace(".", ""); // hhMMssssss
return `${date}T${time}Z`;
}
async function encodeJpegFromBuffer(buf, quality = 85) {
const img = await Jimp.read(buf);
img.quality(quality);
return await img.getBufferAsync(Jimp.MIME_JPEG);
}
async function resizeMaxDimJpegFromFile(srcAbs, maxDim = 1024, quality = 85) {
const img = await Jimp.read(srcAbs);
const w = img.bitmap.width;
const h = img.bitmap.height;
const maxSide = Math.max(w, h);
if (maxSide > maxDim) {
const scale = maxDim / maxSide;
const newW = Math.max(1, Math.round(w * scale));
const newH = Math.max(1, Math.round(h * scale));
img.resize(newW, newH);
}
img.quality(quality);
return await img.getBufferAsync(Jimp.MIME_JPEG);
}
async function copyFile(srcAbs, dstAbs) {
await fs.promises.mkdir(path.dirname(dstAbs), { recursive: true });
await fs.promises.copyFile(srcAbs, dstAbs);
}
function fileUriToPath(uri) {
try {
const p = decodeURIComponent(uri.replace(/^file:\/\//i, "").replace(/^(?!\/)/, "/"));
return p;
}
catch {
return null;
}
}
function replacePrefixGeneratedToAnalysis(fileName) {
if (fileName.startsWith("generated-image-") && fileName.endsWith(".png")) {
const stem = fileName.slice("generated-image-".length, -".png".length);
return `analysis-generated-image-${stem}.jpg`;
}
// Fallback: add analysis- prefix and .jpg
const base = fileName.replace(/\.[^.]+$/, "");
return `analysis-${base}.jpg`;
}
const FILE_NAME = "chat_media_state.json";
// ============================================================================
// Read/Write with atomic writes
// ============================================================================
async function readChatMediaState(chatWd) {
const p = path.join(chatWd, FILE_NAME);
try {
const raw = await fs.promises.readFile(p, "utf8");
const parsed = JSON.parse(raw);
return normalizeState(parsed);
}
catch {
return { attachments: [], variants: [], counters: {} };
}
}
function normalizeState(s) {
return {
attachments: Array.isArray(s?.attachments) ? s.attachments : [],
variants: Array.isArray(s?.variants) ? s.variants : [],
lastEvent: s?.lastEvent,
counters: typeof s?.counters === "object" && s?.counters ? s.counters : {},
injectedMarkdown: Array.isArray(s?.injectedMarkdown) ? s.injectedMarkdown : undefined,
lastVariantsTs: typeof s?.lastVariantsTs === "string" ? s.lastVariantsTs : undefined,
lastPromotedTs: typeof s?.lastPromotedTs === "string" ? s.lastPromotedTs : undefined,
lastPromotedAttachmentN: typeof s?.lastPromotedAttachmentN === "number" ? s.lastPromotedAttachmentN : undefined,
};
}
/**
* Atomic write: write to tmp file then rename to prevent corruption
*/
async function writeChatMediaStateAtomic(chatWd, state) {
await fs.promises.mkdir(chatWd, { recursive: true });
const tmp = path.join(chatWd, `${FILE_NAME}.tmp`);
const dst = path.join(chatWd, FILE_NAME);
const json = JSON.stringify(state, null, 2);
await fs.promises.writeFile(tmp, json, "utf8");
await fs.promises.rename(tmp, dst);
}
/** @deprecated Use writeChatMediaStateAtomic instead */
async function writeChatMediaState(chatWd, state) {
return writeChatMediaStateAtomic(chatWd, state);
}
async function recordVariantsProvision(chatWd, variants) {
const state = await readChatMediaState(chatWd);
const list = variants.map((v, idx) => ({
filename: v.filename,
preview: v.preview,
originAbs: v.originAbs,
createdAt: v.createdAt ?? new Date().toISOString(),
v: v.v ?? (idx + 1),
}));
const nextState = {
...state,
variants: list,
lastEvent: { type: "variants", at: new Date().toISOString() },
counters: { ...(state.counters ?? {}), nextVariantV: list.length + 1 },
};
await writeChatMediaStateAtomic(chatWd, nextState);
return nextState;
}
async function buildPromotionPartsA(params) {
const { chatWd } = params;
const promoParts = [];
const promotedFiles = [];
try {
const state = await readChatMediaState(chatWd).catch(() => ({ attachments: [], variants: [] }));
if (Array.isArray(state.attachments) && state.attachments.length) {
for (let i = 0; i < state.attachments.length; i++) {
const a = state.attachments[i];
const rel = a.filename; // Mode A uses original, not analysis preview
const abs = path.join(chatWd, rel);
const exists = await fs.promises.stat(abs).then(() => true).catch(() => false);
if (exists) {
const toMime = (fn) => (/\.jpe?g$/i.test(fn) ? 'image/jpeg' :
/\.png$/i.test(fn) ? 'image/png' :
/\.webp$/i.test(fn) ? 'image/webp' :
/\.gif$/i.test(fn) ? 'image/gif' :
/\.bmp$/i.test(fn) ? 'image/bmp' :
/\.svg$/i.test(fn) ? 'image/svg+xml' :
/\.tiff?$/i.test(fn) ? 'image/tiff' :
/\.heic$/i.test(fn) ? 'image/heic' : 'image/jpeg');
const buf = await fs.promises.readFile(abs);
const label = state.attachments.length > 1 ? `Attachment (A${i + 1})` : 'Attachment (A)';
promoParts.push({ text: label });
promoParts.push({ inlineData: { data: buf.toString('base64'), mimeType: toMime(rel) } });
promotedFiles.push(path.basename(abs));
}
}
}
}
catch (e) {
// Soft-fail: no promotion in Mode A
}
return { promoParts, promotedFiles };
}
// ============================================================================
// NEW: buildPromotionItems - based on standalone_generator_guide/promotion.ts
// ============================================================================
/**
* Build a list of items to inject as base64 into the model context.
* Uses stable n-field for labels, NOT array index!
*
* @param chatWd - Chat working directory
* @param state - Current media state
* @param options - Configuration options
* @returns Array of PromotionItem with abs, previewAbs, and label
*/
function buildPromotionItems(chatWd, state, { labels = true, maxAttachmentItems = 2, maxVariantItems = 3, } = {}) {
const items = [];
const attachments = Array.isArray(state.attachments) ? state.attachments : [];
// Sort by n-value (ascending) to ensure chronological order for Rolling Window
const sortedAttachments = [...attachments].sort((a, b) => (a.n ?? 0) - (b.n ?? 0));
const cap = Math.max(0, Math.floor(maxAttachmentItems));
// Take the LAST N (highest n-values = most recent attachments)
const cappedAttachments = cap > 0 ? sortedAttachments.slice(-cap) : [];
for (const a of cappedAttachments) {
if (!a)
continue;
// CRITICAL: Use stable n-field, NOT sequential index!
const stableN = typeof a.n === "number" ? a.n : 0;
// originAbs is the source (no copies!)
const abs = typeof a.originAbs === "string" && a.originAbs ? a.originAbs : "";
const pRel = typeof a.preview === "string" && a.preview ? a.preview : "";
const pAbs = pRel ? path.join(chatWd, pRel) : "";
if (!abs) {
throw new Error(`Attachment a${stableN} is missing originAbs (origin=${a.origin})`);
}
if (!pAbs) {
throw new Error(`Attachment a${stableN} is missing preview (origin=${a.origin}, originAbs=${a.originAbs})`);
}
// Build label with stable n and originalName
let label;
if (labels) {
const originalName = a.originalName || a.origin || `attachment-${stableN}`;
label = `Attachment [a${stableN}] ${originalName}`;
}
items.push({ abs, previewAbs: pAbs, label });
}
// Variants
const variants = [...(state.variants || [])];
variants.sort((x, y) => x.v - y.v || x.createdAt.localeCompare(y.createdAt));
const cappedVariants = variants.slice(0, Math.max(0, Math.floor(maxVariantItems)));
for (const v of cappedVariants) {
const abs = path.join(chatWd, v.filename);
const pAbs = path.join(chatWd, v.preview);
items.push({
abs,
previewAbs: pAbs,
label: labels ? `Generated Image [v${v.v}]` : undefined,
});
}
return items;
}
/**
* Convert promotion items to Gemini inlineData parts (base64)
*/
async function toGeminiInlineDataParts(items) {
const parts = [];
const guessMime = (fn) => (/\.jpe?g$/i.test(fn) ? "image/jpeg"
: /\.png$/i.test(fn) ? "image/png"
: /\.webp$/i.test(fn) ? "image/webp"
: /\.gif$/i.test(fn) ? "image/gif"
: "image/jpeg");
for (const it of items) {
if (it.label) {
parts.push({ text: it.label });
}
const buf = await fs.promises.readFile(it.previewAbs);
const mime = guessMime(it.previewAbs);
parts.push({ inlineData: { data: buf.toString("base64"), mimeType: mime } });
}
return parts;
}
/**
* Check if vision promotion should happen based on idempotency tracking
*/
function shouldPromoteImages(state, persistentMode) {
const attachments = Array.isArray(state.attachments) ? state.attachments : [];
const variants = Array.isArray(state.variants) ? state.variants : [];
const attachmentMaxN = attachments.length > 0
? Math.max(0, ...attachments.map((a) => a.n ?? 0))
: 0;
const lastPromotedN = state.lastPromotedAttachmentN ?? 0;
// Extract timestamp group for idempotency.
// Prefer explicit state.lastVariantsTs (tool-generated variants may not keep originals in chatWd).
let currentVariantsTs = state.lastVariantsTs;
if (!currentVariantsTs && variants.length > 0) {
for (const v of variants) {
const m = /^generated-image-(.+)-v\d+\.\w+$/i.exec(v.filename);
if (m) {
currentVariantsTs = m[1];
break;
}
}
}
const lastPromotedTs = state.lastPromotedTs;
if (persistentMode) {
// In persistent mode: always promote if there's something to promote
return {
shouldPromoteAttachment: attachments.length > 0,
shouldPromoteVariants: variants.length > 0,
};
}
// Idempotent mode: only promote if new
return {
shouldPromoteAttachment: attachmentMaxN > 0 && attachmentMaxN !== lastPromotedN,
shouldPromoteVariants: currentVariantsTs !== undefined && currentVariantsTs !== lastPromotedTs,
};
}
/**
* Update idempotency tracking after promotion
*/
async function markAsPromoted(chatWd, state, promotedAttachment, promotedVariants) {
let changed = false;
if (promotedAttachment) {
const attachments = Array.isArray(state.attachments) ? state.attachments : [];
const maxN = attachments.length > 0 ? Math.max(0, ...attachments.map((a) => a.n ?? 0)) : 0;
if (maxN > 0) {
state.lastPromotedAttachmentN = maxN;
changed = true;
}
}
if (promotedVariants) {
// Prefer explicit lastVariantsTs (set by tool harvesting pipeline)
if (typeof state.lastVariantsTs === "string" && state.lastVariantsTs) {
state.lastPromotedTs = state.lastVariantsTs;
changed = true;
}
else {
const variants = Array.isArray(state.variants) ? state.variants : [];
if (variants.length > 0) {
const m = /^generated-image-(.+)-v\d+\.\w+$/i.exec(variants[0].filename);
if (m) {
state.lastPromotedTs = m[1];
changed = true;
}
}
}
}
if (changed) {
await writeChatMediaStateAtomic(chatWd, state);
}
}
// ============================================================================
// Tool-generated variants harvesting (NO copies, NO preview generation)
// ============================================================================
function parseGeneratedTsFromBasename(base) {
const m = /^generated-image-(.+)-v\d+\.\w+$/i.exec(base);
return m ? m[1] : undefined;
}
/**
* Harvest tool-generated variants from the latest tool message and record them into chat_media_state.json.
* - Uses preview JPEGs already written to chatWd by the tool (e.g. image-<epoch>-1.jpg)
* - Uses original abs path from tool text lines (e.g. Original v1: file:///.../generated-image-...-v1.png)
* - Does NOT create previews and does NOT copy originals into chatWd.
* - Optionally injects the provided markdown into the chat output (idempotent via state.injectedMarkdown).
*/
async function harvestToolGeneratedVariantsFromLatestToolMessage(ctl, history, chatWd, debug = false) {
const previewByV = new Map();
const markdownByV = new Map();
const originAbsByV = new Map();
let lastVariantsTs;
const parseVFromBasename = (bn) => {
const m = /-(\d+)\.(?:jpe?g|png|webp|gif|bmp|tiff?|heic)$/i.exec(bn);
if (!m)
return undefined;
const n = parseInt(m[1], 10);
return Number.isFinite(n) && n > 0 ? n : undefined;
};
const extractPreviewBasenameFromMarkdown = (md) => {
try {
const m = /\(\.\/([^\)\s]+\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))\)/i.exec(md);
return m ? m[1] : null;
}
catch {
return null;
}
};
let lastToolMsg;
for (const msg of Array.from(history).reverse()) {
if (msg?.getRole?.() === "tool") {
lastToolMsg = msg;
break;
}
}
const lastAssistantText = () => {
try {
for (const msg of Array.from(history).reverse()) {
if (msg?.getRole?.() !== "assistant")
continue;
const t = msg?.getText?.();
if (typeof t === "string" && t.trim())
return t;
}
}
catch { }
return "";
};
const parseAssistantForVariants = (text) => {
if (!text || typeof text !== "string")
return;
// Extract ./image-....-N.jpg from markdown
const reMd = /!\[[^\]]*\]\(\.\/([^\)\s]+\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))\)/gi;
let m;
while ((m = reMd.exec(text)) !== null) {
const bn = m[1];
const vNum = parseVFromBasename(bn);
if (vNum) {
previewByV.set(vNum, bn);
markdownByV.set(vNum, m[0]);
}
}
// Extract Preview/Original lines
const lines = text.split(/\r?\n/);
for (const line of lines) {
const t = line.trim();
const mPrev = /^Preview v(\d+):\s*(?:\.\/)?([^\s]+\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))$/i.exec(t);
if (mPrev) {
const vNum = parseInt(mPrev[1], 10);
if (Number.isFinite(vNum) && vNum > 0)
previewByV.set(vNum, mPrev[2]);
}
const mOrig = /^Original v(\d+):\s*(file:\/\/\S+)$/i.exec(t);
if (mOrig) {
const vNum = parseInt(mOrig[1], 10);
const abs = fileUriToPath(mOrig[2]);
if (abs) {
originAbsByV.set(vNum, abs);
const ts = parseGeneratedTsFromBasename(path.basename(abs));
if (ts)
lastVariantsTs = ts;
}
}
}
};
if (!lastToolMsg || typeof lastToolMsg.getToolCallResults !== "function") {
// Deterministic secondary source: assistant message text.
const aText = lastAssistantText();
if (!aText) {
return { changedState: false, injectedMarkdown: false, source: "none", reason: "no-tool-message-and-no-assistant-text", foundVariants: 0, recordedVariants: 0 };
}
parseAssistantForVariants(aText);
// Continue into recording flow below.
}
const results = lastToolMsg && typeof lastToolMsg.getToolCallResults === "function"
? lastToolMsg.getToolCallResults()
: [];
const considerArrayPayload = (arr) => {
let fallbackV = 0;
for (const it of arr) {
if (!it || typeof it !== "object")
continue;
if (it.type === "image" && typeof it.fileName === "string") {
// Prefer explicit v from suffix "-N.ext" in the preview filename
let vNum;
const m = /-(\d+)\.(?:jpe?g|png|webp|gif|bmp)$/i.exec(it.fileName);
if (m)
vNum = parseInt(m[1], 10);
if (!vNum || Number.isNaN(vNum))
vNum = ++fallbackV;
previewByV.set(vNum, it.fileName);
if (typeof it.markdown === "string")
markdownByV.set(vNum, it.markdown);
continue;
}
// Some tools only provide markdown in image objects.
if (it.type === "image" && typeof it.markdown === "string") {
const bn = extractPreviewBasenameFromMarkdown(it.markdown);
if (bn) {
const vNum = parseVFromBasename(bn) ?? ++fallbackV;
previewByV.set(vNum, bn);
markdownByV.set(vNum, it.markdown);
}
continue;
}
if (it.type === "text" && typeof it.text === "string") {
const t = it.text.trim();
const mPrev = /^Preview v(\d+):\s*(?:\.\/)?([^\s]+\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))$/i.exec(t);
if (mPrev) {
const vNum = parseInt(mPrev[1], 10);
if (Number.isFinite(vNum) && vNum > 0) {
previewByV.set(vNum, mPrev[2]);
}
}
const mOrig = /^Original v(\d+):\s*(file:\/\/\S+)$/i.exec(t);
if (mOrig) {
const vNum = parseInt(mOrig[1], 10);
const abs = fileUriToPath(mOrig[2]);
if (abs) {
originAbsByV.set(vNum, abs);
const ts = parseGeneratedTsFromBasename(path.basename(abs));
if (ts)
lastVariantsTs = ts;
}
}
// If the tool gave us markdown inline in a text entry, capture it too.
const mdBn = extractPreviewBasenameFromMarkdown(t);
if (mdBn) {
const vNum = parseVFromBasename(mdBn) ?? ++fallbackV;
previewByV.set(vNum, mdBn);
// Store the full markdown string if it looks like markdown
if (t.includes(")
markdownByV.set(vNum, t);
}
}
}
};
const extractArrayPayloads = (payload) => {
const out = [];
const walk = (v) => {
if (!v)
return;
if (Array.isArray(v)) {
out.push(v);
return;
}
if (typeof v === "string") {
const s = v.trim();
if (s.startsWith("[") && s.endsWith("]")) {
try {
const parsed = JSON.parse(s);
if (Array.isArray(parsed))
out.push(parsed);
}
catch { /* ignore */ }
}
return;
}
if (typeof v !== "object")
return;
const obj = v;
const candidates = [
obj.content,
obj.result,
obj.output,
obj.data,
obj.items,
obj.response,
];
for (const c of candidates) {
if (Array.isArray(c))
out.push(c);
else if (typeof c === "string") {
const s = c.trim();
if (s.startsWith("[") && s.endsWith("]")) {
try {
const parsed = JSON.parse(s);
if (Array.isArray(parsed))
out.push(parsed);
}
catch { /* ignore */ }
}
}
}
};
walk(payload);
return out;
};
// Full DFS to catch shapes like: { content: [ { type:"toolCallResult", content:"[ ... ]" } ] }
const collectEmbeddedToolItemArrays = (payload) => {
const out = [];
const seen = new Set();
const visit = (v) => {
if (v === null || v === undefined)
return;
if (seen.has(v))
return;
if (typeof v === "object")
seen.add(v);
if (Array.isArray(v)) {
out.push(v);
for (const it of v)
visit(it);
return;
}
if (typeof v === "string") {
const s = v.trim();
if (s.startsWith("[") && s.endsWith("]")) {
try {
const parsed = JSON.parse(s);
if (Array.isArray(parsed))
out.push(parsed);
}
catch { /* ignore */ }
}
return;
}
if (typeof v !== "object")
return;
for (const val of Object.values(v))
visit(val);
};
visit(payload);
return out;
};
if (Array.isArray(results) && results.length > 0) {
for (const r of results) {
let payload = r?.content ?? r?.result ?? r?.output ?? null;
if (typeof payload === "string") {
try {
payload = JSON.parse(payload);
}
catch { /* ignore */ }
}
if (Array.isArray(payload)) {
considerArrayPayload(payload);
}
else {
for (const arr of extractArrayPayloads(payload))
considerArrayPayload(arr);
for (const arr of collectEmbeddedToolItemArrays(payload))
considerArrayPayload(arr);
}
}
}
if (previewByV.size === 0 && originAbsByV.size === 0) {
const hasTool = Array.isArray(results) && results.length > 0;
return {
changedState: false,
injectedMarkdown: false,
source: hasTool ? "tool" : "assistant",
reason: "no-variant-markers-found",
foundVariants: 0,
recordedVariants: 0,
};
}
const vNums = Array.from(new Set([...previewByV.keys(), ...originAbsByV.keys()]))
.filter((n) => typeof n === "number" && n > 0)
.sort((a, b) => a - b)
.slice(0, 3);
const nowIso = new Date().toISOString();
const harvested = vNums.map((vNum) => {
const originAbs = originAbsByV.get(vNum);
const preview = previewByV.get(vNum);
if (!preview) {
// No preview means we cannot display/promote in Base64 mode.
// Keep state untouched (strict) rather than inventing a preview.
return null;
}
return {
v: vNum,
filename: originAbs ? path.basename(originAbs) : preview,
preview,
originAbs,
createdAt: nowIso,
};
}).filter(Boolean);
if (harvested.length === 0) {
const hasTool = Array.isArray(results) && results.length > 0;
return { changedState: false, injectedMarkdown: false, source: hasTool ? "tool" : "assistant", reason: "no-preview-for-any-variant", foundVariants: vNums.length, recordedVariants: 0 };
}
const state = await readChatMediaState(chatWd);
const injectedSet = new Set(Array.isArray(state.injectedMarkdown) ? state.injectedMarkdown : []);
let injected = false;
if (ctl) {
for (const v of harvested) {
if (injectedSet.has(v.preview))
continue;
const md = markdownByV.get(v.v) || ``;
ctl.fragmentGenerated(`\n\n${md}\n\n`);
injectedSet.add(v.preview);
injected = true;
}
}
const nextState = {
...state,
variants: harvested,
injectedMarkdown: injectedSet.size ? Array.from(injectedSet) : undefined,
lastVariantsTs: lastVariantsTs ?? state.lastVariantsTs,
lastEvent: { type: "variants", at: nowIso },
counters: { ...(state.counters ?? {}), nextVariantV: harvested.length + 1 },
};
const changedState = JSON.stringify(state.variants) !== JSON.stringify(nextState.variants)
|| state.lastVariantsTs !== nextState.lastVariantsTs
|| JSON.stringify(state.injectedMarkdown || []) !== JSON.stringify(nextState.injectedMarkdown || []);
if (changedState) {
await writeChatMediaStateAtomic(chatWd, nextState);
if (debug) {
try {
console.info(`[Tool Variants] Recorded ${harvested.length} variant(s); lastVariantsTs=${nextState.lastVariantsTs ?? ""}`);
}
catch { }
}
}
const hasTool = Array.isArray(results) && results.length > 0;
return {
changedState,
injectedMarkdown: injected,
source: hasTool ? "tool" : "assistant",
reason: changedState ? "recorded" : "no-state-change",
foundVariants: vNums.length,
recordedVariants: harvested.length,
};
}
// Recover Pro variants by parsing the last tool message JSON and copying originals into workdir, creating analysis previews,
// then recording them in chat_media_state.json
async function recoverProVariantsFromHistory(history, chatWd, debug = false, shouldUseFilesApi = false) {
let lastToolMsg;
for (const msg of Array.from(history).reverse()) {
if (msg.getRole && msg.getRole() === "tool") {
lastToolMsg = msg;
break;
}
}
if (!lastToolMsg || typeof lastToolMsg.getToolCallResults !== "function") {
if (debug)
console.warn("[Recover] No tool message for Pro variants");
return;
}
const results = lastToolMsg.getToolCallResults();
if (!Array.isArray(results) || results.length === 0)
return;
const rawParts = [];
for (const r of results) {
const cand = r?.content ?? r?.result ?? r?.output ?? null;
if (typeof cand === "string")
rawParts.push(cand);
}
if (!rawParts.length)
return;
let arr = null;
for (const s of rawParts) {
try {
const parsed = JSON.parse(s);
if (Array.isArray(parsed)) {
arr = parsed;
break;
}
}
catch { }
}
if (!arr)
return;
const imageEntries = [];
const originalsFromFiles = [];
const originalByV = {};
for (const it of arr) {
if (it && it.type === "image") {
imageEntries.push({ fileName: it.fileName, markdown: it.markdown });
}
else if (it && it.type === "text" && typeof it.text === "string") {
try {
const inner = JSON.parse(it.text);
const files = inner?.files;
if (files && typeof files === "object") {
const orig = files.original;
if (typeof orig === "string")
originalsFromFiles.push(orig);
else if (Array.isArray(orig))
originalsFromFiles.push(...orig.filter((x) => typeof x === "string"));
}
}
catch {
// Fallback: parse human-readable "Original vN: file://..." lines only
const text = it.text;
try {
const reOrig = /Original\s*v\s*(\d+)\s*:\s*(file:\/\/\S+)/gi;
let m;
while ((m = reOrig.exec(text)) !== null) {
const v = parseInt(m[1], 10);
if (v >= 1 && v <= 99)
originalByV[v] = m[2];
}
}
catch { }
}
}
}
// Consolidate originals from JSON and v-maps, preserving v-order if available
const orderedOriginals = [];
const byVKeys = Object.keys(originalByV).map(n => parseInt(n, 10)).filter(n => !Number.isNaN(n)).sort((a, b) => a - b);
for (const v of byVKeys)
orderedOriginals.push(originalByV[v]);
for (const o of originalsFromFiles)
if (!orderedOriginals.includes(o))
orderedOriginals.push(o);
if (orderedOriginals.length === 0)
return;
const wd = chatWd;
const previewPairs = [];
for (const origUri of orderedOriginals) {
const srcPng = fileUriToPath(origUri);
if (!srcPng)
continue;
const srcBase = path.basename(srcPng);
const dstPng = path.join(wd, srcBase);
try {
await copyFile(srcPng, dstPng);
if (debug)
console.info("[Recover] Copied original:", srcPng, "->", dstPng);
}
catch (e) {
if (debug)
console.warn("[Recover] Copy failed:", e.message);
continue;
}
try {
if (!shouldUseFilesApi) {
const analysisName = replacePrefixGeneratedToAnalysis(srcBase);
const analysisAbs = path.join(wd, analysisName);
const jpeg = await encodeJpegFromBuffer(await fs.promises.readFile(dstPng), 85);
await fs.promises.writeFile(analysisAbs, jpeg);
// Use analysis JPEG as preview only in Base64 mode (B)
previewPairs.push({ filename: srcBase, preview: analysisName });
}
else {
// In GCS mode (C), keep original PNGs as previews and avoid any JPEGs
previewPairs.push({ filename: srcBase, preview: srcBase });
}
}
catch (e) {
if (debug)
console.warn("[Recover] Analysis/preview generation failed:", e.message);
continue;
}
}
if (previewPairs.length) {
await recordVariantsProvision(wd, previewPairs);
}
}
// (additional vision helpers removed as part of cleanup)
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Basic error type for this SDK.
* @public
*/
class GoogleGenerativeAIError extends Error {
constructor(message) {
super(`[GoogleGenerativeAI Error]: ${message}`);
}
}
/**
* Error class covering HTTP errors when calling the server. Includes HTTP
* status, statusText, and optional details, if provided in the server response.
* @public
*/
class GoogleGenerativeAIFetchError extends GoogleGenerativeAIError {
constructor(message, status, statusText, errorDetails) {
super(message);
this.status = status;
this.statusText = statusText;
this.errorDetails = errorDetails;
}
}
/**
* Errors in the contents of a request originating from user input.
* @public
*/
class GoogleGenerativeAIRequestInputError extends GoogleGenerativeAIError {
}
/**
* Error thrown when a request is aborted, either due to a timeout or
* intentional cancellation by the user.
* @public
*/
class GoogleGenerativeAIAbortError extends GoogleGenerativeAIError {
}
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com";
const DEFAULT_API_VERSION = "v1beta";
/**
* We can't `require` package.json if this runs on web. We will use rollup to
* swap in the version number here at build time.
*/
const PACKAGE_VERSION = "0.24.1";
const PACKAGE_LOG_HEADER = "genai-js";
var Task;
(function (Task) {
Task["GENERATE_CONTENT"] = "generateContent";
Task["STREAM_GENERATE_CONTENT"] = "streamGenerateContent";
Task["COUNT_TOKENS"] = "countTokens";
Task["EMBED_CONTENT"] = "embedContent";
Task["BATCH_EMBED_CONTENTS"] = "batchEmbedContents";
})(Task || (Task = {}));
/**
* Simple, but may become more complex if we add more versions to log.
*/
function getClientHeaders(requestOptions) {
const clientHeaders = [];
if (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.apiClient) {
clientHeaders.push(requestOptions.apiClient);
}
clientHeaders.push(`${PACKAGE_LOG_HEADER}/${PACKAGE_VERSION}`);
return clientHeaders.join(" ");
}
async function makeRequest(url, fetchOptions, fetchFn = fetch) {
let response;
try {
response = await fetchFn(url, fetchOptions);
}
catch (e) {
handleResponseError(e, url);
}
if (!response.ok) {
await handleResponseNotOk(response, url);
}
return response;
}
function handleResponseError(e, url) {
let err = e;
if (err.name === "AbortError") {
err = new GoogleGenerativeAIAbortError(`Request aborted when fetching ${url.toString()}: ${e.message}`);
err.stack = e.stack;
}
else if (!(e instanceof GoogleGenerativeAIFetchError ||
e instanceof GoogleGenerativeAIRequestInputError)) {
err = new GoogleGenerativeAIError(`Error fetching from ${url.toString()}: ${e.message}`);
err.stack = e.stack;
}
throw err;
}
async function handleResponseNotOk(response, url) {
let message = "";
let errorDetails;
try {
const json = await response.json();
message = json.error.message;
if (json.error.details) {
message += ` ${JSON.stringify(json.error.details)}`;
errorDetails = json.error.details;
}
}
catch (e) {
// ignored
}
throw new GoogleGenerativeAIFetchError(`Error fetching from ${url.toString()}: [${response.status} ${response.statusText}] ${message}`, response.status, response.statusText, errorDetails);
}
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var RpcTask;
(function (RpcTask) {
RpcTask["UPLOAD"] = "upload";
RpcTask["LIST"] = "list";
RpcTask["GET"] = "get";
RpcTask["DELETE"] = "delete";
RpcTask["UPDATE"] = "update";
RpcTask["CREATE"] = "create";
})(RpcTask || (RpcTask = {}));
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const taskToMethod = {
[RpcTask.UPLOAD]: "POST",
[RpcTask.LIST]: "GET",
[RpcTask.GET]: "GET",
[RpcTask.DELETE]: "DELETE",
[RpcTask.UPDATE]: "PATCH",
[RpcTask.CREATE]: "POST",
};
class ServerRequestUrl {
constructor(task, apiKey, requestOptions) {
this.task = task;
this.apiKey = apiKey;
this.requestOptions = requestOptions;
}
appendPath(path) {
this._url.pathname = this._url.pathname + `/${path}`;
}
appendParam(key, value) {
this._url.searchParams.append(key, value);
}
toString() {
return this._url.toString();
}
}
class FilesRequestUrl extends ServerRequestUrl {
constructor(task, apiKey, requestOptions) {
var _a, _b;
super(task, apiKey, requestOptions);
this.task = task;
this.apiKey = apiKey;
this.requestOptions = requestOptions;
const apiVersion = ((_a = this.requestOptions) === null || _a === void 0 ? void 0 : _a.apiVersion) || DEFAULT_API_VERSION;
const baseUrl = ((_b = this.requestOptions) === null || _b === void 0 ? void 0 : _b.baseUrl) || DEFAULT_BASE_URL;
let initialUrl = baseUrl;
if (this.task === RpcTask.UPLOAD) {
initialUrl += `/upload`;
}
initialUrl += `/${apiVersion}/files`;
this._url = new URL(initialUrl);
}
}
function getHeaders(url) {
var _a;
const headers = new Headers();
headers.append("x-goog-api-client", getClientHeaders(url.requestOptions));
headers.append("x-goog-api-key", url.apiKey);
let customHeaders = (_a = url.requestOptions) === null || _a === void 0 ? void 0 : _a.customHeaders;
if (customHeaders) {
if (!(customHeaders instanceof Headers)) {
try {
customHeaders = new Headers(customHeaders);
}
catch (e) {
throw new GoogleGenerativeAIRequestInputError(`unable to convert customHeaders value ${JSON.stringify(customHeaders)} to Headers: ${e.message}`);
}
}
for (const [headerName, headerValue] of customHeaders.entries()) {
if (headerName === "x-goog-api-key") {
throw new GoogleGenerativeAIRequestInputError(`Cannot set reserved header name ${headerName}`);
}
else if (headerName === "x-goog-api-client") {
throw new GoogleGenerativeAIRequestInputError(`Header name ${headerName} can only be set using the apiClient field`);
}
headers.append(headerName, headerValue);
}
}
return headers;
}
async function makeServerRequest(url, headers, body, fetchFn = fetch) {
const requestInit = {
method: taskToMethod[url.task],
headers,
};
if (body) {
requestInit.body = body;
}
const signal = getSignal(url.requestOptions);
if (signal) {
requestInit.signal = signal;
}
return makeRequest(url.toString(), requestInit, fetchFn);
}
/**
* Create an AbortSignal based on the timeout and signal in the
* RequestOptions.
*/
function getSignal(requestOptions) {
if ((requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.signal) !== undefined || (requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeout) >= 0) {
const controller = new AbortController();
if ((requestOptions === null || requestOptions === void 0 ? void 0 : requestOptions.timeout) >= 0) {
setTimeout(() => controller.abort(), requestOptions.timeout);
}
if (requestOptions.signal) {
requestOptions.signal.addEventListener("abort", () => {
controller.abort();
});
}
return controller.signal;
}
}
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Class for managing GoogleAI file uploads.
* @public
*/
class GoogleAIFileManager {
constructor(apiKey, _requestOptions = {}) {
this.apiKey = apiKey;
this._requestOptions = _requestOptions;
}
/**
* Upload a file.
*/
async uploadFile(fileData, fileMetadata) {
const file = fileData instanceof Buffer ? fileData : fs.readFileSync(fileData);
const url = new FilesRequestUrl(RpcTask.UPLOAD, this.apiKey, this._requestOptions);
const uploadHeaders = getHeaders(url);
const boundary = generateBoundary();
uploadHeaders.append("X-Goog-Upload-Protocol", "multipart");
uploadHeaders.append("Content-Type", `multipart/related; boundary=${boundary}`);
const uploadMetadata = getUploadMetadata(fileMetadata);
// Multipart formatting code taken from @firebase/storage
const metadataString = JSON.stringify({ file: uploadMetadata });
const preBlobPart = "--" +
boundary +
"\r\n" +
"Content-Type: application/json; charset=utf-8\r\n\r\n" +
metadataString +
"\r\n--" +
boundary +
"\r\n" +
"Content-Type: " +
fileMetadata.mimeType +
"\r\n\r\n";
const postBlobPart = "\r\n--" + boundary + "--";
const blob = new Blob([preBlobPart, file, postBlobPart]);
const response = await makeServerRequest(url, uploadHeaders, blob);
return response.json();
}
/**
* List all uploaded files.
*
* Any fields set in the optional {@link SingleRequestOptions} parameter will take
* precedence over the {@link RequestOptions} values provided at the time of the
* {@link GoogleAIFileManager} initialization.
*/
async listFiles(listParams, requestOptions = {}) {
const filesRequestOptions = Object.assign(Object.assign({}, this._requestOptions), requestOptions);
const url = new FilesRequestUrl(RpcTask.LIST, this.apiKey, filesRequestOptions);
if (listParams === null || listParams === void 0 ? void 0 : listParams.pageSize) {
url.appendParam("pageSize", listParams.pageSize.toString());
}
if (listParams === null || listParams === void 0 ? void 0 : listParams.pageToken) {
url.appendParam("pageToken", listParams.pageToken);
}
const uploadHeaders = getHeaders(url);
const response = await makeServerRequest(url, uploadHeaders);
return response.json();
}
/**
* Get metadata for file with given ID.
*
* Any fields set in the optional {@link SingleRequestOptions} parameter will take
* precedence over the {@link RequestOptions} values provided at the time of the
* {@link GoogleAIFileManager} initialization.
*/
async getFile(fileId, requestOptions = {}) {
const filesRequestOptions = Object.assign(Object.assign({}, this._requestOptions), requestOptions);
const url = new FilesRequestUrl(RpcTask.GET, this.apiKey, filesRequestOptions);
url.appendPath(parseFileId(fileId));
const uploadHeaders = getHeaders(url);
const response = await makeServerRequest(url, uploadHeaders);
return response.json();
}
/**
* Delete file with given ID.
*/
async deleteFile(fileId) {
const url = new FilesRequestUrl(RpcTask.DELETE, this.apiKey, this._requestOptions);
url.appendPath(parseFileId(fileId));
const uploadHeaders = getHeaders(url);
await makeServerRequest(url, uploadHeaders);
}
}
/**
* If fileId is prepended with "files/", remove prefix
*/
function parseFileId(fileId) {
if (fileId.startsWith("files/")) {
return fileId.split("files/")[1];
}
if (!fileId) {
throw new GoogleGenerativeAIError(`Invalid fileId ${fileId}. ` +
`Must be in the format "files/filename" or "filename"`);
}
return fileId;
}
function generateBoundary() {
let str = "";
for (let i = 0; i < 2; i++) {
str = str + Math.random().toString().slice(2);
}
return str;
}
function getUploadMetadata(inputMetadata) {
if (!inputMetadata.mimeType) {
throw new GoogleGenerativeAIRequestInputError("Must provide a mimeType.");
}
const uploadMetadata = {
mimeType: inputMetadata.mimeType,
displayName: inputMetadata.displayName,
};
if (inputMetadata.name) {
uploadMetadata.name = inputMetadata.name.includes("/")
? inputMetadata.name
: `files/${inputMetadata.name}`;
}
return uploadMetadata;
}
/**
* Processing state of the `File`.
* @public
*/
var FileState;
(function (FileState) {
// The default value. This value is used if the state is omitted.
FileState["STATE_UNSPECIFIED"] = "STATE_UNSPECIFIED";
// File is being processed and cannot be used for inference yet.
FileState["PROCESSING"] = "PROCESSING";
// File is processed and available for inference.
FileState["ACTIVE"] = "ACTIVE";
// File failed processing.
FileState["FAILED"] = "FAILED";
})(FileState || (FileState = {}));
/**
* Contains the list of OpenAPI data types
* as defined by https://swagger.io/docs/specification/data-models/data-types/
* @public
*/
var SchemaType;
(function (SchemaType) {
/** String type. */
SchemaType["STRING"] = "string";
/** Number type. */
SchemaType["NUMBER"] = "number";
/** Integer type. */
SchemaType["INTEGER"] = "integer";
/** Boolean type. */
SchemaType["BOOLEAN"] = "boolean";
/** Array type. */
SchemaType["ARRAY"] = "array";
/** Object type. */
SchemaType["OBJECT"] = "object";
})(SchemaType || (SchemaType = {}));
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @public
*/
var ExecutableCodeLanguage;
(function (ExecutableCodeLanguage) {
ExecutableCodeLanguage["LANGUAGE_UNSPECIFIED"] = "language_unspecified";
ExecutableCodeLanguage["PYTHON"] = "python";
})(ExecutableCodeLanguage || (ExecutableCodeLanguage = {}));
/**
* Possible outcomes of code execution.
* @public
*/
var Outcome;
(function (Outcome) {
/**
* Unspecified status. This value should not be used.
*/
Outcome["OUTCOME_UNSPECIFIED"] = "outcome_unspecified";
/**
* Code execution completed successfully.
*/
Outcome["OUTCOME_OK"] = "outcome_ok";
/**
* Code execution finished but with a failure. `stderr` should contain the
* reason.
*/
Outcome["OUTCOME_FAILED"] = "outcome_failed";
/**
* Code execution ran for too long, and was cancelled. There may or may not
* be a partial output present.
*/
Outcome["OUTCOME_DEADLINE_EXCEEDED"] = "outcome_deadline_exceeded";
})(Outcome || (Outcome = {}));
/**
* @license
* Copyright 2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Possible roles.
* @public
*/
/**
* Harm categories that would cause prompts or candidates to be blocked.
* @public
*/
var HarmCategory;
(function (HarmCategory) {
HarmCategory["HARM_CATEGORY_UNSPECIFIED"] = "HARM_CATEGORY_UNSPECIFIED";
HarmCategory["HARM_CATEGORY_HATE_SPEECH"] = "HARM_CATEGORY_HATE_SPEECH";
HarmCategory["HARM_CATEGORY_SEXUALLY_EXPLICIT"] = "HARM_CATEGORY_SEXUALLY_EXPLICIT";
HarmCategory["HARM_CATEGORY_HARASSMENT"] = "HARM_CATEGORY_HARASSMENT";
HarmCategory["HARM_CATEGORY_DANGEROUS_CONTENT"] = "HARM_CATEGORY_DANGEROUS_CONTENT";
HarmCategory["HARM_CATEGORY_CIVIC_INTEGRITY"] = "HARM_CATEGORY_CIVIC_INTEGRITY";
})(HarmCategory || (HarmCategory = {}));
/**
* Threshold above which a prompt or candidate will be blocked.
* @public
*/
var HarmBlockThreshold;
(function (HarmBlockThreshold) {
/** Threshold is unspecified. */
HarmBlockThreshold["HARM_BLOCK_THRESHOLD_UNSPECIFIED"] = "HARM_BLOCK_THRESHOLD_UNSPECIFIED";
/** Content with NEGLIGIBLE will be allowed. */
HarmBlockThreshold["BLOCK_LOW_AND_ABOVE"] = "BLOCK_LOW_AND_ABOVE";
/** Content with NEGLIGIBLE and LOW will be allowed. */
HarmBlockThreshold["BLOCK_MEDIUM_AND_ABOVE"] = "BLOCK_MEDIUM_AND_ABOVE";
/** Content with NEGLIGIBLE, LOW, and MEDIUM will be allowed. */
HarmBlockThreshold["BLOCK_ONLY_HIGH"] = "BLOCK_ONLY_HIGH";
/** All content will be allowed. */
HarmBlockThreshold["BLOCK_NONE"] = "BLOCK_NONE";
})(HarmBlockThreshold || (HarmBlockThreshold = {}));
/**
* Probability that a prompt or candidate matches a harm category.
* @public
*/
var HarmProbability;
(function (HarmProbability) {
/** Probability is unspecified. */
HarmProbability["HARM_PROBABILITY_UNSPECIFIED"] = "HARM_PROBABILITY_UNSPECIFIED";
/** Content has a negligible chance of being unsafe. */
HarmProbability["NEGLIGIBLE"] = "NEGLIGIBLE";
/** Content has a low chance of being unsafe. */
HarmProbability["LOW"] = "LOW";
/** Content has a medium chance of being unsafe. */
HarmProbability["MEDIUM"] = "MEDIUM";
/** Content has a high chance of being unsafe. */
HarmProbability["HIGH"] = "HIGH";
})(HarmProbability || (HarmProbability = {}));
/**
* Reason that a prompt was blocked.
* @public
*/
var BlockReason;
(function (BlockReason) {
// A blocked reason was not specified.
BlockReason["BLOCKED_REASON_UNSPECIFIED"] = "BLOCKED_REASON_UNSPECIFIED";
// Content was blocked by safety settings.
BlockReason["SAFETY"] = "SAFETY";
// Content was blocked, but the reason is uncategorized.
BlockReason["OTHER"] = "OTHER";
})(BlockReason || (BlockReason = {}));
/**
* Reason that a candidate finished.
* @public
*/
var FinishReason;
(function (FinishReason) {
// Default value. This value is unused.
FinishReason["FINISH_REASON_UNSPECIFIED"] = "FINISH_REASON_UNSPECIFIED";
// Natural stop point of the model or provided stop sequence.
FinishReason["STOP"] = "STOP";
// The maximum number of tokens as specified in the request was reached.
FinishReason["MAX_TOKENS"] = "MAX_TOKENS";
// The candidate content was flagged for safety reasons.
FinishReason["SAFETY"] = "SAFETY";
// The candidate content was flagged for recitation reasons.
FinishReason["RECITATION"] = "RECITATION";
// The candidate content was flagged for using an unsupported language.
FinishReason["LANGUAGE"] = "LANGUAGE";
// Token generation stopped because the content contains forbidden terms.
FinishReason["BLOCKLIST"] = "BLOCKLIST";
// Token generation stopped for potentially containing prohibited content.
FinishReason["PROHIBITED_CONTENT"] = "PROHIBITED_CONTENT";
// Token generation stopped because the content potentially contains Sensitive Personally Identifiable Information (SPII).
FinishReason["SPII"] = "SPII";
// The function call generated by the model is invalid.
FinishReason["MALFORMED_FUNCTION_CALL"] = "MALFORMED_FUNCTION_CALL";
// Unknown reason.
FinishReason["OTHER"] = "OTHER";
})(FinishReason || (FinishReason = {}));
/**
* Task type for embedding content.
* @public
*/
var TaskType;
(function (TaskType) {
TaskType["TASK_TYPE_UNSPECIFIED"] = "TASK_TYPE_UNSPECIFIED";
TaskType["RETRIEVAL_QUERY"] = "RETRIEVAL_QUERY";
TaskType["RETRIEVAL_DOCUMENT"] = "RETRIEVAL_DOCUMENT";
TaskType["SEMANTIC_SIMILARITY"] = "SEMANTIC_SIMILARITY";
TaskType["CLASSIFICATION"] = "CLASSIFICATION";
TaskType["CLUSTERING"] = "CLUSTERING";
})(TaskType || (TaskType = {}));
/**
* @public
*/
var FunctionCallingMode;
(function (FunctionCallingMode) {
// Unspecified function calling mode. This value should not be used.
FunctionCallingMode["MODE_UNSPECIFIED"] = "MODE_UNSPECIFIED";
// Default model behavior, model decides to predict either a function call
// or a natural language repspose.
FunctionCallingMode["AUTO"] = "AUTO";
// Model is constrained to always predicting a function call only.
// If "allowed_function_names" are set, the predicted function call will be
// limited to any one of "allowed_function_names", else the predicted
// function call will be any one of the provided "function_declarations".
FunctionCallingMode["ANY"] = "ANY";
// Model will not predict any function call. Model behavior is same as when
// not passing any function declarations.
FunctionCallingMode["NONE"] = "NONE";
})(FunctionCallingMode || (FunctionCallingMode = {}));
/**
* The mode of the predictor to be used in dynamic retrieval.
* @public
*/
var DynamicRetrievalMode;
(function (DynamicRetrievalMode) {
// Unspecified function calling mode. This value should not be used.
DynamicRetrievalMode["MODE_UNSPECIFIED"] = "MODE_UNSPECIFIED";
// Run retrieval only when system decides it is necessary.
DynamicRetrievalMode["MODE_DYNAMIC"] = "MODE_DYNAMIC";
})(DynamicRetrievalMode || (DynamicRetrievalMode = {}));
// src/files-api.ts
/**
* This module encapsulates all interactions with the Google AI Files API.
* It handles uploading files, managing their state via a log file,
* checking for expiration, re-uploading if necessary, and cleaning up.
*/
function getVisionContextPath(chatWd) {
// Canary filename for cross-plugin compatibility
return path.join(chatWd, "vision_context.canary.json");
}
async function readVisionContext(chatWd) {
const contextPath = getVisionContextPath(chatWd);
try {
if (fs.existsSync(contextPath)) {
const raw = await fs.promises.readFile(contextPath, "utf-8");
const parsed = JSON.parse(raw);
try {
console.info("[FilesAPI] Read vision context:", contextPath);
}
catch { }
return {
// Support both old (activeAttachment) and new (activeAttachments) format
activeAttachments: Array.isArray(parsed.activeAttachments) ? parsed.activeAttachments : (parsed.activeAttachment ? [parsed.activeAttachment] : []),
activeGenerated: Array.isArray(parsed.activeGenerated) ? parsed.activeGenerated : [],
};
}
}
catch (error) {
console.error("Error reading vision_context.json:", error);
}
// Return a default empty context if file doesn't exist or is invalid
return { activeAttachments: [], activeGenerated: [] };
}
async function writeVisionContext(chatWd, context) {
const contextPath = getVisionContextPath(chatWd);
try {
await fs.promises.mkdir(path.dirname(contextPath), { recursive: true });
await fs.promises.writeFile(contextPath, JSON.stringify(context, null, 2));
try {
console.info("[FilesAPI] Wrote vision context:", contextPath);
}
catch { }
}
catch (error) {
console.error("Error writing vision_context.json:", error);
}
}
/**
* Sets the active attachments in the vision context. This REPLACES the entire
* attachment set (a1..aN). Old attachments will be detected as orphans and deleted
* by synchronizeVisionContext when they no longer match chat_media_state.json.
* @param chatWd The working directory for the current chat.
* @param attachments Array of attachment info with localPath, mimeType, uploadResult, and optional origin.
* @returns Array of previously active attachments that are no longer in the new set (dropped from Rolling Window).
*/
async function setActiveAttachments(chatWd, attachments) {
const context = await readVisionContext(chatWd);
const previousAttachments = context.activeAttachments || [];
// Sort by n-value (ascending) for consistent display
const sorted = [...attachments].sort((a, b) => (a.n ?? 0) - (b.n ?? 0));
const newFileNames = new Set(sorted.map(a => a.uploadResult.fileName));
// Find attachments that were active before but are not in the new set
const dropped = previousAttachments
.filter(prev => !newFileNames.has(prev.fileName))
.map(prev => ({ fileName: prev.fileName, localPath: prev.localPath, n: prev.n }));
context.activeAttachments = sorted.map(att => ({
source: "attachment",
localPath: att.localPath,
origin: att.origin || path.basename(att.localPath),
originalName: att.originalName,
n: att.n,
fileName: att.uploadResult.fileName,
fileUri: att.uploadResult.fileUri,
mimeType: att.mimeType,
createdAt: new Date().toISOString(),
}));
await writeVisionContext(chatWd, context);
return dropped;
}
/**
* Deletes files from the Files API that were dropped from the Rolling Window.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat.
* @param droppedFiles Array of files that were dropped (returned from setActiveAttachments).
*/
async function cleanupDroppedFromRollingWindow(apiKey, chatWd, droppedFiles) {
if (droppedFiles.length === 0)
return;
console.info(`[Rolling Window Cleanup] Deleting ${droppedFiles.length} file(s) dropped from window: ${droppedFiles.map(f => `a${f.n ?? '?'}`).join(', ')}`);
for (const dropped of droppedFiles) {
await deleteFile(apiKey, chatWd, dropped.fileName, {
localPath: dropped.localPath,
reason: `dropped-from-rolling-window (a${dropped.n ?? '?'})`,
});
}
}
/**
* Adds or updates multiple generated file entries in the vision context. Used to
* track Files API uploads for model-generated images so they can be
* cleaned up later by synchronizeVisionContext. This function is designed to be
* atomic for a batch of new files to prevent race conditions.
*/
async function addMultipleActiveGenerated(chatWd, generatedFiles) {
// Idempotent upsert that PRESERVES createdAt for existing entries.
// Only updates fileName/fileUri/mimeType if the same localPath already exists.
if (!generatedFiles.length)
return;
const context = await readVisionContext(chatWd);
const existing = Array.isArray(context.activeGenerated) ? context.activeGenerated.slice() : [];
const byPath = new Map(existing.map(e => [e.localPath, e]));
// Update existing entries in place
for (const gf of generatedFiles) {
const prev = byPath.get(gf.localPath);
if (prev) {
prev.fileName = gf.uploadResult.fileName;
prev.fileUri = gf.uploadResult.fileUri;
prev.mimeType = gf.mimeType;
// Keep prev.createdAt as-is
}
}
// Append only truly new entries
for (const gf of generatedFiles) {
if (!byPath.has(gf.localPath)) {
const entry = {
source: "generated",
localPath: gf.localPath,
fileName: gf.uploadResult.fileName,
fileUri: gf.uploadResult.fileUri,
mimeType: gf.mimeType,
createdAt: new Date().toISOString(),
};
existing.push(entry);
byPath.set(gf.localPath, entry);
}
}
context.activeGenerated = existing;
await writeVisionContext(chatWd, context);
}
function getLogPath(chatWd) {
// Extract a chat ID to create a unique log file per conversation.
const chatId = path.basename(chatWd).match(/(\d{13})/)?.[1] || "unknown-chat";
return path.join(chatWd, `${chatId}.files-api.jsonl`);
}
async function logEvent(chatWd, event) {
const logPath = getLogPath(chatWd);
// Use local timezone format (like other logs) instead of UTC
const now = new Date();
const localTimestamp = now.toLocaleString("sv-SE", { timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone }).replace(" ", "T") + "." + String(now.getMilliseconds()).padStart(3, "0");
const logEntry = {
timestamp: localTimestamp,
...event,
};
try {
const formattedJson = JSON.stringify(logEntry, null, 2);
const entrySeparator = "\n\n";
await fs.promises.appendFile(logPath, formattedJson + entrySeparator);
}
catch (error) {
console.error("Failed to write to Files API log:", error);
}
}
async function getFileManager(apiKey) {
// It's safer to initialize with the key when needed.
return new GoogleAIFileManager(apiKey);
}
/**
* Deletes a file from the Google AI Files API storage.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat, for logging.
* @param fileName The NAME of the file to delete (e.g., "files/xyz123").
*/
async function deleteFile(apiKey, chatWd, fileName, info) {
try {
const fileManager = await getFileManager(apiKey);
await fileManager.deleteFile(fileName);
console.info(`Successfully deleted file from Files API: ${fileName}`);
await logEvent(chatWd, {
type: "DELETE_SUCCESS",
fileName,
fileUri: "", // URI not needed for deletion log
localPath: info?.localPath,
details: info?.reason || "File deleted.",
});
}
catch (error) {
console.error(`Failed to delete file ${fileName}:`, error);
}
}
/**
* Reads the log file for a chat and deletes ALL uploaded files (full cleanup).
* Use this when switching away from Files API mode or cleaning up a chat entirely.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat.
*/
async function cleanupChatFiles(apiKey, chatWd) {
const logPath = getLogPath(chatWd);
try {
if (!fs.existsSync(logPath)) {
return; // No log file, nothing to clean up.
}
const logContent = await fs.promises.readFile(logPath, "utf-8");
const eventsRaw = logContent.split("\n\n").filter(s => s.trim());
const namesToDelete = new Set();
const namesDeleted = new Set();
// Read from bottom to top to get the latest status
for (const eventRaw of eventsRaw.reverse()) {
try {
const event = JSON.parse(eventRaw);
if (event.type.includes("DELETE")) {
namesDeleted.add(event.fileName);
}
else if (event.type.includes("UPLOAD") || event.type.includes("REUPLOAD")) {
if (!namesDeleted.has(event.fileName)) {
namesToDelete.add(event.fileName);
}
}
}
catch (e) {
// Ignore parsing errors for malformed entries
}
}
if (namesToDelete.size > 0) {
console.info(`[Files API Cleanup] Found ${namesToDelete.size} file(s) to delete for chat.`);
for (const name of namesToDelete) {
await deleteFile(apiKey, chatWd, name, { reason: "full-chat-cleanup" });
}
console.info(`[Files API Cleanup] Deleted ${namesToDelete.size} file(s). DELETE_SUCCESS entries logged to .files-api.jsonl`);
}
else {
console.info(`[Files API Cleanup] No files to delete (all already cleaned up).`);
}
// NOTE: vision_context.canary.json is NOT cleared here - it tracks active images for both
// Files API and Base64 modes. The .files-api.jsonl is also preserved as audit log.
}
catch (error) {
console.error("Error during Files API cleanup:", error);
}
}
/**
* Deletes files from the Files API that are NOT in vision_context.canary.json (orphan cleanup).
* This catches files that were uploaded but never properly tracked or fell out of sync.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat.
*/
async function cleanupOrphanedFiles(apiKey, chatWd) {
const logPath = getLogPath(chatWd);
try {
if (!fs.existsSync(logPath)) {
return; // No log file, nothing to clean up.
}
// Read current vision context to know what should be kept
const context = await readVisionContext(chatWd);
const activeFileNames = new Set([
...context.activeAttachments.map(a => a.fileName),
...context.activeGenerated.map(g => g.fileName),
]);
// Parse the log to find all uploaded files that haven't been deleted
const logContent = await fs.promises.readFile(logPath, "utf-8");
const eventsRaw = logContent.split("\n\n").filter(s => s.trim());
const uploadedFiles = new Map(); // fileName -> localPath
const deletedFiles = new Set();
for (const eventRaw of eventsRaw) {
try {
const event = JSON.parse(eventRaw);
if (event.type.includes("DELETE")) {
deletedFiles.add(event.fileName);
}
else if (event.type.includes("UPLOAD") || event.type.includes("REUPLOAD")) {
if (!deletedFiles.has(event.fileName)) {
uploadedFiles.set(event.fileName, event.localPath || "unknown");
}
}
}
catch (e) {
// Ignore parsing errors
}
}
// Find orphans: uploaded but not in vision_context and not deleted
const orphans = [];
for (const [fileName, localPath] of uploadedFiles) {
if (!activeFileNames.has(fileName) && !deletedFiles.has(fileName)) {
orphans.push({ fileName, localPath });
}
}
if (orphans.length > 0) {
console.info(`[Orphan Cleanup] Found ${orphans.length} orphaned file(s) not in vision_context.canary.json`);
for (const orphan of orphans) {
await deleteFile(apiKey, chatWd, orphan.fileName, {
localPath: orphan.localPath,
reason: "orphaned-not-in-vision-context"
});
}
}
else {
console.info(`[Orphan Cleanup] No orphaned files found.`);
}
}
catch (error) {
console.error("Error during orphan cleanup:", error);
}
}
/**
* Ensures a local file is uploaded to the Files API, handling state, expiration, and re-uploads.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat.
* @param localPath The absolute path to the local file.
* @param mimeType The MIME type of the file.
* @returns The valid full `fileUri` for the uploaded file, or null on failure.
*/
async function ensureFileIsUploaded(apiKey, chatWd, localPath, mimeType) {
const logPath = getLogPath(chatWd);
const fileManager = await getFileManager(apiKey);
let lastKnownUri = null;
let lastKnownName = null;
let lastKnownTimestamp = null;
let lastDeleteForKnownName = null;
// 1. Check log for an existing, non-deleted entry for this local file
if (fs.existsSync(logPath)) {
const logContent = await fs.promises.readFile(logPath, "utf-8");
const eventsRaw = logContent.split("\n\n").filter(s => s.trim());
const events = [];
for (const er of eventsRaw) {
try {
events.push(JSON.parse(er));
}
catch { }
}
// Find last known upload/reupload for this localPath
for (const event of [...events].reverse()) {
try {
if (event.localPath === localPath) {
if (event.type.includes("DELETE")) {
break; // The last action for this file was deletion, so stop.
}
if ((event.type.includes("UPLOAD") || event.type.includes("REUPLOAD")) && event.fileName && event.fileUri) {
lastKnownUri = event.fileUri;
lastKnownName = event.fileName;
lastKnownTimestamp = event.timestamp;
break;
}
}
}
catch { }
}
// Track any DELETE for that fileName later in the log
if (lastKnownName) {
for (const event of [...events].reverse()) {
if (event.type.includes("DELETE") && event.fileName === lastKnownName) {
lastDeleteForKnownName = event.timestamp;
break;
}
}
}
}
// 2. If a file was found in the log, validate its age.
if (lastKnownName && lastKnownUri && lastKnownTimestamp) {
// Invalidate if we have a DELETE record for that file after the upload timestamp
try {
if (lastDeleteForKnownName) {
const del = new Date(lastDeleteForKnownName).getTime();
const up = new Date(lastKnownTimestamp).getTime();
if (!Number.isNaN(del) && !Number.isNaN(up) && del >= up) {
lastKnownName = null;
lastKnownUri = null;
lastKnownTimestamp = null;
}
}
}
catch { }
try {
if (lastKnownName && lastKnownUri && lastKnownTimestamp) {
const now = new Date();
const uploadTime = new Date(lastKnownTimestamp);
const ageInHours = (now.getTime() - uploadTime.getTime()) / (1000 * 60 * 60);
if (ageInHours < 47) {
// Remote validation: verify file still exists on server
try {
await fileManager.getFile(lastKnownName);
await logEvent(chatWd, { type: "VALIDATION_SUCCESS", fileName: lastKnownName, fileUri: lastKnownUri, localPath, details: `File is ${ageInHours.toFixed(2)} hours old.` });
return { fileUri: lastKnownUri, fileName: lastKnownName };
}
catch (e) {
// Treat as stale/removed on server; fall through to re-upload
await logEvent(chatWd, { type: "REUPLOAD_SUCCESS", fileName: lastKnownName, fileUri: lastKnownUri, localPath, details: "Remote validation failed; reuploading." });
}
}
else {
console.warn(`File ${lastKnownName} is older than 47 hours. Re-uploading...`);
}
}
}
catch (e) {
console.error(`Error parsing timestamp for ${lastKnownName}:`, e);
// Continue to upload logic below as a fallback.
}
}
// 3. If no valid file exists, upload it.
try {
console.info(`Uploading new file to Files API: ${localPath}`);
const response = await fileManager.uploadFile(localPath, { mimeType, displayName: path.basename(localPath) });
const newFileUri = response.file.uri;
const newFileName = response.file.name;
console.info(`File uploaded successfully. URI: ${newFileUri}, Name: ${newFileName}`);
await logEvent(chatWd, {
type: lastKnownName ? "REUPLOAD_SUCCESS" : "UPLOAD_SUCCESS",
localPath,
fileName: newFileName,
fileUri: newFileUri,
mimeType,
});
return { fileUri: newFileUri, fileName: newFileName };
}
catch (error) {
console.error("Files API upload failed:", error);
return null;
}
}
/**
* Constructs the fileData part for a generateContent request.
* @param fileUri The URI of the file obtained from the Files API.
* @param mimeType The MIME type of the file.
* @returns The part object for the API call.
*/
function buildFileDataPart(fileUri, mimeType) {
return {
fileData: {
mimeType,
fileUri,
},
};
}
/**
* Synchronizes the vision context with the actual chat history. It finds orphaned
* files (entries in vision_context.json that are no longer referenced in the chat history)
* and deletes them from the remote server.
* @param apiKey The Gemini API key.
* @param chatWd The working directory for the current chat.
* @param history The current chat history object from the SDK.
*/
async function synchronizeVisionContext(apiKey, chatWd, history /* Chat */) {
// Use chat_media_state.json as single source of truth for referenced files
const statePath = path.join(chatWd, "chat_media_state.json");
let state = null;
try {
state = JSON.parse(await fs.promises.readFile(statePath, "utf8"));
}
catch (e) {
throw new Error("synchronizeVisionContext: chat_media_state.json missing or invalid. Aborting cleanup.");
}
const referenced = new Set();
const addRef = (p) => {
if (typeof p !== "string" || !p.length)
return;
referenced.add(path.isAbsolute(p) ? p : path.join(chatWd, p));
};
try {
// Add ALL attachments to referenced set (supports array)
if (Array.isArray(state.attachments)) {
state.attachments.forEach((a) => {
addRef(a.originAbs);
addRef(a.preview || a.filename);
});
}
if (Array.isArray(state.variants)) {
state.variants.forEach((v) => {
addRef(v.originAbs);
addRef(v.preview || v.filename);
});
}
}
catch { }
const context = await readVisionContext(chatWd);
const activeFiles = [...context.activeAttachments, ...context.activeGenerated];
if (activeFiles.length === 0)
return;
const orphans = activeFiles.filter(f => !referenced.has(f.localPath));
if (!orphans.length)
return;
console.info(`[Vision Sync] Found ${orphans.length} orphaned file(s) to delete (Files API).`);
for (const o of orphans) {
await deleteFile(apiKey, chatWd, o.fileName, { localPath: o.localPath, reason: "orphaned-not-in-state" });
}
const updatedContext = {
activeAttachments: context.activeAttachments.filter(f => !orphans.some(o => o.localPath === f.localPath)),
activeGenerated: context.activeGenerated.filter(f => !orphans.some(o => o.localPath === f.localPath)),
};
await writeVisionContext(chatWd, updatedContext);
}
// ============================================================================
// Helper to find LM Studio home directory
// ============================================================================
function findLMStudioHome() {
// Check environment variable first
if (process.env.LMSTUDIO_HOME)
return process.env.LMSTUDIO_HOME;
// Default to ~/.lmstudio
return path.join(os.homedir(), ".lmstudio");
}
async function pathExists(p) {
try {
await fs.promises.access(p, fs.constants.F_OK);
return true;
}
catch {
return false;
}
}
// ============================================================================
// Extract a 13-digit chat id from the chat working directory name
// ============================================================================
// Extract a 13-digit chat id from the chat working directory name (per LM Studio convention)
function findChatIdFromWd(chatWd) {
try {
const base = path.basename(chatWd || "");
const m = base.match(/(\d{13})/);
return m ? m[1] : null;
}
catch {
return null;
}
}
// ============================================================================
// findAllAttachmentsFromLastTurn - prefer pending clientInputFiles, else latest user message
// Based on: instructions/standalone_generator_guide/src/orchestrator.ts
// ============================================================================
/**
* Scan conversation.json for attachments that are relevant "now":
* 1) clientInputFiles (pending attachments before message is sent)
* 2) attachments from the most recent user message
*
* IMPORTANT: This intentionally does NOT accumulate attachments from older turns.
* Downstream state keeps attachments across text-only turns; this function is
* only responsible for detecting the current attachment set.
*/
async function findAllAttachmentsFromLastTurn(chatWd, debug = false) {
if (!chatWd)
return [];
const chatId = findChatIdFromWd(chatWd);
const lmHome = findLMStudioHome();
const conversationsDir = path.join(lmHome, "conversations");
const userFilesDir = path.join(lmHome, "user-files");
// Try multiple candidate paths
const candidates = [
chatId ? path.join(conversationsDir, `${chatId}.conversation.json`) : null,
path.join(chatWd, ".conversation.json"),
path.join(chatWd, "conversation.json"),
].filter((p) => p !== null);
for (const convPath of candidates) {
if (!(await pathExists(convPath)))
continue;
try {
const raw = await fs.promises.readFile(convPath, "utf-8");
const json = JSON.parse(raw);
const normalizeMaybeFileUri = (p) => {
const asPath = fileUriToPath(p);
if (asPath)
return asPath;
return p;
};
const tryExtractClientInputFiles = (root) => {
try {
const files = root?.clientInputFiles;
if (!Array.isArray(files) || files.length === 0)
return null;
const found = [];
for (const f of files) {
const id = f?.fileIdentifier;
const type = f?.fileType;
if (typeof id === "string" && id.trim() && type === "image") {
found.push(path.join(userFilesDir, id));
}
}
return found.length > 0 ? found : null;
}
catch {
return null;
}
};
// Priority 0: clientInputFiles can appear on root or nested objects.
const fromClientInput = tryExtractClientInputFiles(json) ??
tryExtractClientInputFiles(json?.conversation) ??
tryExtractClientInputFiles(json?.chat) ??
tryExtractClientInputFiles(json?.state);
if (fromClientInput && fromClientInput.length > 0) {
const unique = [];
const seen = new Set();
for (const f of fromClientInput) {
const n = path.resolve(normalizeMaybeFileUri(f));
if (seen.has(n))
continue;
seen.add(n);
unique.push(n);
}
if (debug)
console.info(`[findAllAttachments] Using clientInputFiles (pending): ${unique.length} item(s)`);
return unique;
}
// Priority 1: attachments from the most recent USER message.
const tryExtractMostRecentUserMessageAttachments = (root) => {
const candidateArrays = [];
const maybePushArray = (a) => { if (Array.isArray(a) && a.length)
candidateArrays.push(a); };
try {
maybePushArray(root?.messages);
maybePushArray(root?.conversation?.messages);
maybePushArray(root?.chat?.messages);
maybePushArray(root?.history);
maybePushArray(root?.turns);
maybePushArray(root?.items);
}
catch { /* ignore */ }
for (const arr of candidateArrays) {
for (let i = arr.length - 1; i >= 0; i--) {
const m = arr[i];
if (!m || typeof m !== "object")
continue;
// Unwrap LM Studio versioned messages
let msgObj = m;
try {
const versions = Array.isArray(m.versions) ? m.versions : null;
const selRaw = m.currentlySelected;
const sel = typeof selRaw === "number" && Number.isFinite(selRaw) ? selRaw : 0;
if (versions && versions.length) {
msgObj = sel >= 0 && sel < versions.length ? versions[sel] : versions[versions.length - 1];
}
}
catch { /* ignore */ }
const role = msgObj.role ?? msgObj.author ?? msgObj.sender;
const type = msgObj.type ?? msgObj.messageType;
const isUser = role === "user" || role === "human" || type === "user" || type === "user_message";
if (!isUser)
continue;
const found = [];
try {
const content = msgObj?.content;
if (Array.isArray(content)) {
for (const part of content)
collectImageFileCandidates(part, found, userFilesDir);
}
else {
collectImageFileCandidates(msgObj, found, userFilesDir);
}
}
catch {
collectImageFileCandidates(msgObj, found, userFilesDir);
}
if (!found.length) {
// Most recent user message has no attachments; do not fall back to older turns.
return [];
}
const seen = new Set();
const unique = [];
for (const f of found) {
const n = path.resolve(normalizeMaybeFileUri(f));
if (seen.has(n))
continue;
seen.add(n);
unique.push(n);
}
return unique;
}
}
return null;
};
const fromLatestUserMsg = tryExtractMostRecentUserMessageAttachments(json);
if (fromLatestUserMsg !== null) {
if (debug)
console.info(`[findAllAttachments] Using latest user-message attachments: ${fromLatestUserMsg.length} item(s)`);
return fromLatestUserMsg;
}
// Last resort: deep scan entire JSON
const allFound = [];
collectImageFileCandidates(json, allFound, userFilesDir);
const seen = new Set();
const unique = [];
for (const f of allFound) {
const n = path.resolve(normalizeMaybeFileUri(f));
if (seen.has(n))
continue;
seen.add(n);
unique.push(n);
}
if (debug && unique.length > 0) {
console.info(`[findAllAttachments] Fallback deep-scan found ${unique.length} attachment(s) in ${convPath}`);
}
return unique;
}
catch (e) {
if (debug)
console.warn(`[findAllAttachments] Failed to parse ${convPath}:`, e.message);
}
}
return [];
}
function collectImageFileCandidates(obj, found, userFilesDir) {
if (!obj || typeof obj !== "object")
return;
// Various structures LM Studio uses
const fileId = obj.fileIdentifier ?? obj.identifier ?? obj.file_id;
if (typeof fileId === "string" && fileId.trim()) {
found.push(path.join(userFilesDir, fileId));
return;
}
// Recurse into nested objects
for (const v of Object.values(obj)) {
if (Array.isArray(v)) {
for (const item of v)
collectImageFileCandidates(item, found, userFilesDir);
}
else if (typeof v === "object" && v !== null) {
collectImageFileCandidates(v, found, userFilesDir);
}
}
}
// ============================================================================
// findAllAttachmentsFromConversation - ALL attachments from entire history
// ============================================================================
/**
* Scan conversation.json for ALL attachments ever added to the chat.
* Returns attachments in chronological order (as they appear in conversation.json).
*
* This is different from findAllAttachmentsFromLastTurn which only looks at
* the latest user message or clientInputFiles.
*
* Use this function to build a complete inventory of all attachments in chat_media_state.json.
*
* Sources scanned (in order):
* 1. clientInputFiles (pending attachments in draft)
* 2. All user messages with file attachments (chronological order)
*
* @returns Array of absolute paths to user-files, in order of first appearance
*/
async function findAllAttachmentsFromConversation(chatWd, debug = false) {
if (!chatWd)
return [];
const chatId = findChatIdFromWd(chatWd);
const lmHome = findLMStudioHome();
const conversationsDir = path.join(lmHome, "conversations");
const userFilesDir = path.join(lmHome, "user-files");
const candidates = [
chatId ? path.join(conversationsDir, `${chatId}.conversation.json`) : null,
path.join(chatWd, ".conversation.json"),
path.join(chatWd, "conversation.json"),
].filter((p) => p !== null);
for (const convPath of candidates) {
if (!(await pathExists(convPath)))
continue;
try {
const raw = await fs.promises.readFile(convPath, "utf-8");
const json = JSON.parse(raw);
const normalizeMaybeFileUri = (p) => {
const asPath = fileUriToPath(p);
if (asPath)
return asPath;
return p;
};
const allFound = [];
const seenPaths = new Set();
const addUnique = (p) => {
const normalized = path.resolve(normalizeMaybeFileUri(p));
if (!seenPaths.has(normalized)) {
seenPaths.add(normalized);
allFound.push(normalized);
}
};
// Helper to extract image file candidates from an object
const collectFromObj = (obj) => {
if (!obj || typeof obj !== "object")
return;
// Check for file attachment patterns
const fileId = obj.fileIdentifier ?? obj.identifier ?? obj.file_id;
const fileType = obj.fileType ?? obj.type;
if (typeof fileId === "string" && fileId.trim() && fileType === "image") {
addUnique(path.join(userFilesDir, fileId));
return;
}
// Recurse into nested objects
for (const v of Object.values(obj)) {
if (Array.isArray(v)) {
for (const item of v)
collectFromObj(item);
}
else if (typeof v === "object" && v !== null) {
collectFromObj(v);
}
}
};
// 1. Collect from clientInputFiles (pending/draft attachments)
const tryClientInputFiles = (root) => {
const files = root?.clientInputFiles;
if (Array.isArray(files)) {
for (const f of files) {
const id = f?.fileIdentifier;
const type = f?.fileType;
if (typeof id === "string" && id.trim() && type === "image") {
addUnique(path.join(userFilesDir, id));
}
}
}
};
tryClientInputFiles(json);
tryClientInputFiles(json?.conversation);
tryClientInputFiles(json?.chat);
tryClientInputFiles(json?.state);
// 2. Collect from ALL messages (chronological order)
const messageArrays = [];
const maybeAddArray = (a) => {
if (Array.isArray(a) && a.length > 0)
messageArrays.push(a);
};
maybeAddArray(json?.messages);
maybeAddArray(json?.conversation?.messages);
maybeAddArray(json?.chat?.messages);
maybeAddArray(json?.history);
maybeAddArray(json?.turns);
maybeAddArray(json?.items);
for (const messages of messageArrays) {
// Process messages in chronological order (first to last)
for (const msg of messages) {
if (!msg || typeof msg !== "object")
continue;
// Unwrap LM Studio versioned messages
let msgObj = msg;
try {
const versions = Array.isArray(msg?.versions) ? msg.versions : null;
const selRaw = msg?.currentlySelected;
const sel = typeof selRaw === "number" && Number.isFinite(selRaw) ? selRaw : 0;
if (versions && versions.length) {
msgObj = sel >= 0 && sel < versions.length ? versions[sel] : versions[versions.length - 1];
}
}
catch { /* ignore */ }
// Check if this is a user message
const role = msgObj?.role ?? msgObj?.author ?? msgObj?.sender;
const type = msgObj?.type ?? msgObj?.messageType;
const isUser = role === "user" || role === "human" || type === "user" || type === "user_message";
if (!isUser)
continue;
// Collect attachments from this user message
try {
const content = msgObj?.content;
if (Array.isArray(content)) {
for (const part of content)
collectFromObj(part);
}
else {
collectFromObj(msgObj);
}
}
catch {
collectFromObj(msgObj);
}
}
}
if (debug && allFound.length > 0) {
console.info(`[findAllAttachmentsComplete] Found ${allFound.length} attachment(s) in ${convPath}`);
}
return allFound;
}
catch (e) {
if (debug)
console.warn(`[findAllAttachmentsComplete] Failed to parse ${convPath}:`, e.message);
}
}
return [];
}
// ============================================================================
// NEW: getOriginalFileName - resolve LM Studio metadata for original filename
// ============================================================================
/**
* Resolve the original filename from LM Studio user-files metadata.
* LM Studio stores metadata in <userFilesDir>/<fileIdentifier>.meta.json
*/
async function getOriginalFileName(fileIdentifier) {
const lmHome = findLMStudioHome();
const metaCandidates = [
path.join(lmHome, "user-files", `${fileIdentifier}.metadata.json`),
path.join(lmHome, "user-files", `${fileIdentifier}.meta.json`),
];
try {
for (const p of metaCandidates) {
try {
const raw = await fs.promises.readFile(p, "utf-8");
const meta = JSON.parse(raw);
const resolved = meta?.originalName ?? meta?.name ?? meta?.fileName ?? meta?.filename ?? undefined;
if (typeof resolved === "string" && resolved.trim().length > 0)
return resolved;
}
catch {
// keep trying candidates
}
}
return undefined;
}
catch {
// No metadata file - fall back to fileIdentifier as name
return undefined;
}
}
/**
* Batch-import attachments from SSOT (conversation.json).
*
* Key behaviors:
* - Replaces entire attachments array with exactly what's in SSOT
* - Preserves n-values for existing attachments (stable numbering)
* - At empty state, n starts at 1
* - Empty SSOT → keeps existing state (attachments persist across text-only turns)
* - Only generates previews, NO copies of originals
* - Preview naming: preview-<origin> (e.g., preview-1766100380042 - 811.jpg)
*/
async function importAttachmentBatch(chatWd, state, sourcePaths, previewOpts = { maxDim: 1024, quality: 85 }, maxPreviewAttachments = 2, debug = false) {
const markerPath = path.join(chatWd, "attachment-i2i-pending.json");
const normalizeAbs = (p) => {
try {
return path.resolve(p);
}
catch {
return p;
}
};
const normalizedSource = sourcePaths
.filter((p) => typeof p === "string" && p.trim().length > 0)
.map(normalizeAbs);
// De-dupe while preserving order
const normalizedSourceDeduped = [];
{
const seen = new Set();
for (const p of normalizedSource) {
if (!seen.has(p)) {
seen.add(p);
normalizedSourceDeduped.push(p);
}
}
}
// CASE 1: No NEW attachments in this turn → keep existing state
// Attachments persist across text-only turns until replaced by new attachments.
if (normalizedSourceDeduped.length === 0) {
if (debug)
console.info("[importAttachmentBatch] No new attachments in current turn; keeping existing state.");
return { changed: false };
}
// Idempotence: if SSOT paths match current state, skip re-import
try {
const current = Array.isArray(state.attachments) ? state.attachments : [];
const currentOrigins = current
.map((a) => a && typeof a.originAbs === "string" ? a.originAbs : "")
.filter((p) => p.trim().length > 0)
.map(normalizeAbs);
const same = currentOrigins.length === normalizedSourceDeduped.length &&
currentOrigins.every((p, i) => p === normalizedSourceDeduped[i]);
if (same && current.length > 0) {
const shouldHavePreview = (i) => {
const total = currentOrigins.length;
const cap = Math.max(0, Math.floor(maxPreviewAttachments));
return cap > 0 && i >= Math.max(0, total - cap);
};
// Check if previews exist (for LAST maxPreviewAttachments)
let allOk = true;
for (let i = 0; i < current.length; i++) {
if (!shouldHavePreview(i))
continue;
const a = current[i];
const pv = a && typeof a.preview === "string" ? a.preview : "";
if (!pv) {
allOk = false;
break;
}
const pvAbs = path.join(chatWd, pv);
if (!(await pathExists(pvAbs))) {
allOk = false;
break;
}
}
if (allOk) {
if (debug)
console.info("[importAttachmentBatch] SSOT matches current state; skipping re-import (idempotent).");
return { changed: false };
}
}
}
catch (e) {
if (debug)
console.warn("[importAttachmentBatch] Idempotence check failed; continuing with import:", e.message);
}
// CASE 2: SSOT has attachments → Import ALL into state + pending
// Build a map of existing attachments by originAbs for stable numbering
const existingByOrigin = new Map();
for (const a of state.attachments || []) {
if (a && typeof a.originAbs === "string") {
existingByOrigin.set(normalizeAbs(a.originAbs), a);
}
}
const imported = [];
const pendingPaths = [];
// CRITICAL: If no existing attachments, reset n to 1 for clean numbering
let nextN = existingByOrigin.size === 0
? 1
: Math.max(1, state.counters?.nextAttachmentN ?? 1);
const shouldHavePreview = (i) => {
const total = normalizedSourceDeduped.length;
const cap = Math.max(0, Math.floor(maxPreviewAttachments));
return cap > 0 && i >= Math.max(0, total - cap);
};
for (let i = 0; i < normalizedSourceDeduped.length; i++) {
const abs = normalizedSourceDeduped[i];
if (!(await pathExists(abs))) {
if (debug)
console.warn(`[importAttachmentBatch] Source not found, skipping: ${abs}`);
continue;
}
// Check if this attachment already exists (preserve its n)
const existing = existingByOrigin.get(normalizeAbs(abs));
if (existing) {
// Keep existing record (preserve n, createdAt, etc.)
// Ensure preview is present for LAST maxPreviewAttachments and absent for others.
// Refresh originalName if it was previously unknown / looked like fileIdentifier.
try {
const fileIdentifier = path.basename(abs);
if (!existing.originalName || existing.originalName === existing.origin) {
const resolvedName = await getOriginalFileName(fileIdentifier);
if (resolvedName)
existing.originalName = resolvedName;
}
}
catch { /* best-effort */ }
if (shouldHavePreview(i)) {
const desiredPreviewName = `preview-${path.basename(abs)}`;
if (!existing.preview) {
existing.preview = desiredPreviewName;
}
const previewAbs = path.join(chatWd, existing.preview);
if (!(await pathExists(previewAbs))) {
if (debug)
console.info(`[importAttachmentBatch] Generating/regenerating preview for n=${existing.n}`);
await encodeJpegPreview(abs, previewAbs, previewOpts);
}
}
else {
// Metadata-only attachments should not carry preview in state.
existing.preview = undefined;
}
imported.push(existing);
pendingPaths.push(abs);
if (debug)
console.info(`[importAttachmentBatch] Keeping existing attachment n=${existing.n}: ${path.basename(abs)}`);
continue;
}
// New attachment - assign next n
const origin = path.basename(abs);
// Generate preview (NO copy of original!)
let previewName = undefined;
if (shouldHavePreview(i)) {
previewName = `preview-${origin}`;
const previewAbs = path.join(chatWd, previewName);
await encodeJpegPreview(abs, previewAbs, previewOpts);
}
// Resolve originalName from LM Studio metadata
let originalName = undefined;
const userFilesDir = path.join(findLMStudioHome(), "user-files");
if (abs.startsWith(userFilesDir)) {
const fileIdentifier = path.basename(abs);
const resolvedName = await getOriginalFileName(fileIdentifier);
originalName = resolvedName || fileIdentifier;
if (debug && resolvedName) {
console.info(`[importAttachmentBatch] Resolved original filename: ${fileIdentifier} → ${originalName}`);
}
}
else {
// Non-LM Studio attachments: use basename as original name
originalName = path.basename(abs);
}
imported.push({
filename: undefined, // NO copy - originAbs is the source
origin,
originAbs: abs,
originalName,
preview: previewName,
createdAt: new Date().toISOString(),
n: nextN++,
});
pendingPaths.push(abs);
}
if (imported.length === 0) {
if (debug)
console.warn("[importAttachmentBatch] No valid attachments imported");
return { changed: false };
}
// MERGE: Keep existing attachments that are NOT in the new SSOT, then append new ones
// This preserves attachment history across turns (Rolling Window only limits promotion, not state)
const newOriginSet = new Set(imported.map(a => normalizeAbs(a.originAbs || "")));
const existingToKeep = (state.attachments || []).filter(a => {
const origin = a && typeof a.originAbs === "string" ? normalizeAbs(a.originAbs) : "";
return origin && !newOriginSet.has(origin);
});
// Combine: existing attachments (not in new set) + newly imported, then SORT by n-value
const merged = [...existingToKeep, ...imported];
merged.sort((a, b) => (a.n ?? 0) - (b.n ?? 0));
state.attachments = merged;
state.counters = state.counters || {};
state.counters.nextAttachmentN = nextN;
state.lastEvent = { type: "attachment", at: new Date().toISOString() };
// NOTE: Preserve idempotency tracking fields (lastPromotedAttachmentN, lastPromotedTs, lastVariantsTs)
// These are set by markAsPromoted() and must survive import cycles
if (debug)
console.info(`[importAttachmentBatch] State now has ${state.attachments.length} attachment(s): ${existingToKeep.length} kept + ${imported.length} new/updated`);
// Write pending.json with ALL imported paths
const pending = {
files: pendingPaths,
createdAt: new Date().toISOString(),
usedAt: "",
consumed: false,
};
await fs.promises.mkdir(chatWd, { recursive: true });
await fs.promises.writeFile(markerPath, JSON.stringify(pending, null, 2), "utf-8");
await writeChatMediaStateAtomic(chatWd, state);
if (debug)
console.info(`[importAttachmentBatch] Imported ${imported.length} attachment(s) from SSOT`);
return { changed: true };
}
/**
* Generate JPEG preview from source image
*/
async function encodeJpegPreview(srcAbs, dstAbs, opts) {
const maxDim = opts.maxDim ?? 1024;
const quality = opts.quality ?? 85;
const jpeg = await resizeMaxDimJpegFromFile(srcAbs, maxDim, quality);
await fs.promises.writeFile(dstAbs, jpeg);
}
/**
* Build promotion parts for Base64 mode (Mode B).
*
* REFACTORED (2025-12) to use:
* - importAttachmentBatch (stable n-numbering, no copies)
* - buildPromotionItems (stable labels with n-field)
* - toGeminiInlineDataParts (unified base64 encoding)
*/
async function buildPromotionPartsB(params) {
const { ctl, history, chatWd, debugChunks, showOnlyLastImageVariant, visionPromotionPersistent } = params;
const promoParts = [];
const promotedFiles = [];
try {
// Step 1: Deterministic variants pipeline.
// - External tools: harvest from history (tool/assistant) WITHOUT copies or preview generation.
// - Gemini-native (gemini-3-pro-image-preview): use legacy recover path.
if (params.model === "gemini-3-pro-image-preview") {
try {
await recoverProVariantsFromHistory(history, chatWd, debugChunks, false);
}
catch (e) {
if (debugChunks)
console.warn('[Variants] Gemini-native recover failed:', e.message);
}
}
else {
try {
const res = await harvestToolGeneratedVariantsFromLatestToolMessage(ctl, history, chatWd, debugChunks);
if (debugChunks) {
console.info(`[Variants] Harvest result: source=${res.source} found=${res.foundVariants} recorded=${res.recordedVariants} reason=${res.reason}`);
}
}
catch (e) {
if (debugChunks)
console.warn('[Variants] Harvest failed:', e.message);
}
}
// Step 2: Load current state
let state = await readChatMediaState(chatWd).catch(() => ({ attachments: [], variants: [], counters: {} }));
// Step 3: Import attachments from SSOT (conversation.json)
// This uses the new importAttachmentBatch with stable n-numbering
const ssotPaths = await findAllAttachmentsFromConversation(chatWd, debugChunks);
// Get model-specific limit for visual promotion (Rolling Window size)
const maxPromotedAttachments = getMaxPromotedAttachments(params.model || "");
if (debugChunks)
console.info(`[Promotion B] Model=${params.model} maxPromotedAttachments=${maxPromotedAttachments}`);
if (ssotPaths.length > 0) {
// Use new batch import with stable n-numbering
const importResult = await importAttachmentBatch(chatWd, state, ssotPaths, { maxDim: 1024, quality: 85 }, maxPromotedAttachments, // model-specific limit
debugChunks);
if (importResult.changed) {
state = await readChatMediaState(chatWd);
}
}
// Step 4: Idempotency check - skip if nothing new (unless persistent mode)
const persistentMode = visionPromotionPersistent === true;
const { shouldPromoteAttachment, shouldPromoteVariants } = shouldPromoteImages(state, persistentMode);
if (!shouldPromoteAttachment && !shouldPromoteVariants) {
if (debugChunks)
console.info('[Promotion B] Idempotent mode: no new attachments/variants, skipping promotion');
return { promoParts: [], promotedFiles: [] };
}
// Step 5: Build promotion items using new function
// This uses stable n-field for labels
const hasAttachments = Array.isArray(state.attachments) && state.attachments.length > 0;
const hasVariants = Array.isArray(state.variants) && state.variants.length > 0;
if (!hasAttachments && !hasVariants) {
if (debugChunks)
console.info('[Promotion B] No attachments or variants to promote');
return { promoParts: [], promotedFiles: [] };
}
// Determine variant limit
let maxVariantItems = 3;
if (showOnlyLastImageVariant && state.variants?.length > 0) {
maxVariantItems = 1;
if (debugChunks)
console.info('[Promotion B] Limiting to only last variant due to showOnlyLastImageVariant');
}
const items = buildPromotionItems(chatWd, state, {
labels: true,
maxAttachmentItems: maxPromotedAttachments,
maxVariantItems,
});
// NOTE: Attachments outside the Rolling Window are NOT promoted at all (no text-only labels).
// The model only sees the last N attachments. If user references an older one, the agent
// should communicate that it's outside the visual context.
if (items.length === 0) {
if (debugChunks)
console.info('[Promotion B] buildPromotionItems returned empty');
return { promoParts: [], promotedFiles: [] };
}
// Step 6: Convert to Gemini inlineData parts
const parts = await toGeminiInlineDataParts(items);
promoParts.push(...parts);
// Track promoted files
for (const it of items) {
promotedFiles.push(path.basename(it.previewAbs));
}
// Step 7: Write canary for debugging/audit
try {
const generatedForCanary = items.map((it) => ({
localPath: it.previewAbs,
mimeType: /\.png$/i.test(it.previewAbs) ? "image/png" : "image/jpeg",
uploadResult: { fileName: path.basename(it.previewAbs), fileUri: it.previewAbs },
}));
if (generatedForCanary.length) {
await addMultipleActiveGenerated(chatWd, generatedForCanary);
}
}
catch { /* best-effort canary */ }
if (debugChunks) {
console.info(`[Promotion B] Promoted ${items.length} item(s): ${promotedFiles.join(', ')}`);
}
// Step 8: Mark as promoted for idempotency tracking
await markAsPromoted(chatWd, state, shouldPromoteAttachment, shouldPromoteVariants);
}
catch (e) {
const err = e;
const msg = (err && (err.stack || err.message)) ? (err.stack || err.message) : String(err);
console.error("Promotion parts error (Base64):", msg);
}
return { promoParts, promotedFiles };
}
async function buildPromotionPartsFiles(params) {
const { ctl, history, apiKey, chatWd, debugChunks, visionPromotionPersistent } = params;
const promoParts = [];
const promotedFiles = [];
if (!apiKey) {
if (debugChunks)
console.warn("[PromotionFiles] No API key provided; skipping Files API promotion.");
return { promoParts, promotedFiles };
}
// 0. Harvest tool-generated variants (no copies, no preview generation) so state is up to date.
try {
await harvestToolGeneratedVariantsFromLatestToolMessage(ctl, history, chatWd, debugChunks);
}
catch (e) {
if (debugChunks)
console.warn("[PromotionFiles] Tool variants harvest failed:", e.message);
}
// 1. Load current state and import attachments from SSOT (conversation.json)
let state = await readChatMediaState(chatWd).catch(() => ({ attachments: [], variants: [], counters: { nextN: 1, nextV: 1 } }));
// Try to import attachments from SSOT (ALL attachments from entire conversation)
const ssotPaths = await findAllAttachmentsFromConversation(chatWd, debugChunks);
// Get model-specific limit for visual promotion (Rolling Window size)
const maxPromotedAttachments = getMaxPromotedAttachments(params.model || "");
if (debugChunks)
console.info(`[PromotionFiles] Model=${params.model} maxPromotedAttachments=${maxPromotedAttachments}`);
if (ssotPaths.length > 0) {
const result = await importAttachmentBatch(chatWd, state, ssotPaths, { maxDim: 1024, quality: 85 }, maxPromotedAttachments, debugChunks);
if (result.changed) {
state = await readChatMediaState(chatWd);
if (debugChunks)
console.info('[PromotionFiles] Imported attachments from SSOT');
}
}
// 2. Idempotency check - determines what to INJECT into prompt (not what to track in Files API)
const persistentMode = visionPromotionPersistent === true;
const { shouldPromoteAttachment, shouldPromoteVariants } = shouldPromoteImages(state, persistentMode);
// ALWAYS log idempotency check for debugging
const attachments = Array.isArray(state.attachments) ? state.attachments : [];
const maxN = attachments.length > 0 ? Math.max(0, ...attachments.map((a) => a.n ?? 0)) : 0;
console.info(`[PromotionFiles] Idempotency: persistent=${persistentMode} maxN=${maxN} lastPromoted=${state.lastPromotedAttachmentN ?? 'undefined'} → shouldPromoteAtt=${shouldPromoteAttachment} shouldPromoteVar=${shouldPromoteVariants}`);
// NOTE: We always proceed to ensure Files API registration is up-to-date (vision_context.canary.json)
// The idempotency flags control whether we inject into the prompt, not whether we track files.
const skipPromptInjection = !shouldPromoteAttachment && !shouldPromoteVariants;
if (skipPromptInjection) {
console.info('[PromotionFiles] ✓ Skipping prompt injection (idempotent), but will update Files API tracking...');
}
else {
console.info('[PromotionFiles] → Proceeding with promotion...');
}
// 3. Synchronize context (cleanup orphans) - AFTER importing attachments
try {
await synchronizeVisionContext(apiKey, chatWd, history);
}
catch (e) {
if (debugChunks)
console.warn("[PromotionFiles] Sync failed:", e.message);
}
// Attachments from State
// In Files API mode, we upload directly from originAbs (no local copies)
// NOTE: Attachments outside the Rolling Window are NOT promoted at all (no text-only labels).
if (state.attachments && state.attachments.length > 0) {
// Sort by n-value (ascending) to ensure chronological order for Rolling Window
const allAttachments = [...state.attachments].sort((a, b) => (a.n ?? 0) - (b.n ?? 0));
// Take the LAST N (highest n-values = most recent attachments)
const promotedAttachments = allAttachments.slice(-maxPromotedAttachments);
if (debugChunks) {
const promotedNs = promotedAttachments.map(a => `a${a.n}:${a.originalName || a.origin}`).join(', ');
console.info(`[PromotionFiles] Rolling Window: total=${allAttachments.length} promoting=${promotedAttachments.length} [${promotedNs}]`);
}
const uploadedForContext = [];
for (const att of promotedAttachments) {
const uploadPath = att.originAbs || (att.preview ? path.join(chatWd, att.preview) : path.join(chatWd, att.filename || ""));
const origin = att.origin || att.filename || path.basename(uploadPath);
const ext = path.extname(uploadPath).toLowerCase();
let mime = "image/png";
if (ext === ".jpg" || ext === ".jpeg")
mime = "image/jpeg";
else if (ext === ".webp")
mime = "image/webp";
const upload = await ensureFileIsUploaded(apiKey, chatWd, uploadPath, mime);
if (upload) {
const stableN = typeof att.n === "number" ? att.n : 0;
const originalName = att.originalName || att.origin || `attachment-${stableN}`;
// Only inject into prompt if shouldPromoteAttachment is true
if (shouldPromoteAttachment) {
const label = `Attachment [a${stableN}] ${originalName}`;
promoParts.push({ text: label });
promoParts.push(buildFileDataPart(upload.fileUri, mime));
promotedFiles.push(uploadPath);
}
// Always track in vision_context for Files API management
uploadedForContext.push({ localPath: uploadPath, mimeType: mime, uploadResult: upload, origin, originalName, n: stableN });
}
}
// Register the full promoted attachment set atomically (prevents clobbering to a single entry)
if (uploadedForContext.length > 0) {
const droppedFiles = await setActiveAttachments(chatWd, uploadedForContext);
if (debugChunks)
console.info('[PromotionFiles] Registered active attachments in vision context:', uploadedForContext.map(x => x.localPath).join(', '));
// Cleanup files that fell out of the Rolling Window
if (droppedFiles.length > 0 && apiKey) {
await cleanupDroppedFromRollingWindow(apiKey, chatWd, droppedFiles);
}
}
}
// Variants from State
if (state.variants && state.variants.length > 0) {
const generatedFilesToRegister = [];
let variantsToProcess = [...state.variants];
// Sort by v field for consistent ordering
variantsToProcess.sort((a, b) => a.v - b.v || a.createdAt.localeCompare(b.createdAt));
if (params.showOnlyLastImageVariant && variantsToProcess.length > 0) {
variantsToProcess = [variantsToProcess[variantsToProcess.length - 1]];
if (debugChunks)
console.info(`[PromotionFiles] Filtering variants to only the last one (V${variantsToProcess[0].v}) due to showOnlyLastImageVariant`);
}
// Limit to max 3 variants
variantsToProcess = variantsToProcess.slice(0, 3);
for (const v of variantsToProcess) {
// Files-API mode must upload the original if available.
// - External tool variants: use originAbs
// - Legacy variants: use chatWd/filename (original PNG in workdir)
const localPath = v.originAbs
? v.originAbs
: path.join(chatWd, v.filename);
const ext = path.extname(localPath).toLowerCase();
let mime = "image/png";
if (ext === ".jpg" || ext === ".jpeg")
mime = "image/jpeg";
else if (ext === ".webp")
mime = "image/webp";
const upload = await ensureFileIsUploaded(apiKey, chatWd, localPath, mime);
if (upload) {
// Only inject into prompt if shouldPromoteVariants is true
if (shouldPromoteVariants) {
const stableV = typeof v.v === "number" ? v.v : undefined;
const vTag = stableV ? `v${stableV}` : "v?";
const label = `Generated Image [${vTag}]`;
promoParts.push({ text: label });
promoParts.push(buildFileDataPart(upload.fileUri, mime));
promotedFiles.push(localPath);
}
// Always track in vision_context for Files API management
generatedFilesToRegister.push({ localPath, mimeType: mime, uploadResult: upload });
}
}
// Register generated files in vision_context so they are tracked (always, for Files API management)
if (generatedFilesToRegister.length > 0) {
await addMultipleActiveGenerated(chatWd, generatedFilesToRegister);
}
}
// Mark as promoted for idempotency tracking
if (promotedFiles.length > 0) {
await markAsPromoted(chatWd, state, shouldPromoteAttachment, shouldPromoteVariants);
if (debugChunks)
console.info(`[PromotionFiles] Marked as promoted: attachments=${shouldPromoteAttachment} variants=${shouldPromoteVariants}`);
}
// Cleanup orphaned files (uploaded but not in vision_context.canary.json)
// This catches files that fell out of sync due to bugs or old code versions
try {
await cleanupOrphanedFiles(apiKey, chatWd);
}
catch (e) {
if (debugChunks)
console.warn("[PromotionFiles] Orphan cleanup failed:", e.message);
}
return { promoParts, promotedFiles };
}
async function buildPromotionPartsForMode(params) {
const { useFilesApiForVision, suppressVisionPromotionForThisTurn, visionPromotionPersistent, ...rest } = params;
// Hard override: if this turn is marked as unsafe for promotion
// (e.g. tool/functionResponse replay with thought_signature), skip images entirely.
if (suppressVisionPromotionForThisTurn) {
return buildPromotionPartsA(rest);
}
// Vision Promotion is always ON - two modes:
// - Persistent (visionPromotionPersistent=true): re-inject every turn
// - Idempotent (visionPromotionPersistent=false): inject only when new
// The idempotency logic is handled inside the promotion functions via shouldPromoteImages()
if (useFilesApiForVision) {
return buildPromotionPartsFiles({ ...rest, visionPromotionPersistent });
}
// Switched to Base64 mode - cleanup any Files API uploads for this chat
// This prevents orphaned files when user toggles the setting
if (rest.apiKey && rest.chatWd) {
try {
await cleanupChatFiles(rest.apiKey, rest.chatWd);
}
catch (e) {
console.warn("[VisionModeSelector] Failed to cleanup Files API on mode switch:", e.message);
}
}
// Fallback to Base64 mode (Mode B)
return buildPromotionPartsB({ ...rest, visionPromotionPersistent });
}
function parseAttachmentWrappers$1(text) {
const parts = [];
if (!text)
return parts;
const re = /\[\[LMSTUDIO_ATTACHMENT:\s*(\{[\s\S]*?\})\s*\]\]/g;
let m;
while ((m = re.exec(text)) !== null) {
try {
const obj = JSON.parse(m[1]);
if (obj && obj.kind === "image" && typeof obj.url === "string") {
parts.push({ kind: "image", url: obj.url });
}
else if (obj && obj.kind === "text" && typeof obj.text === "string") {
parts.push({ kind: "text", text: obj.text });
}
else if (obj && obj.kind === "text_link" && typeof obj.url === "string") {
parts.push({ kind: "text_link", url: obj.url });
}
}
catch { /* ignore */ }
}
// Also detect plain file:// URIs and absolute image paths in the text
try {
const seen = new Set();
const pushOnce = (u) => { if (!seen.has(u)) {
parts.push({ kind: 'image', url: u });
seen.add(u);
} };
// file:// URIs
const reFile = /file:\/\/[^\s)]+/gi;
while ((m = reFile.exec(text)) !== null) {
const u = m[0];
pushOnce(u);
}
// Absolute paths with common image extensions
const reAbs = /(^|\s)(\/[\w@#$%&+.,:\-\/]+?\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))(\s|$|[)])/gi;
let m2;
while ((m2 = reAbs.exec(text)) !== null) {
const u = m2[2];
pushOnce(u);
}
}
catch { /* ignore */ }
return parts;
}
function extractMarkdownLocalImageLinks(text) {
const out = [];
if (!text)
return out;
const mdRel = /!\[[^\]]*\]\(((?:\.?\.?\/)[^\s)]+\.(?:png|jpe?g|webp))\)/gi;
let m;
while ((m = mdRel.exec(text)) !== null)
out.push(m[1]);
return out;
}
async function snapshotHistoryMediaState(ctl, history, chatWd, model) {
// IMPORTANT: chat_media_state.json is now maintained by dedicated import/record pipelines.
// This snapshot helper is only allowed to initialize state when it does not exist.
// Otherwise it would clobber fields like originAbs, stable n-numbering, and variant provenance.
try {
const statePath = path.join(chatWd, "chat_media_state.json");
if (fs.existsSync(statePath)) {
return await readChatMediaState(chatWd);
}
}
catch {
// best-effort only
}
const caps = detectCapabilities(model);
const maxVariants = caps.imageGeneration?.numberOfImages || 3;
let attachment = null;
let variants = [];
// Scan history backwards
const arr = Array.from(history);
for (let i = arr.length - 1; i >= 0 && (!attachment || variants.length === 0); i--) {
const msg = arr[i];
const role = typeof msg.getRole === 'function' ? msg.getRole() : undefined;
const text = typeof msg.getText === 'function' ? (msg.getText() || "") : "";
if (!role)
continue;
if (!attachment && role === 'user') {
const parts = parseAttachmentWrappers$1(text);
for (const p of parts) {
if (p.kind === 'image' && typeof p.url === 'string') {
// Map to relative filename if possible (./file) else keep basename
const base = path.basename(p.url.replace(/^file:\/\//, ""));
attachment = { filename: base, origin: base, preview: base };
break;
}
}
}
if (variants.length === 0 && role === 'assistant') {
const links = extractMarkdownLocalImageLinks(text);
if (links.length) {
const rel = links.filter(l => l.startsWith('./') || l.startsWith('../'));
const picked = rel.slice(0, maxVariants).map(r => path.basename(r));
if (picked.length)
variants = picked.map(fn => ({ filename: fn, preview: fn }));
}
}
}
const nowIso = new Date().toISOString();
const state = {
attachments: attachment ? [{ filename: attachment.filename, origin: attachment.origin, preview: attachment.preview, createdAt: nowIso, n: 1 }] : [],
variants: variants.map((v, idx) => ({ filename: v.filename, preview: v.preview, createdAt: nowIso, v: idx + 1 })),
// Prefer variants over attachment when both are present, since variants denote a generation event.
lastEvent: (attachment || variants.length)
? { type: (variants.length ? 'variants' : 'attachment'), at: nowIso }
: { type: 'none', at: nowIso },
counters: { nextAttachmentN: (attachment ? 2 : 1), nextVariantV: Math.min(maxVariants, variants.length) + 1 },
};
await writeChatMediaState(chatWd, state);
try {
console.info("[MediaState] Pre-turn snapshot written:", path.join(chatWd, "chat_media_state.json"));
}
catch { }
return state;
}
const FILENAME = "thought_signatures.json";
function getFilePath(cwd) {
return path.join(cwd, FILENAME);
}
// Simple normalization: trim and collapse whitespace before hashing
function computeContentHash$1(text) {
return crypto.createHash("sha256").update(text ? text.trim().replace(/\s+/g, " ") : "").digest("hex");
}
async function loadSignatures(cwd) {
try {
const fp = getFilePath(cwd);
const content = await fs.promises.readFile(fp, "utf-8");
return JSON.parse(content);
}
catch {
return { signatures: [] };
}
}
async function saveSignatures(cwd, state) {
const fp = getFilePath(cwd);
await fs.promises.writeFile(fp, JSON.stringify(state, null, 2), "utf-8");
}
async function appendSignature(cwd, signature, contentHash) {
const state = await loadSignatures(cwd);
// Avoid duplicates for the exact same content hash (idempotency)
if (!state.signatures.find(s => s.contentHash === contentHash && s.signature === signature)) {
state.signatures.push({
signature,
contentHash,
timestamp: Date.now(),
});
await saveSignatures(cwd, state);
}
}
async function pruneSignatures(cwd, history) {
const state = await loadSignatures(cwd);
if (state.signatures.length === 0)
return;
const validHashes = new Set();
for (const msg of history) {
if (msg.getRole() === "assistant") {
const text = msg.getText();
if (text) {
validHashes.add(computeContentHash$1(text));
}
const calls = msg.getToolCallRequests?.();
if (Array.isArray(calls)) {
for (const call of calls) {
const orig = String(call?.name || "tool");
// We assume no collisions for pruning purposes to keep it simple.
// If there's a collision, we might delete a signature, which is a trade-off.
const safe = sanitizeToolName(orig);
const args = call?.arguments ?? {};
const id = `${safe}:${JSON.stringify(args)}`;
validHashes.add(computeContentHash$1(id));
}
}
}
}
const newSignatures = state.signatures.filter(s => validHashes.has(s.contentHash));
if (newSignatures.length !== state.signatures.length) {
state.signatures = newSignatures;
await saveSignatures(cwd, state);
}
}
function stableJsonStringify(obj) {
if (typeof obj !== "object" || obj === null)
return JSON.stringify(obj);
if (Array.isArray(obj))
return "[" + obj.map(stableJsonStringify).join(",") + "]";
const keys = Object.keys(obj).sort();
const parts = keys.map(key => JSON.stringify(key) + ":" + stableJsonStringify(obj[key]));
return "{" + parts.join(",") + "}";
}
function safeStringify(obj, secrets = []) {
const secretSet = new Set(secrets.filter(Boolean));
const redactor = (key, value) => {
if (typeof value === "string") {
// Field-based redaction
if (/^(x-goog-api-key|authorization|apiKey|api_key)$/i.test(key)) {
return "***";
}
}
return value;
};
let s = "";
try {
s = JSON.stringify(obj, redactor);
}
catch {
try {
s = String(obj);
}
catch {
s = "[unserializable]";
}
}
// Content-based redaction
for (const sec of secretSet) {
if (!sec)
continue;
try {
const esc = sec.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
s = s.replace(new RegExp(esc, "g"), "***");
}
catch { /* ignore */ }
}
return s;
}
function normalizeFunctionResponsePayload(payload) {
if (payload === null || payload === undefined)
return {};
if (Array.isArray(payload))
return { result: payload };
if (typeof payload !== "object")
return { result: payload };
return payload;
}
function toGeminiMessages(history, nameMap, signatures = []) {
const contents = [];
const signatureMap = new Map(signatures.map(s => [s.contentHash, s.signature]));
function getSignatureForText(text) {
const hash = computeContentHash$1(text);
if (signatureMap.has(hash))
return signatureMap.get(hash);
if (signatureMap.has("LATEST_TEXT_SIG"))
return signatureMap.get("LATEST_TEXT_SIG");
return undefined;
}
function getSignatureForFunctionCall(name, args) {
const id = `${name}:${stableJsonStringify(args ?? {})}`;
const hash = computeContentHash$1(id);
if (signatureMap.has(hash))
return signatureMap.get(hash);
// Fallback to LATEST_TEXT_SIG if available. This is critical for retries where the exact hash might be missed,
// or if the tool call signature wasn't explicitly saved under the tool-hash key in previous versions.
// While this might technically map a newer signature to an older call in rare cases, it prevents 400 errors.
if (signatureMap.has("LATEST_TEXT_SIG"))
return signatureMap.get("LATEST_TEXT_SIG");
return undefined;
}
for (const message of history) {
const parts = [];
switch (message.getRole()) {
case "system":
// System messages will be handled via systemInstruction; skip here
break;
case "user": {
const parsed = parseAttachmentWrappers(message.getText());
if (parsed.text.trim().length > 0) {
parts.push({ text: parsed.text });
}
// Handle attachments nur textuell; wir erzeugen KEINE echten Bild-Parts
// in der Reasoning-History, um thought_signature-Pflichten zu vermeiden.
// NOTE: We intentionally do NOT add text for image URLs to avoid leaking
// absolute paths (like /Users/helge/...) to the model. Images are promoted
// separately via the vision promotion system.
for (const p of parsed.parts) {
if (p.kind === "image" && p.url) ;
else if (p.kind === "text" && p.text) {
parts.push({ text: p.text });
}
else if (p.kind === "text_link" && p.url) {
// Only show basename for text links to avoid leaking paths
const basename = p.url.split('/').pop() || p.url;
parts.push({ text: `Attached file: ${basename}` });
}
}
contents.push({ role: "user", parts });
break;
}
case "assistant": {
// Include assistant text and any functionCall requests from history
const text = message.getText();
if (text && text.trim()) {
const part = { text };
const sig = getSignatureForText(text);
if (sig)
part.thought_signature = sig;
parts.push(part);
}
const calls = message.getToolCallRequests?.();
if (Array.isArray(calls) && calls.length) {
for (const c of calls) {
const orig = String(c?.name || "tool");
const safe = nameMap?.get(orig) || sanitizeToolName(orig);
const args = c?.arguments ?? {};
const part = { functionCall: { name: safe, args } };
const sig = getSignatureForFunctionCall(safe, args);
if (sig)
part.thought_signature = sig;
parts.push(part);
}
}
if (parts.length)
contents.push({ role: "model", parts });
break;
}
case "tool": {
// Map tool call results to Gemini functionResponse parts
const results = message.getToolCallResults();
for (const r of results) {
const rName = (r?.toolName || r?.name || r?.functionName || r?.tool || r?.id || "tool");
const safeName = nameMap?.get(rName) || sanitizeToolName(String(rName));
let payload = r?.content ?? r?.result ?? r?.output ?? null;
if (typeof payload === "string") {
try {
payload = JSON.parse(payload);
}
catch { /* keep as string */ }
}
// Sanitize tool result for model context (avoid double-render of tool-provided image objects)
payload = sanitizeToolResponseForModel(payload);
const responsePayload = normalizeFunctionResponsePayload(payload);
const frPart = { functionResponse: { name: safeName, response: responsePayload } };
const sig = getSignatureForFunctionCall(safeName, (r?.arguments ?? {}));
if (sig)
frPart.thought_signature = sig;
parts.push(frPart);
}
if (parts.length)
contents.push({ role: "user", parts });
break;
}
}
}
return contents;
}
function parseAttachmentWrappers(text) {
const parts = [];
if (!text)
return { text, parts };
const re = /\[\[LMSTUDIO_ATTACHMENT:\s*(\{[\s\S]*?\})\s*\]\]/g;
let cleaned = text;
let m;
while ((m = re.exec(text)) !== null) {
try {
const obj = JSON.parse(m[1]);
if (obj && obj.kind === "image" && typeof obj.url === "string") {
parts.push({ kind: "image", url: obj.url });
}
else if (obj && obj.kind === "text" && typeof obj.text === "string") {
parts.push({ kind: "text", text: obj.text });
}
else if (obj && obj.kind === "text_link" && typeof obj.url === "string") {
parts.push({ kind: "text_link", url: obj.url });
}
cleaned = cleaned.replace(m[0], "");
}
catch {
// ignore malformed wrappers
}
}
// Also detect plain file:// URIs and absolute image paths; do not remove from text
try {
const seen = new Set();
const pushOnce = (u) => { if (!seen.has(u)) {
parts.push({ kind: 'image', url: u });
seen.add(u);
} };
let m2;
const reFile = /file:\/\/[^\s)]+/gi;
while ((m2 = reFile.exec(text)) !== null)
pushOnce(m2[0]);
const reAbs = /(^|\s)(\/[\w@#$%&+.,:\-\/]+?\.(?:png|jpe?g|webp|gif|bmp|tiff?|heic))(\s|$|[)])/gi;
while ((m2 = reAbs.exec(text)) !== null)
pushOnce(m2[2]);
}
catch { /* ignore */ }
return { text: cleaned, parts };
}
function getLastUserText(history) {
let last = "";
for (const msg of history) {
if (msg.getRole() === "user")
last = msg.getText() || "";
}
return last;
}
function collectSystemText(history) {
const parts = [];
for (const msg of history) {
if (msg.getRole() === "system") {
const t = msg.getText();
if (t && t.trim())
parts.push(t.trim());
}
}
return parts.join("\n\n");
}
function pad2(n) { return n.toString().padStart(2, "0"); }
async function streamTextFragments(ctl, text) {
const chunkSize = 512;
for (let i = 0; i < text.length; i += chunkSize) {
const chunk = text.slice(i, i + chunkSize);
ctl.fragmentGenerated(chunk);
// Small yield to keep UI responsive; adjust or remove if undesired
// eslint-disable-next-line no-await-in-loop
await new Promise(res => setTimeout(res, 10));
}
}
class BaseGeminiStrategy {
// Hook for subclasses (e.g. GeminiThinkingStrategy) to persist thought_signature for tool calls.
// Default: no-op (keeps BaseGeminiStrategy generic).
async onObservedFunctionCallPart(_context, _safeName, _args, _sig) {
return;
}
async generate(context) {
const { ctl, history, model, apiKey, globalConfig, pluginConfig, debugChunks, logRequests } = context;
const visionPromotionPersistent = globalConfig.get("visionPromotionPersistent");
const useFilesApiForVision = pluginConfig.get("useFilesApiForVision");
const redactSecrets = [];
const genAI = new generativeAi.GoogleGenerativeAI(apiKey);
const caps = detectCapabilities(model);
if (debugChunks) {
try {
console.info("[Capabilities] model=", model, {
supportsTools: caps.supportsTools,
supportsVision: caps.supportsVision,
supportsImage: caps.supportsImage,
supportsThinking: caps.supportsThinking,
supportsStreaming: caps.supportsStreaming,
imageGeneration: caps.imageGeneration,
});
}
catch { /* ignore */ }
}
const lastUserText = getLastUserText(history);
const supportsFunctionCalling = caps.supportsTools;
const systemText = collectSystemText(history);
const chatWd = ctl.getWorkingDirectory();
// Reconcile media state strictly from chat history (SSoT)
await snapshotHistoryMediaState(ctl, history, chatWd, model);
// Decide promotion transport mode early
const shouldUseFilesApi = shouldUseFilesApiForModel(model, useFilesApiForVision);
await this.reconcileAttachments(context, shouldUseFilesApi);
// Backfill: in Base64 mode only (not GCS).
await this.backfillAnalysisPreviews(context, shouldUseFilesApi);
let promotedFiles = [];
const { tools: geminiTools, originalToSafe, safeToOriginal} = supportsFunctionCalling
? buildGeminiTools(ctl, lastUserText || "")
: { tools: undefined, originalToSafe: new Map(), safeToOriginal: new Map()};
// Manage Thought Signatures for legacy / non-isolated thinking models only
// gemini-3-pro-image-preview is handled by GeminiImageThinkingStrategy
let contents;
if (caps.supportsThinking && model !== "gemini-3-pro-image-preview") {
let signatures = [];
try {
await pruneSignatures(chatWd, history);
}
catch (e) {
if (debugChunks)
console.warn("Failed to prune signatures:", e);
}
const sigState = await loadSignatures(chatWd);
signatures = sigState.signatures;
contents = toGeminiMessages(history, originalToSafe, signatures);
}
else {
contents = toGeminiMessages(history, originalToSafe);
}
// Hook for subclasses to modify contents (e.g. flattening for Thinking models)
this.modifyContents(contents, caps);
// Vision promotion handover
try {
if (debugChunks) {
try {
console.info("[Vision Path] Toggle=", !!useFilesApiForVision, "→ shouldUseFilesApi=", !!shouldUseFilesApi);
}
catch { /* ignore */ }
}
let promoParts = [];
let pf = [];
const res = await buildPromotionPartsForMode({
ctl,
history,
apiKey,
chatWd,
debugChunks,
shouldUseFilesApi,
model,
visionPromotionPersistent,
useFilesApiForVision: !!useFilesApiForVision,
});
promoParts = res.promoParts || [];
pf = res.promotedFiles || [];
promotedFiles = pf;
if (promoParts.length) {
let lastUserMessage = contents.slice().reverse().find(m => m.role === 'user');
if (lastUserMessage) {
lastUserMessage.parts = [...promoParts, ...(lastUserMessage.parts || [])];
}
else {
contents.push({ role: "user", parts: promoParts });
}
if (debugChunks)
console.info(`Prepended ${promoParts.length} vision part(s) to the user message.`);
}
}
catch (e) {
if (debugChunks)
console.error("Promotion parts error:", e.message);
}
// Prepare system instruction
let systemInstruction;
{
const parts = [];
if (systemText)
parts.push({ text: systemText });
const now = new Date();
const tz = (() => {
try {
return Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC";
}
catch {
return "UTC";
}
})();
const yyyy = now.getFullYear();
const MM = pad2(now.getMonth() + 1);
const DD = pad2(now.getDate());
const hh = pad2(now.getHours());
const mm = pad2(now.getMinutes());
const ss = pad2(now.getSeconds());
parts.push({ text: `Current date/time: ${yyyy}-${MM}-${DD} ${hh}:${mm}:${ss} (${tz})` });
if (!supportsFunctionCalling) {
const imgPolicy = "You are an image-capable model without tool support. Do not attempt to call tools or functions. " +
"You can describe, analyze, and generate images. Use only information explicitly provided by the user; " +
"leave unspecified parameters at defaults and avoid guesses.";
parts.push({ text: imgPolicy });
}
else if (geminiTools) {
const policy = "Tool use policy: Use available tools only when necessary to fulfill the request. " +
"Use only arguments explicitly provided or clearly implied by the user; leave unspecified parameters at their defaults. " +
"Do not ask for missing arguments and do not guess values. At most one tool call per turn. " +
"When a tool result describes an image, follow its $hint and markdown instructions: show the image using the provided markdown and then describe it briefly. " +
"Images that appear again in later turns are persistent context (from earlier uploads or tool outputs); do not assume the user has just uploaded them again. " +
"For non-image tools, do not echo raw JSON or internal fields such as 'markdown'; instead, summarize the result in a short, natural sentence.";
parts.push({ text: policy });
}
if (parts.length)
systemInstruction = { parts };
}
/* 2. Prepare the request */
const generateContent = {
contents,
};
if (geminiTools && supportsFunctionCalling) {
generateContent.tools = geminiTools;
generateContent.toolConfig = {
functionCallingConfig: {
mode: "AUTO",
},
};
}
if (systemInstruction) {
generateContent.systemInstruction = systemInstruction;
}
// Hook for subclasses to modify generation config
this.modifyGenerationConfig(generateContent, context, caps);
try {
/* 3. Make the API call using SDK */
// logRequests: nur Rohdaten (ohne Telemetrie-Präfixe)
if (logRequests)
console.info(safeStringify({ direction: "request", model, payload: generateContent }, redactSecrets));
const generativeModel = genAI.getGenerativeModel({
model: model,
...(systemInstruction ? { system_instruction: systemInstruction } : {}),
safetySettings: [{
category: generativeAi.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
threshold: generativeAi.HarmBlockThreshold.BLOCK_NONE
}],
});
const isStreamingCandidate = caps.supportsStreaming;
if (debugChunks) {
try {
console.info("[Streaming] isStreamingCandidate=", isStreamingCandidate);
}
catch { /* ignore */ }
}
let response;
let candidates;
if (isStreamingCandidate) {
if (debugChunks)
console.info("[Streaming] Using generateContentStream for this request.");
const stream = await generativeModel.generateContentStream({
contents: generateContent.contents,
tools: generateContent.tools,
toolConfig: generateContent.toolConfig,
generationConfig: generateContent.generationConfig,
});
let toolCallEmitted = false;
const allowedSafe = new Set(safeToOriginal ? Array.from(safeToOriginal.keys()) : []);
for await (const item of stream.stream) {
const scands = item?.candidates;
if (!Array.isArray(scands) || !scands.length)
continue;
const candidate = scands[0];
const parts = candidate?.content?.parts;
if (!Array.isArray(parts))
continue;
let textBuf = "";
const toolCalls = [];
for (const part of parts) {
const p = part;
const isImageLink = typeof p.text === 'string' && /!\[Image\]\(.*?\)/.test(p.text);
if (p.thought && !isImageLink) {
const content = typeof p.thought === 'string' ? p.thought : (p.text || "");
if (content) {
ctl.fragmentGenerated(content, { reasoningType: "reasoning" });
}
}
else if (p.text) {
if (!p.thought || isImageLink) {
textBuf += (textBuf ? "\n" : "") + p.text;
}
}
const fcall = part?.functionCall || part?.function_call;
if (fcall && fcall.name) {
let args = fcall.args;
if (typeof args === "string") {
try {
args = JSON.parse(args);
}
catch { /* keep as string */ }
}
toolCalls.push({ name: String(fcall.name), args });
// Subclass hook: persist thought_signature for tool calls if needed.
try {
const sig = part?.thought_signature || part?.thoughtSignature;
await this.onObservedFunctionCallPart(context, String(fcall.name), args, sig);
}
catch { /* best-effort */ }
}
}
if (textBuf.trim().length) {
if (debugChunks)
console.info("[Streaming] Text update (raw from SDK):", textBuf.slice(0, 120));
ctl.fragmentGenerated(textBuf);
}
if (!toolCallEmitted && toolCalls.length) {
const streamingToolCall = toolCalls.find(tc => allowedSafe.has(tc.name));
if (streamingToolCall) {
const originalName = safeToOriginal.get(streamingToolCall.name) || streamingToolCall.name;
const argsJson = typeof streamingToolCall.args === "string"
? streamingToolCall.args
: JSON.stringify(streamingToolCall.args ?? {});
const callId = `gemini-fc-${Date.now()}-0`;
ctl.toolCallGenerationStarted();
ctl.toolCallGenerationNameReceived(originalName);
ctl.toolCallGenerationArgumentFragmentGenerated(argsJson);
ctl.toolCallGenerationEnded({ type: "function", name: originalName, arguments: streamingToolCall.args ?? {}, id: callId });
toolCallEmitted = true;
// CRITICAL (LM Studio tool-call loop): once we emit a tool call, we must
// end this generate() invocation immediately so LM Studio can run the tool
// and re-invoke generate() with the tool result.
ctl.fragmentGenerated("");
return;
}
}
}
ctl.fragmentGenerated(""); // Close reasoning block
response = stream.response;
candidates = response?.candidates;
// Robustness: if the final response snapshot contains additional
// non-thinking text that was never streamed, render it once here.
if (Array.isArray(candidates) && candidates.length > 0) {
try {
let responseText = "";
for (const cand of candidates) {
const parts = cand?.content?.parts;
if (!Array.isArray(parts))
continue;
for (const p of parts) {
const isImageLink = typeof p.text === "string" && /!\[Image\]\(.*?\)/.test(p.text);
const isThought = !isImageLink && !!p.thought;
if (p.text && !isImageLink && !isThought) {
responseText += (responseText ? "\n" : "") + p.text;
}
}
}
if (responseText.trim().length) {
if (debugChunks)
console.info("[Streaming] Final snapshot text update (Base):", responseText.slice(0, 200));
ctl.fragmentGenerated(responseText);
}
}
catch { /* best-effort only */ }
}
}
else {
if (debugChunks)
console.info("[Streaming] Using non-streaming generateContent for this request.");
const result = await generativeModel.generateContent({
contents: generateContent.contents,
tools: generateContent.tools,
toolConfig: generateContent.toolConfig,
});
response = result.response;
candidates = response?.candidates;
}
if (logRequests)
console.info(safeStringify({ direction: "response", model, payload: { candidates, promptFeedback: response?.promptFeedback } }, redactSecrets));
/* 4. Process response */
if (!Array.isArray(candidates) || candidates.length === 0) {
if (debugChunks)
console.warn("Gemini: no candidates in response. Raw response:", safeStringify(response, redactSecrets));
return;
}
// Vision Promotion is always ON
await this.processCandidates(candidates, context, safeToOriginal, true, shouldUseFilesApi, caps);
if (debugChunks)
console.info("Generation completed.");
}
catch (error) {
this.handleError(error, context, genAI, generateContent, systemInstruction, shouldUseFilesApi);
}
}
modifyContents(contents, caps) {
// Default: do nothing
}
modifyGenerationConfig(generateContent, context, caps) {
// Default: do nothing
}
async reconcileAttachments(context, shouldUseFilesApi) {
const { ctl, history, debugChunks, globalConfig } = context;
const chatWd = ctl.getWorkingDirectory();
try {
// Use unified SSOT scan from attachments.ts
const ssotPaths = await findAllAttachmentsFromConversation(chatWd, !!debugChunks);
if (ssotPaths.length === 0) {
if (debugChunks)
console.info('[Attachment Reconcile] No attachments in history; preserving existing state');
return;
}
// Read current state (or initialize empty)
const state = await readChatMediaState(chatWd).catch(() => ({
attachments: [],
variants: [],
counters: { nextN: 1, nextV: 1 }
}));
// Use importAttachmentBatch for stable n-numbering, idempotent, no copies
const result = await importAttachmentBatch(chatWd, state, ssotPaths, { maxDim: 1024, quality: 85 }, 2, // max 2 attachments
!!debugChunks);
if (result.changed && debugChunks) {
console.info(`[Attachment Reconcile] Imported attachments from SSOT`);
}
}
catch (e) {
if (debugChunks)
console.warn('[Attachment Reconcile] Error:', e.message);
}
}
async backfillAnalysisPreviews(context, shouldUseFilesApi) {
// DEPRECATED: Preview generation is now handled by importAttachmentBatch in reconcileAttachments
// This method is kept for compatibility but does nothing
// The new preview naming is: preview-<origin> (e.g., preview-1766100380042 - 811.jpg)
}
async processCandidates(candidates, context, safeToOriginal, _allowVisionPromotion, shouldUseFilesApi, caps) {
// Note: _allowVisionPromotion is deprecated - Vision Promotion is always ON
const { ctl, debugChunks } = context;
const mimeToExt = (mime) => {
const m = (mime || "").toLowerCase();
if (m.includes("jpeg") || m === "image/jpg")
return ".jpg";
if (m.includes("png"))
return ".png";
if (m.includes("webp"))
return ".webp";
if (m.includes("gif"))
return ".gif";
if (m.includes("bmp"))
return ".bmp";
if (m.includes("svg"))
return ".svg";
return ".png";
};
for (const candidate of candidates) {
const parts = candidate?.content?.parts;
if (debugChunks)
console.info("Processing candidate parts:", JSON.stringify(parts));
if (Array.isArray(parts)) {
let textBuf = "";
const images = [];
const toolCalls = [];
for (const part of parts) {
// Capture Thought Signature (Only if model supports thinking)
if (caps?.supportsThinking) {
const sig = part.thought_signature || part.thoughtSignature;
if (sig) {
try {
let hash = "";
if (part.text && !part.functionCall && !part.function_call) {
hash = computeContentHash$1(part.text);
}
else {
const fcall = part.functionCall || part.function_call;
if (fcall && fcall.name) {
const name = String(fcall.name);
let args = fcall.args;
if (typeof args === "string") {
try {
args = JSON.parse(args);
}
catch {
args = {};
}
}
const id = `${name}:${stableJsonStringify(args || {})}`;
hash = computeContentHash$1(id);
}
}
if (hash) {
await appendSignature(ctl.getWorkingDirectory(), sig, hash);
if (debugChunks)
console.info("Captured thought signature for hash:", hash);
}
}
catch (e) {
if (debugChunks)
console.warn("Failed to capture thought signature:", e);
}
}
}
if (part?.text)
textBuf += (textBuf ? "\n" : "") + part.text;
const b64 = part?.inline_data?.data || part?.inlineData?.data;
if (b64) {
const mime = part?.inline_data?.mime_type || part?.inlineData?.mimeType || "image/png";
images.push({ data: b64, mimeType: mime });
}
const fcall = part?.functionCall || part?.function_call;
if (fcall && fcall.name) {
let args = fcall.args;
if (typeof args === "string") {
try {
args = JSON.parse(args);
}
catch { /* keep as string */ }
}
toolCalls.push({ name: String(fcall.name), args });
// Subclass hook: persist thought_signature for tool calls if needed.
try {
const sig = part?.thought_signature || part?.thoughtSignature;
await this.onObservedFunctionCallPart(context, String(fcall.name), args, sig);
}
catch { /* best-effort */ }
}
}
const allowedSafe = new Set(safeToOriginal ? Array.from(safeToOriginal.keys()) : []);
const filteredToolCalls = toolCalls.filter(tc => allowedSafe.has(tc.name));
if (textBuf.trim().length) {
if (debugChunks)
console.info("Streaming text (simulated):", textBuf.slice(0, 120));
await streamTextFragments(ctl, textBuf);
}
if (filteredToolCalls.length) {
const [tc] = filteredToolCalls;
const originalName = safeToOriginal.get(tc.name) || tc.name;
const argsJson = typeof tc.args === "string" ? tc.args : JSON.stringify(tc.args ?? {});
const callId = `gemini-fc-${Date.now()}-0`;
ctl.toolCallGenerationStarted();
ctl.toolCallGenerationNameReceived(originalName);
ctl.toolCallGenerationArgumentFragmentGenerated(argsJson);
ctl.toolCallGenerationEnded({ type: "function", name: originalName, arguments: tc.args ?? {}, id: callId });
// Same reasoning as streaming path: stop immediately after emitting a tool call.
return;
}
if (images.length > 0) {
const wd = ctl.getWorkingDirectory();
const fileNames = [];
const analysisNames = [];
const ts = toIsoLikeTimestamp(new Date());
let idx = 0;
for (const img of images) {
const baseName = images.length > 1 ? `image-${ts}-v${++idx}` : `image-${ts}`;
const ext = mimeToExt(img.mimeType || "");
const fileName = `${baseName}${ext}`;
const abs = path.join(wd, fileName);
try {
const buf = Buffer.from(img.data, "base64");
await fs.promises.writeFile(abs, buf);
fileNames.push(fileName);
if (!shouldUseFilesApi) {
// Vision Promotion is always ON - create analysis preview
try {
const iso = toIsoLikeTimestamp(new Date());
const v = images.length > 1 ? idx : 1;
const analysisName = `analysis-generated-image-${iso}-v${v}.jpg`;
const analysisAbs = path.join(wd, analysisName);
const jpeg = await encodeJpegFromBuffer(buf, 85);
await fs.promises.writeFile(analysisAbs, jpeg);
analysisNames.push(analysisName);
if (debugChunks)
console.info("Flash analysis JPEG written:", analysisAbs);
}
catch (e) {
if (debugChunks)
console.error("Failed to write analysis JPEG:", e?.message);
}
}
}
catch (e) {
if (debugChunks)
console.error("Failed to write image file:", e?.message);
}
}
if (fileNames.length > 0) {
const md = fileNames.map(fn => ``).join("\n\n");
ctl.fragmentGenerated("\n\n" + md + "\n");
try {
const variants = fileNames.map((fn, i) => ({ filename: fn, preview: analysisNames[i] ?? fn }));
if (variants.length)
await recordVariantsProvision(wd, variants);
}
catch { }
}
}
}
}
}
async handleError(error, context, genAI, generateContent, systemInstruction, shouldUseFilesApi) {
const { ctl, debugChunks, logRequests, model } = context;
const rawMessage = error?.message || String(error);
if (rawMessage.includes("Unexpected token") && (rawMessage.includes("JSON") || rawMessage.includes("<"))) {
console.error("----------------------------------------------------------------");
console.error("CRITICAL ERROR: The Vertex AI SDK received an HTML response instead of JSON.");
console.error("This usually indicates a configuration issue (wrong Project ID, Location, or Model Name).");
console.error("It can also mean the Service Account lacks permissions or the API is down.");
if (error.response) {
const r = error.response;
console.error(`HTTP Status: ${r.status} ${r.statusText}`);
console.error(`URL: ${r.url}`);
}
console.error("----------------------------------------------------------------");
}
if (logRequests || debugChunks) {
try {
console.error("Full Error Object Dump:", JSON.stringify(error, Object.getOwnPropertyNames(error), 2));
}
catch {
console.error("Full Error Object Dump: [Circular or Unserializable]");
}
}
if (/function calling is not enabled/i.test(rawMessage)) {
try {
const generativeModel = genAI.getGenerativeModel({
model: model,
...(systemInstruction ? { system_instruction: systemInstruction } : {}),
});
const result2 = await generativeModel.generateContent({
contents: generateContent.contents,
});
const response2 = result2.response;
const candidates = response2?.candidates;
if (Array.isArray(candidates) && candidates.length) {
// Vision Promotion is always ON
await this.processCandidates(candidates, context, new Map(), true, shouldUseFilesApi);
return;
}
}
catch (e2) {
console.error("Gemini fallback error:", e2);
}
}
console.error("Gemini SDK error:", rawMessage);
throw new Error(`Gemini SDK error: ${rawMessage}`);
}
}
// import { flattenToolCallsToText } from "../generator-utils";
class GeminiThinkingStrategy extends BaseGeminiStrategy {
// Persist thought_signature for tool calls so LM Studio's auto-continue tool-loop
// can replay functionCall parts without 400s on gemini-3-pro-preview.
async onObservedFunctionCallPart(context, safeName, args, sig) {
try {
if (!sig)
return;
const id = `${String(safeName)}:${stableJsonStringify(args ?? {})}`;
const hash = computeContentHash$1(id);
await appendSignature(context.ctl.getWorkingDirectory(), sig, hash);
if (context.debugChunks)
console.info("[ThoughtSig] Captured tool-call signature for hash:", hash);
}
catch {
// best-effort
}
}
modifyContents(contents, caps) {
// IMPORTANT (gemini-3-pro-preview): the Gemini API requires thought_signature on *request*
// functionCall parts for thinking models. In LM Studio's tool-call loop, replaying the
// prior assistant tool call as a functionCall part can 400 if the signature is missing.
//
// To keep the system robust and encapsulated, we flatten historical functionCall parts
// into plain text. Tools still work: new tool calls can be emitted by the model, and
// tool results remain in history as functionResponse parts.
if (!Array.isArray(contents))
return;
for (const msg of contents) {
if (!msg || typeof msg !== "object")
continue;
const parts = msg.parts;
if (!Array.isArray(parts))
continue;
let changed = false;
const newParts = [];
for (const p of parts) {
const fc = p?.functionCall || p?.function_call;
if (fc && fc.name) {
const name = String(fc.name);
const args = fc.args;
const argsJson = typeof args === "string" ? args : JSON.stringify(args ?? {});
newParts.push({ text: `[ToolCall] ${name} ${argsJson}` });
changed = true;
}
else {
newParts.push(p);
}
}
if (changed)
msg.parts = newParts;
}
}
modifyGenerationConfig(generateContent, context, caps) {
if (caps.supportsThinking) {
const { pluginConfig } = context;
const thinkingLevel = pluginConfig.get("thinkingLevel");
const thinkingConfig = {
includeThoughts: true,
};
// Only add thinkingLevel if the model supports specific levels
if (caps.thinking?.levels && caps.thinking.levels.length > 0) {
thinkingConfig.thinkingLevel = thinkingLevel;
}
generateContent.generationConfig = {
...(generateContent.generationConfig || {}),
thinkingConfig
};
}
}
}
function getThoughtStatePath(wd) {
// Use a separate file to avoid conflict with legacy array-based thought-signatures.json
return path.join(wd, "thought-signatures-image.json");
}
function computeContentHash(text) {
return crypto
.createHash("sha256")
.update(text ? text.trim().replace(/\s+/g, " ") : "")
.digest("hex");
}
function loadThoughtState(wd) {
try {
// 1. Try loading the new isolated file
const fp = getThoughtStatePath(wd);
if (fs.existsSync(fp)) {
const content = fs.readFileSync(fp, "utf-8");
const parsed = JSON.parse(content);
if (parsed.signatures && !Array.isArray(parsed.signatures)) {
return parsed;
}
}
// 2. Fallback: Try loading the legacy file (migration)
const legacyFp = path.join(wd, "thought_signatures.json");
if (fs.existsSync(legacyFp)) {
const content = fs.readFileSync(legacyFp, "utf-8");
const parsed = JSON.parse(content);
if (parsed.signatures && Array.isArray(parsed.signatures)) {
const newSigs = {};
for (const s of parsed.signatures) {
if (s.contentHash && s.signature) {
newSigs[s.contentHash] = s.signature;
}
}
return { signatures: newSigs };
}
}
// 3. Fallback: Try loading the V3 file (migration from previous isolated strategy)
const v3Fp = path.join(wd, "thought_signatures_v3.json");
if (fs.existsSync(v3Fp)) {
const content = fs.readFileSync(v3Fp, "utf-8");
const parsed = JSON.parse(content);
if (parsed.signatures && Array.isArray(parsed.signatures)) {
const newSigs = {};
for (const s of parsed.signatures) {
// V3 format had { signature, contentHash, timestamp }
if (s.contentHash && s.signature) {
newSigs[s.contentHash] = s.signature;
}
}
return { signatures: newSigs };
}
}
}
catch { }
return { signatures: {} };
}
function saveThoughtState(wd, state) {
try {
const fp = getThoughtStatePath(wd);
fs.writeFileSync(fp, JSON.stringify(state, null, 2), "utf-8");
}
catch { }
}
/**
* Leichtgewichtige Policy speziell für gemini-3-pro-image-preview.
*
* - ignoriert Bild-Entwürfe in der Signatur-Logik
* - speichert nur Signaturen für den aktuellen Turn (Text/Tools)
* - pruned ThoughtState nach jedem Turn aggressiv auf ein Minimal-Set
*/
class ImageLightweightReasoningPolicy {
buildContents(params) {
const contents = params.baseContentsBuilder();
const chatWd = params.context.ctl.getWorkingDirectory();
const state = loadThoughtState(chatWd);
const { debugChunks } = params.context;
if (debugChunks) {
const keys = Object.keys(state.signatures);
console.info("[ImageReasoningPolicy] buildContents - signatures keys:", keys.slice(0, 10), "total=", keys.length);
}
// Aktuell keine Injection; der State wird nur beobachtet.
return contents;
}
updateFromResponse(params) {
const { capturedSignature, collectedFullText, collectedToolCalls, context } = params;
if (!capturedSignature)
return;
const chatWd = context.ctl.getWorkingDirectory();
const state = loadThoughtState(chatWd);
const { debugChunks } = context;
// 1) Volltext (inkl. thoughts) hashen und auf Signatur mappen
if (collectedFullText && collectedFullText.trim()) {
const fullHash = computeContentHash(collectedFullText);
state.signatures[fullHash] = capturedSignature;
// optional: getrimmte Variante für Robustheit
const trimmed = collectedFullText.trim();
if (trimmed !== collectedFullText) {
const trimmedHash = computeContentHash(trimmed);
state.signatures[trimmedHash] = capturedSignature;
}
}
// 2) Fallback-Schlüssel für "letzter Text-Turn"
state.signatures["LATEST_TEXT_SIG"] = capturedSignature;
// 3) ToolCalls des Turns auf dieselbe Signatur mappen
for (const tc of collectedToolCalls || []) {
const id = `${tc.name}:${JSON.stringify(tc.args ?? {})}`;
const hash = computeContentHash(id);
state.signatures[hash] = capturedSignature;
}
// 4) Aggressives Pruning: nur noch das Minimal-Set behalten.
const minimal = { signatures: {} };
for (const [key, value] of Object.entries(state.signatures)) {
// Wir behalten nur Einträge aus diesem Turn plus LATEST_TEXT_SIG.
// Für das Skeleton gehen wir pragmatisch vor und behalten alles,
// was gerade auf capturedSignature zeigt.
if (value === capturedSignature || key === "LATEST_TEXT_SIG") {
minimal.signatures[key] = value;
}
}
saveThoughtState(chatWd, minimal);
if (debugChunks) {
const keys = Object.keys(minimal.signatures);
console.info("[ImageReasoningPolicy] updateFromResponse - kept signature keys:", keys);
}
}
}
class GeminiImageThinkingStrategy extends BaseGeminiStrategy {
async saveImage(wd, data, filename) {
const abs = path.join(wd, filename);
try {
const buf = Buffer.from(data, "base64");
await fs.promises.writeFile(abs, buf);
return filename;
}
catch (e) {
console.error(`[GeminiImageThinkingStrategy] Failed to save image ${filename}:`, e);
// Fallback: Check if file exists and has content (maybe saved by parallel process or race condition?)
try {
const stats = await fs.promises.stat(abs);
if (stats.size > 0) {
console.warn(`[GeminiImageThinkingStrategy] Write failed but file exists ${filename}. Recovering.`);
return filename;
}
}
catch { }
return null;
}
}
// Fully isolated implementation of generate for this strategy
async generate(context) {
const { ctl, history, model, apiKey, globalConfig, pluginConfig, debugChunks, logRequests } = context;
const visionPromotionPersistent = globalConfig.get("visionPromotionPersistent");
const useFilesApiForVision = pluginConfig.get("useFilesApiForVision");
const showOnlyLastImageVariant = pluginConfig.get("showOnlyLastImageVariant");
const redactSecrets = [];
const genAI = new generativeAi.GoogleGenerativeAI(apiKey);
const caps = detectCapabilities(model);
if (debugChunks)
console.info("[GeminiImageThinkingStrategy] Isolated generate call. model=", model);
const lastUserText = getLastUserText(history);
const supportsFunctionCalling = caps.supportsTools;
const systemText = collectSystemText(history);
const chatWd = ctl.getWorkingDirectory();
await snapshotHistoryMediaState(ctl, history, chatWd, model);
const shouldUseFilesApi = shouldUseFilesApiForModel(model, useFilesApiForVision);
// Reconcile Attachments (Duplicated logic from Base)
await this.localReconcileAttachments(context, shouldUseFilesApi);
await this.localBackfillAnalysisPreviews(context, shouldUseFilesApi);
const { tools: geminiTools, originalToSafe, safeToOriginal } = supportsFunctionCalling
? buildGeminiTools(ctl, lastUserText || "")
: { tools: undefined, originalToSafe: new Map(), safeToOriginal: new Map() };
// Manage Thought Signatures (via optional policy, gated to gemini-3-pro-image-preview)
let contents;
if (model === "gemini-3-pro-image-preview" && caps.supportsThinking) {
const policy = new ImageLightweightReasoningPolicy();
contents = policy.buildContents({
history,
context,
baseContentsBuilder: () => {
const sigState = loadThoughtState(chatWd);
const signaturesArray = Object.entries(sigState.signatures).map(([contentHash, signature]) => ({ contentHash, signature }));
return toGeminiMessages(history, originalToSafe, signaturesArray);
},
});
}
else {
let signaturesArray = [];
if (caps.supportsThinking) {
const sigState = loadThoughtState(chatWd);
signaturesArray = Object.entries(sigState.signatures).map(([contentHash, signature]) => ({ contentHash, signature }));
}
contents = toGeminiMessages(history, originalToSafe, signaturesArray);
}
this.modifyContents(contents, caps);
// Vision promotion (Duplicated logic)
try {
// Auto-detect turns that are pure tool/functionResponse replays and
// suppress vision promotion for them to avoid mismatched thought_signature
// requirements on implicitly promoted images.
if (typeof context.suppressVisionPromotionForThisTurn === "undefined") {
try {
const msgs = Array.from(history);
if (msgs.length > 0) {
const last = msgs[msgs.length - 1];
const role = typeof last.getRole === "function" ? last.getRole() : undefined;
const results = typeof last.getToolCallResults === "function" ? last.getToolCallResults() : undefined;
if (role === "tool" && Array.isArray(results) && results.length > 0) {
if (debugChunks)
console.info("[ImageThinking Vision Suppress] Last history message is a tool result; suppressing vision promotion for this turn.");
context.suppressVisionPromotionForThisTurn = true;
}
}
}
catch { /* best-effort only */ }
}
let promoParts = [];
const res = await buildPromotionPartsForMode({
ctl,
history,
apiKey,
chatWd,
debugChunks,
shouldUseFilesApi,
model,
visionPromotionPersistent,
useFilesApiForVision: !!useFilesApiForVision,
suppressVisionPromotionForThisTurn: !!context.suppressVisionPromotionForThisTurn,
});
promoParts = res.promoParts || [];
if (promoParts.length) {
let lastUserMessage = contents.slice().reverse().find(m => m.role === 'user');
if (lastUserMessage) {
lastUserMessage.parts = [...promoParts, ...(lastUserMessage.parts || [])];
}
else {
contents.push({ role: "user", parts: promoParts });
}
}
}
catch (e) {
if (debugChunks)
console.error("Promotion parts error:", e.message);
}
// System Instruction
let systemInstruction;
{
const parts = [];
if (systemText)
parts.push({ text: systemText });
const now = new Date();
const tz = "UTC";
parts.push({ text: `Current date/time: ${now.toISOString()} (${tz})` });
if (!supportsFunctionCalling) {
parts.push({ text: "You are an image-capable model without tool support." });
}
else if (geminiTools) {
parts.push({ text: "Tool use policy: Use available tools only when necessary." });
}
if (parts.length)
systemInstruction = { parts };
}
const generateContent = { contents };
if (geminiTools && supportsFunctionCalling) {
generateContent.tools = geminiTools;
generateContent.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
}
if (systemInstruction)
generateContent.systemInstruction = systemInstruction;
this.modifyGenerationConfig(generateContent, context, caps);
try {
// logRequests: nur Rohdaten ohne Telemetrie-Präfixe
if (logRequests)
console.info(safeStringify({ direction: "request", model, payload: generateContent }, redactSecrets));
const generativeModel = genAI.getGenerativeModel({
model: model,
...(systemInstruction ? { system_instruction: systemInstruction } : {}),
safetySettings: [{ category: generativeAi.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, threshold: generativeAi.HarmBlockThreshold.BLOCK_NONE }],
});
let capturedSignature;
let collectedText = "";
let collectedFullText = ""; // Includes thoughts, for hashing
let collectedToolCalls = [];
// Image handling state
const ts = toIsoLikeTimestamp(new Date());
let imageCount = 0;
const collectedImages = [];
if (caps.supportsStreaming) {
const stream = await generativeModel.generateContentStream({
contents: generateContent.contents,
tools: generateContent.tools,
toolConfig: generateContent.toolConfig,
generationConfig: generateContent.generationConfig,
});
let toolCallEmitted = false;
const allowedSafe = new Set(safeToOriginal ? Array.from(safeToOriginal.keys()) : []);
for await (const item of stream.stream) {
if (debugChunks)
console.info("[Chunk]", safeStringify(item));
const scands = item?.candidates;
if (!Array.isArray(scands) || !scands.length)
continue;
const candidate = scands[0];
const parts = candidate?.content?.parts;
if (!Array.isArray(parts))
continue;
if (debugChunks) {
try {
const partKinds = parts.map((p) => ({
hasInline: !!(p?.inline_data || p?.inlineData),
hasThought: !!p?.thought,
hasSig: !!(p?.thought_signature || p?.thoughtSignature),
hasText: typeof p?.text === "string",
}));
}
catch { /* ignore */ }
}
let textBuf = "";
const toolCalls = [];
for (const part of parts) {
const p = part;
const isImageLink = typeof p.text === 'string' && /!\[Image\]\(.*?\)/.test(p.text);
const hasSig = p.thought_signature || p.thoughtSignature;
// Handle inline images (Thinking images or final images)
const b64 = p.inline_data?.data || p.inlineData?.data;
if (b64) {
imageCount++;
const mime = p.inline_data?.mime_type || p.inlineData?.mimeType || "image/png";
const ext = mime.includes("jpeg") ? ".jpg" : ".png";
// Use standard naming convention: image-TIMESTAMP-vCOUNT.ext
const filename = `image-${ts}-v${imageCount}${ext}`;
const savedName = await this.saveImage(chatWd, b64, filename);
if (savedName) {
let preview = savedName;
// ALWAYS create analysis preview for variants (for local display & Base64 fallback)
try {
const analysisName = `analysis-generated-image-${ts}-v${imageCount}.jpg`;
const buf = Buffer.from(b64, "base64");
const jpeg = await encodeJpegFromBuffer(buf, 85);
await fs.promises.writeFile(path.join(chatWd, analysisName), jpeg);
preview = analysisName;
if (debugChunks)
console.info("[Streaming] Created analysis preview:", analysisName);
}
catch (e) {
if (debugChunks)
console.warn("[Streaming] Failed to create analysis preview:", e);
}
// Always collect every physical variant for media-state,
// regardless of showOnlyLastImageVariant (which is a pure UI/promotion flag).
collectedImages.push({ filename: savedName, preview });
if (debugChunks)
console.info(`[Streaming] Collected image ${imageCount}: ${savedName} (Total: ${collectedImages.length})`);
// Injection into the chat stream is controlled by showOnlyLastImageVariant,
// but this must NOT affect what we persist in chat_media_state.json.
if (!showOnlyLastImageVariant) {
const md = `\n\n\n\n`;
ctl.fragmentGenerated(md);
collectedText += md;
collectedFullText += md;
if (debugChunks)
console.info("[Streaming] Image saved and injected:", savedName);
}
else {
if (debugChunks)
console.info("[Streaming] Image saved but injection suppressed (showOnlyLastImageVariant=true):", savedName);
}
}
else {
if (debugChunks)
console.error("[Streaming] Failed to save image, skipping collection:", filename);
}
}
if (hasSig) {
capturedSignature = hasSig;
if (debugChunks)
console.info("[GeminiImageThinkingStrategy] Captured thought signature (truncated):", hasSig.slice(0, 50) + "...");
}
// Vertrauen in die API: Gedanken werden explizit mit thought=true markiert.
const isThought = !isImageLink && !!p.thought;
if (p.text && !isImageLink && !isThought) {
collectedText += p.text;
}
if (isThought) {
const content = typeof p.thought === 'string' ? p.thought : (p.text || "");
if (content) {
if (debugChunks)
console.info("[Streaming] Thought update:", content.slice(0, 120));
ctl.fragmentGenerated(content, { reasoningType: "reasoning" });
collectedFullText += content;
}
}
else if (p.text) {
// ungekennzeichneter Text ist finale Antwort (kein Thought)
textBuf += (textBuf ? "\n" : "") + p.text;
}
const fcall = part?.functionCall || part?.function_call;
if (fcall && fcall.name) {
// Function-Calls signalisieren das Ende der reinen Thinking-Phase,
// beeinflussen aber nur das Signatur-Handling, nicht isThought.
let args = fcall.args;
if (typeof args === "string") {
try {
args = JSON.parse(args);
}
catch { }
}
toolCalls.push({ name: String(fcall.name), args });
collectedToolCalls.push({ name: String(fcall.name), args });
if (debugChunks)
console.info("[Streaming] Tool call detected:", JSON.stringify({ name: fcall.name, args }));
}
}
if (textBuf.trim().length) {
if (debugChunks)
console.info("[Streaming] Text update:", textBuf.slice(0, 120));
ctl.fragmentGenerated(textBuf);
collectedFullText += textBuf;
}
if (!toolCallEmitted && toolCalls.length) {
const streamingToolCall = toolCalls.find(tc => allowedSafe.has(tc.name));
if (streamingToolCall) {
const originalName = safeToOriginal.get(streamingToolCall.name) || streamingToolCall.name;
const argsJson = typeof streamingToolCall.args === "string" ? streamingToolCall.args : JSON.stringify(streamingToolCall.args ?? {});
const callId = `gemini-fc-${Date.now()}-0`;
ctl.toolCallGenerationStarted();
ctl.toolCallGenerationNameReceived(originalName);
ctl.toolCallGenerationArgumentFragmentGenerated(argsJson);
ctl.toolCallGenerationEnded({ type: "function", name: originalName, arguments: streamingToolCall.args ?? {}, id: callId });
toolCallEmitted = true;
// CRITICAL (LM Studio tool-call loop): once we emit a tool call, end this
// generate() invocation immediately so LM Studio can run the tool and
// re-invoke generate() with the tool result.
ctl.fragmentGenerated("");
return;
}
}
}
ctl.fragmentGenerated("");
const response = stream.response;
if (logRequests)
console.info(safeStringify({ direction: "response", model, payload: response }, redactSecrets));
const candidates = response?.candidates;
if (Array.isArray(candidates) && candidates.length > 0) {
// Vision Promotion is always ON
const { markdown, savedImages } = await this.localProcessCandidates(candidates, context, safeToOriginal, true, shouldUseFilesApi, caps, true);
collectedText += markdown;
collectedFullText += markdown;
// In streaming, localProcessCandidates ignores images, so savedImages will be empty.
// Robustness: if the final response snapshot contains additional
// non-thinking text that was never streamed, render it once here.
try {
let responseText = "";
for (const cand of candidates) {
const parts = cand?.content?.parts;
if (!Array.isArray(parts))
continue;
for (const p of parts) {
const isImageLink = typeof p.text === "string" && /!\[Image\]\(.*?\)/.test(p.text);
const isThought = !isImageLink && !!p.thought;
if (p.text && !isImageLink && !isThought) {
responseText += (responseText ? "\n" : "") + p.text;
}
}
}
if (responseText.trim().length) {
if (debugChunks)
console.info("[Streaming] Final snapshot text update:", responseText.slice(0, 200));
ctl.fragmentGenerated(responseText);
collectedText += responseText;
collectedFullText += responseText;
}
}
catch { /* best-effort only */ }
}
// If showOnlyLastImageVariant is enabled, inject the LAST image now
if (showOnlyLastImageVariant && collectedImages.length > 0) {
const lastImage = collectedImages[collectedImages.length - 1];
const md = `\n\n\n\n`;
ctl.fragmentGenerated(md);
collectedText += md;
collectedFullText += md;
}
}
else {
const result = await generativeModel.generateContent({
contents: generateContent.contents,
tools: generateContent.tools,
toolConfig: generateContent.toolConfig,
});
const candidates = result.response?.candidates;
if (logRequests)
console.info(safeStringify({ direction: "response", model, payload: result.response }, redactSecrets));
if (candidates) {
// For non-streaming, we need to extract signature and text from candidates
for (const cand of candidates) {
const parts = cand?.content?.parts;
if (Array.isArray(parts)) {
for (const p of parts) {
if (p.thought_signature || p.thoughtSignature)
capturedSignature = p.thought_signature || p.thoughtSignature;
if (p.text) {
collectedText += p.text;
collectedFullText += p.text;
}
const fcall = p.functionCall || p.function_call;
if (fcall && fcall.name) {
let args = fcall.args;
if (typeof args === "string") {
try {
args = JSON.parse(args);
}
catch { }
}
collectedToolCalls.push({ name: String(fcall.name), args });
}
}
}
}
// Vision Promotion is always ON
const { markdown, savedImages } = await this.localProcessCandidates(candidates, context, safeToOriginal, true, shouldUseFilesApi, caps, false);
collectedText += markdown;
collectedFullText += markdown;
collectedImages.push(...savedImages);
}
}
// Save Thought Signature (Post-Turn)
if (capturedSignature && (collectedFullText.trim() || collectedToolCalls.length > 0)) {
if (model === "gemini-3-pro-image-preview" && caps.supportsThinking) {
const policy = new ImageLightweightReasoningPolicy();
policy.updateFromResponse({
response: undefined,
history,
context,
collectedFullText,
collectedToolCalls,
capturedSignature,
});
if (debugChunks)
console.info("[ReasoningPolicy] Updated thought state for gemini-3-pro-image-preview.");
}
else {
const thoughtState = loadThoughtState(chatWd);
// 1. Hash full text (including thoughts)
if (collectedFullText) {
const hash = computeContentHash(collectedFullText);
thoughtState.signatures[hash] = capturedSignature;
// 2. Hash trimmed text (robustness)
if (collectedFullText.trim() !== collectedFullText) {
const trimmedHash = computeContentHash(collectedFullText.trim());
thoughtState.signatures[trimmedHash] = capturedSignature;
}
}
// 3. ALWAYS Save as LATEST_TEXT_SIG for immediate fallback (robustness against hash mismatches)
thoughtState.signatures["LATEST_TEXT_SIG"] = capturedSignature;
// 4. Fallback for tool-only responses (if no text at all)
if (collectedToolCalls.length > 0 && !collectedFullText.trim()) {
const emptyHash = computeContentHash("");
thoughtState.signatures[emptyHash] = capturedSignature;
}
// 5. Save for tool calls (CRITICAL for mixed text/tool turns)
if (collectedToolCalls.length > 0) {
for (const tc of collectedToolCalls) {
// Must match getSignatureForFunctionCall in generator-utils.ts
const id = `${tc.name}:${stableJsonStringify(tc.args ?? {})}`;
const hash = computeContentHash(id);
thoughtState.signatures[hash] = capturedSignature;
}
}
saveThoughtState(chatWd, thoughtState);
if (debugChunks)
console.info("Saved thought signature for turn.");
}
}
// Update Media State (for Vision Promotion)
// OLD/CONSISTENT BEHAVIOR: keep only the LAST visible variant in state,
// so both first and subsequent runs behave identically.
if (collectedImages.length > 0) {
const lastImage = collectedImages[collectedImages.length - 1];
await recordVariantsProvision(chatWd, [lastImage]);
}
else {
if (debugChunks && imageCount > 0) {
}
}
}
catch (error) {
console.error("Gemini Isolated Strategy Error:", error);
throw error;
}
}
/**
* @param _allowVisionPromotion - deprecated; Vision Promotion is now always ON.
* Kept in signature for call-site compat; ignored internally.
*/
async localProcessCandidates(candidates, context, safeToOriginal, _allowVisionPromotion, shouldUseFilesApi, caps, isStreaming = false) {
const { ctl, debugChunks, pluginConfig } = context;
const showOnlyLastImageVariant = pluginConfig.get("showOnlyLastImageVariant");
ctl.getWorkingDirectory();
const mimeToExt = (mime) => {
if (mime.includes("jpeg"))
return ".jpg";
if (mime.includes("png"))
return ".png";
return ".png";
};
// Refactoring to return generated markdown and saved images
let generatedMarkdown = "";
const savedImages = [];
for (const candidate of candidates) {
const parts = candidate?.content?.parts;
if (Array.isArray(parts)) {
let textBuf = "";
caps?.supportsThinking;
const images = [];
for (const part of parts) {
part.thought_signature || part.thoughtSignature;
const isImageLink = typeof part.text === 'string' && /!\[Image\]\(.*?\)/.test(part.text);
const isThought = !isImageLink && !!part.thought;
if (part?.text && !isThought) {
textBuf += (textBuf ? "\n" : "") + part.text;
}
// Only process images here if NOT streaming (streaming handles them in real-time)
if (!isStreaming) {
const b64 = part?.inline_data?.data || part?.inlineData?.data;
if (b64) {
images.push({ data: b64, mimeType: part?.inline_data?.mime_type || "image/png" });
}
}
}
if (textBuf.trim().length && !isStreaming) {
await streamTextFragments(ctl, textBuf);
}
if (images.length > 0) {
const wd = ctl.getWorkingDirectory();
const fileNames = [];
const ts = toIsoLikeTimestamp(new Date());
let idx = 0;
for (const img of images) {
const baseName = images.length > 1 ? `image-${ts}-v${++idx}` : `image-${ts}`;
const ext = mimeToExt(img.mimeType);
const fileName = `${baseName}${ext}`;
const abs = path.join(wd, fileName);
try {
const buf = Buffer.from(img.data, "base64");
await fs.promises.writeFile(abs, buf);
fileNames.push(fileName);
// For non-streaming, we don't generate separate analysis previews yet, so use the file itself
savedImages.push({ filename: fileName, preview: fileName });
}
catch { }
}
if (fileNames.length > 0) {
if (showOnlyLastImageVariant) {
// Only inject the LAST image
const lastFn = fileNames[fileNames.length - 1];
const md = `\n\n\n\n`;
ctl.fragmentGenerated(md);
generatedMarkdown += md;
}
else {
const md = fileNames.map(fn => ``).join("\n\n");
const fragment = "\n\n" + md + "\n";
ctl.fragmentGenerated(fragment);
generatedMarkdown += fragment;
}
}
}
}
}
return { markdown: generatedMarkdown, savedImages };
}
async localReconcileAttachments(context, shouldUseFilesApi) {
const { ctl, history, debugChunks } = context;
const chatWd = ctl.getWorkingDirectory();
try {
// Use unified SSOT scan from attachments.ts (last turn only for reconcile)
const ssotPaths = await findAllAttachmentsFromLastTurn(chatWd, !!debugChunks);
if (ssotPaths.length === 0) {
if (debugChunks)
console.info('[Image Strategy Attachment Reconcile] No attachments in history; preserving existing state');
return;
}
// Read current state (or initialize empty)
const state = await readChatMediaState(chatWd).catch(() => ({
attachments: [],
variants: [],
counters: { nextN: 1, nextV: 1 }
}));
// Use importAttachmentBatch for stable n-numbering, idempotent, no copies
const result = await importAttachmentBatch(chatWd, state, ssotPaths, { maxDim: 1024, quality: 85 }, 2, // max 2 attachments
!!debugChunks);
if (result.changed && debugChunks) {
console.info(`[Image Strategy Attachment Reconcile] Imported attachments from SSOT`);
}
}
catch (e) {
if (debugChunks)
console.warn('[Image Strategy Attachment Reconcile] Error:', e.message);
}
}
async localBackfillAnalysisPreviews(context, shouldUseFilesApi) {
// DEPRECATED: Preview generation is now handled by importAttachmentBatch in localReconcileAttachments
// This method is kept for compatibility but does nothing
// The new preview naming is: preview-<origin> (e.g., preview-1766100380042 - 811.jpg)
}
modifyContents(contents, caps) {
// No-op here because we handle signature injection in generate() via toGeminiMessages
// But wait, toGeminiMessages needs the signatures passed to it.
// In my generate() override, I call:
// const sigState = await loadSignaturesV3(chatWd);
// const contents = toGeminiMessages(history, originalToSafe, sigState.signatures);
// So it is handled there.
}
modifyGenerationConfig(generateContent, context, caps) {
const { pluginConfig } = context;
const thinkingConfig = {
includeThoughts: true,
};
// Only add thinkingLevel if the model supports specific levels (gemini-3-pro-preview does, gemini-3-pro-image-preview does not)
if (caps.thinking?.levels && caps.thinking.levels.length > 0) {
const thinkingLevel = pluginConfig.get("thinkingLevel");
thinkingConfig.thinkingLevel = thinkingLevel;
}
generateContent.generationConfig = {
...(generateContent.generationConfig || {}),
thinkingConfig
};
if (caps.responseModalities) {
generateContent.generationConfig.responseModalities = caps.responseModalities;
}
}
}
class StrategyFactory {
static getStrategy(modelName) {
// gemini-3-pro-image-preview has strict thought-signature constraints for image parts.
// In LM Studio's tool-call auto-continue loop, we must suppress vision promotion on the
// tool-result replay turn to avoid mismatched signatures.
if (modelName === "gemini-3-pro-image-preview") {
return new GeminiImageThinkingStrategy();
}
const caps = detectCapabilities(modelName);
if (caps.supportsThinking) {
return new GeminiThinkingStrategy();
}
return new BaseGeminiStrategy();
}
}
async function generate(ctl, history) {
const pluginConfig = ctl.getPluginConfig(configSchematics);
const globalConfig = ctl.getGlobalPluginConfig(globalConfigSchematics);
const model = pluginConfig.get("model");
const apiKey = globalConfig.get("apiKey");
// Resolve runtime debug flags
const envDebug = typeof process !== "undefined" && process?.env?.LMS_PLUGIN_DEBUG;
const envLogReq = typeof process !== "undefined" && process?.env?.LMS_PLUGIN_LOG_REQUESTS;
const debugChunks = envDebug === "1" ? true : envDebug === "0" ? false : pluginConfig.get("debugChunks");
const logRequests = envLogReq === "1" ? true : envLogReq === "0" ? false : pluginConfig.get("logRequests");
if (!apiKey) {
throw new Error("Google AI Studio API Key is required. Set it in the plugin's global configuration.");
}
const context = {
ctl,
history,
model,
apiKey,
globalConfig,
pluginConfig,
debugChunks,
logRequests
};
const strategy = StrategyFactory.getStrategy(model);
await strategy.generate(context);
}
// Proper prompt preprocessor using LM Studio SDK types.
// - Text/doc attachments are parsed server-side and injected into the user message text.
// - Images are not accepted by the UI for this generator (no image capability flag),
// but we support inline image references written by the user (markdown or [image: URL]).
async function preprocess(ctl, userMessage) {
try {
const originalText = userMessage.getText();
// 1) Pull non-image files attached in this message and parse them to text
const files = userMessage.consumeFiles(ctl.client, file => file.type !== "image");
let injected = "";
for (const file of files) {
const { content } = await ctl.client.files.parseDocument(file, { signal: ctl.abortSignal });
const snippet = content.length > 200_000 ? content.slice(0, 200_000) + "\n...[truncated]" : content;
injected += `\n\n[Attachment: ${file.name}]\n${snippet}\n`;
}
// 1b) Pull image files and create wrappers for them (enables UI attachment support)
const imageFiles = userMessage.consumeFiles(ctl.client, file => file.type === "image");
const wrappers = [];
for (const file of imageFiles) {
// Get file path (FileHandle.getFilePath() is async)
const filePath = await file.getFilePath();
// Create file:// URI wrapper so reconcileAttachments() can find and process them
const fileUri = `file://${filePath}`;
wrappers.push(`[[LMSTUDIO_ATTACHMENT: ${JSON.stringify({ kind: "image", url: fileUri })}]]`);
}
// 2) Inline image references (markdown or [image: URL]) → kept as-is; generator will convert
// them to image_url content parts using wrappers we add below.
const inlineImageUrls = new Set();
const mdImg = /!\[[^\]]*\]\((https?:[^\s)]+)\)/g;
let m;
while ((m = mdImg.exec(originalText)) !== null)
inlineImageUrls.add(m[1]);
const tagImg = /\[image:\s*(https?:[^\]\s]+)\s*\]/gi;
while ((m = tagImg.exec(originalText)) !== null)
inlineImageUrls.add(m[1]);
for (const url of inlineImageUrls) {
// Add inline URL wrappers to the existing wrappers array
wrappers.push(`[[LMSTUDIO_ATTACHMENT: ${JSON.stringify({ kind: "image", url })}]]`);
}
if (injected.length === 0 && wrappers.length === 0)
return userMessage;
const combined = `${originalText}${injected}${wrappers.length ? "\n\n" + wrappers.join("\n") : ""}`.trim();
userMessage.replaceText(combined);
return userMessage;
}
catch (err) {
return userMessage;
}
}
/**
* Vision Capability Primer
*
* Workaround to enable vision capabilities in LM Studio chat UI.
*
* IMPORTANT:
* Newer LM Studio `lms` CLI versions may block during model discovery/authentication.
* Awaiting `lms load` during plugin startup can therefore cause the Plugin Loader to time out.
*
* Pattern used here:
* - `checkVisionPrimerStatus()` is fast (short timeouts) and safe to await during startup
* - `loadVisionPrimerModel()` is slow and should be fire-and-forget
*/
const execAsync = util.promisify(child_process.exec);
/** Find the lms CLI path */
async function findLmsCli() {
const candidates = [
path.join(os.homedir(), ".lmstudio", "bin", "lms"),
"/usr/local/bin/lms",
"/opt/homebrew/bin/lms",
];
for (const candidate of candidates) {
try {
await execAsync(`"${candidate}" -h`);
return candidate;
}
catch {
// Not found, try next
}
}
try {
const { stdout } = await execAsync("which lms");
const lmsPath = stdout.trim();
if (lmsPath)
return lmsPath;
}
catch { }
return null;
}
/**
* Check if a model is installed locally using `lms ls --json`.
*
* NOTE: `lms ls --json` output size can be large; use a larger buffer.
* Also fall back to plain `lms ls` if JSON is truncated/invalid.
*/
async function isModelInstalled(lmsCli, modelKey, timeoutMs = 5000) {
const normalizedKey = modelKey.toLowerCase().trim();
try {
const { stdout } = await execAsync(`"${lmsCli}" ls --json`, {
timeout: timeoutMs,
maxBuffer: 10 * 1024 * 1024,
});
let models;
try {
models = JSON.parse(stdout);
}
catch (parseErr) {
try {
const { stdout: textOut, stderr: textErr } = await execAsync(`"${lmsCli}" ls`, {
timeout: Math.max(1000, timeoutMs),
maxBuffer: 10 * 1024 * 1024,
});
const haystack = `${textOut}\n${textErr}`.toLowerCase();
if (haystack.includes(normalizedKey)) {
return { installed: true, modelInfo: { modelKey } };
}
return {
installed: false,
error: `Failed to parse lms ls --json output (fallback used): ${parseErr?.message || parseErr}`,
};
}
catch (fallbackErr) {
return {
installed: false,
error: `Failed to parse lms ls --json output and fallback ls failed: ${parseErr?.message || parseErr} / ${fallbackErr?.message || String(fallbackErr)}`,
};
}
}
if (!Array.isArray(models)) {
return {
installed: false,
error: "lms ls --json returned non-array",
};
}
const found = models.find((m) => {
const key = String(m?.modelKey || "").toLowerCase().trim();
return key === normalizedKey;
});
if (found) {
return {
installed: true,
modelInfo: {
modelKey: found.modelKey,
displayName: found.displayName,
sizeBytes: found.sizeBytes,
vision: found.vision === true,
},
};
}
return { installed: false };
}
catch (e) {
return {
installed: false,
error: e?.message || String(e),
};
}
}
/**
* Check if a model with the given identifier is already loaded using `lms ps --json`.
*/
async function isModelLoaded(lmsCli, identifier, timeoutMs = 5000) {
const normalizedId = identifier.toLowerCase().trim();
try {
const { stdout } = await execAsync(`"${lmsCli}" ps --json`, {
timeout: timeoutMs,
});
let instances;
try {
instances = JSON.parse(stdout);
}
catch {
return false;
}
if (!Array.isArray(instances)) {
return false;
}
return instances.some((inst) => {
const instId = String(inst?.identifier || "").toLowerCase().trim();
return instId === normalizedId;
});
}
catch {
return false;
}
}
/**
* Quick status check for vision primer (FAST - safe to await during startup).
* Checks CLI availability, installation status, and loaded status.
* Does NOT attempt to load the model.
*/
async function checkVisionPrimerStatus(config) {
const { modelKey, identifier = "vision-capability-priming" } = config;
console.debug("[VisionPrimer] Quick status check...");
const lmsCli = await findLmsCli();
if (!lmsCli) {
console.warn("[VisionPrimer] lms CLI not found (infrastructure error, silent)");
return {
lmsCli: null,
installed: false,
alreadyLoaded: false,
needsLoad: false,
infrastructureError: true,
error: "lms CLI not found. Is LM Studio installed?",
};
}
const installResult = await isModelInstalled(lmsCli, modelKey, 5000);
if (installResult.error) {
console.warn("[VisionPrimer] Installation check failed:", installResult.error);
return {
lmsCli,
installed: false,
alreadyLoaded: false,
needsLoad: false,
infrastructureError: true,
error: `Installation check failed: ${installResult.error}`,
};
}
if (!installResult.installed) {
const userMsg = `**Vision Attachment Support:**\n\nThe vision-capability-priming model \`${modelKey}\` is not installed.\n\nThis model enables image attachments in the chat UI. To install it:\n1. Open LM Studio\n2. Search for \`${modelKey}\`\n3. Download the model\n\nWithout this model, you can still use text prompts.`;
console.warn(`[VisionPrimer] Model not installed: ${modelKey}`);
return {
lmsCli,
installed: false,
alreadyLoaded: false,
needsLoad: false,
notInstalled: true,
userFacingError: userMsg,
error: `Model '${modelKey}' is not installed locally.`,
};
}
const loaded = await isModelLoaded(lmsCli, identifier, 5000);
if (loaded) {
console.debug(`[VisionPrimer] Model already loaded with identifier: ${identifier}`);
return {
lmsCli,
installed: true,
alreadyLoaded: true,
needsLoad: false,
};
}
return {
lmsCli,
installed: true,
alreadyLoaded: false,
needsLoad: true,
};
}
/**
* Load a vision model to prime capabilities in LM Studio UI.
*/
/**
* Load the vision primer model (SLOW - should be fire-and-forget).
* Only call this after checkVisionPrimerStatus() returns needsLoad: true.
*/
async function loadVisionPrimerModel(lmsCli, config) {
const startTs = Date.now();
const { modelKey, contextLength = 4096, gpuMode = "off", ttlSeconds = 3600, identifier = "vision-capability-priming", } = config;
const gpuArg = gpuMode === "off"
? "--gpu off"
: gpuMode === "max"
? "--gpu max"
: `--gpu ${gpuMode}`;
const cmd = [
`"${lmsCli}"`,
"load",
modelKey,
`--context-length ${contextLength}`,
gpuArg,
`--ttl ${ttlSeconds}`,
`--identifier "${identifier}"`,
].join(" ");
console.debug("[VisionPrimer] Running:", cmd);
try {
const { stdout, stderr } = await execAsync(cmd, {
timeout: 120_000,
});
const output = stdout + stderr;
const loadTimeSec = Math.round(((Date.now() - startTs) / 1000) * 100) / 100;
const sizeMatch = output.match(/\(([0-9.]+ [A-Z]+)\)/i);
const size = sizeMatch ? sizeMatch[1] : undefined;
console.debug(`[VisionPrimer] ✓ Model loaded in ${loadTimeSec}s`);
return { ok: true, alreadyLoaded: false, identifier, size, loadTimeSec };
}
catch (e) {
const error = e.stderr || e.message || String(e);
console.debug("[VisionPrimer] Load command failed, checking if model is now loaded anyway...");
const nowLoaded = await isModelLoaded(lmsCli, identifier, 5000);
if (nowLoaded) {
return { ok: true, alreadyLoaded: true, identifier };
}
console.warn("[VisionPrimer] Failed:", error);
return { ok: false, loadFailed: true, error };
}
}
async function main(context) {
// Register plugin components
context.withConfigSchematics(configSchematics);
context.withGlobalConfigSchematics(globalConfigSchematics);
context.withPromptPreprocessor(preprocess);
context.withGenerator(generate);
context.withToolsProvider(toolsProvider);
// Vision Capability Primer (HYBRID: quick checks awaited, load fire-and-forget)
// Reason: `lms load` may block (auth/model discovery) and cause Plugin Loader timeouts.
const primerConfig = {
modelKey: "qwen/qwen3-vl-4b",
contextLength: 4096,
gpuMode: "off",
ttlSeconds: 3600,
identifier: "vision-capability-priming",
};
const quickCheck = await checkVisionPrimerStatus(primerConfig);
if (quickCheck.alreadyLoaded) {
console.debug("[VisionPrimer] Model already loaded, injecting lastUsedModel immediately");
await injectLastUsedModelIntoNewestChat(primerConfig.modelKey, primerConfig.identifier);
globalThis.__dtc_visionPrimerResult = { ok: true, alreadyLoaded: true };
}
else if (quickCheck.needsLoad && quickCheck.lmsCli) {
console.debug("[VisionPrimer] Model needs loading, starting fire-and-forget load");
const loadPromise = loadVisionPrimerModel(quickCheck.lmsCli, primerConfig);
globalThis.__dtc_visionPrimerPromise = loadPromise;
loadPromise
.then(async (loadResult) => {
globalThis.__dtc_visionPrimerResult = loadResult;
if (loadResult.ok) {
console.debug(`[VisionPrimer] Model loaded: ${loadResult.size} in ${loadResult.loadTimeSec}s`);
await injectLastUsedModelIntoNewestChat(primerConfig.modelKey, primerConfig.identifier);
}
else {
console.warn("[VisionPrimer] Load failed:", loadResult.error);
}
})
.catch((err) => {
console.warn("[VisionPrimer] Unexpected load error:", err?.message || err);
});
}
else if (quickCheck.userFacingError) {
console.warn("[VisionPrimer] User-facing error:", quickCheck.error);
globalThis.__dtc_visionPrimerResult = {
ok: false,
notInstalled: quickCheck.notInstalled,
userFacingError: quickCheck.userFacingError,
error: quickCheck.error,
};
}
else if (quickCheck.infrastructureError) {
console.warn("[VisionPrimer] Infrastructure error (silent):", quickCheck.error);
}
}
/**
* Inject lastUsedModel into the newest conversation.json file, but ONLY if:
* 1. We (ceveyne/gemini-compat-endpoint) are in the plugins array
* 2. There is no lastUsedModel entry yet
*
* @param modelKey - Model key for indexedModelIdentifier (e.g., "qwen/qwen3-vl-4b")
* @param identifier - Runtime identifier (must match lms load --identifier!)
*/
async function injectLastUsedModelIntoNewestChat(modelKey, identifier) {
try {
const conversationsDir = path.join(os.homedir(), ".lmstudio", "conversations");
const dirExists = await fs.promises.stat(conversationsDir).then(() => true).catch(() => false);
if (!dirExists) {
console.info("[VisionPrimer] Conversations directory not found, skipping");
return;
}
const files = await fs.promises.readdir(conversationsDir);
const conversationFiles = files.filter(f => f.endsWith(".conversation.json"));
if (conversationFiles.length === 0) {
console.info("[VisionPrimer] No conversation files found");
return;
}
// Find newest conversation by creation time
let newestFile = null;
let newestBirthtime = 0;
for (const file of conversationFiles) {
const filePath = path.join(conversationsDir, file);
try {
const stats = await fs.promises.stat(filePath);
if (stats.birthtimeMs > newestBirthtime) {
newestBirthtime = stats.birthtimeMs;
newestFile = file;
}
}
catch {
// Skip files we can't stat
}
}
if (!newestFile) {
console.info("[VisionPrimer] Could not determine newest conversation");
return;
}
const filePath = path.join(conversationsDir, newestFile);
const raw = await fs.promises.readFile(filePath, "utf8");
const conversation = JSON.parse(raw);
// Check if we're in the plugins array
const ourPluginId = "ceveyne/gemini-compat-endpoint";
const isActive = Array.isArray(conversation.plugins) && conversation.plugins.includes(ourPluginId);
if (!isActive) {
console.info(`[VisionPrimer] We're not active in newest chat (${newestFile}), skipping`);
return;
}
// Check if lastUsedModel already exists
if (conversation.lastUsedModel) {
console.info(`[VisionPrimer] lastUsedModel already exists in ${newestFile}, skipping`);
return;
}
// Inject lastUsedModel
const runtimeIdentifier = identifier ?? modelKey;
conversation.lastUsedModel = {
indexedModelIdentifier: modelKey,
identifier: runtimeIdentifier, // ← Referenz auf bereits geladenes Modell!
instanceLoadTimeConfig: { fields: [] },
instanceOperationTimeConfig: { fields: [] }
};
await fs.promises.writeFile(filePath, JSON.stringify(conversation, null, 2), "utf8");
// First touch to ensure LM Studio's file watcher picks up the change
const now = new Date();
await fs.promises.utimes(filePath, now, now);
console.info(`[VisionPrimer] ✓ Injected lastUsedModel (model=${modelKey}, id=${runtimeIdentifier}) into ${newestFile}`);
console.info(`[VisionPrimer] ✓ First touch to ${newestFile}`);
// Second touch after 2 seconds to ensure UI update
setTimeout(async () => {
try {
const now2 = new Date();
await fs.promises.utimes(filePath, now2, now2);
console.info(`[VisionPrimer] ✓ Second touch to ${newestFile}`);
}
catch (e) {
console.warn("[VisionPrimer] Failed second touch:", e.message || e);
}
}, 2000);
}
catch (e) {
console.warn("[VisionPrimer] Failed to process newest chat:", e.message || e);
}
}
exports.main = main;