dist-mcp / core / tools.js
dist-mcp / core / tools.js
"use strict";
/*
* Core tool handlers for LM Studio Plugin (transport-agnostic)
* This file extracts the logic from LM Studio Plugin tool handlers in src/index.ts
* with minimal changes to keep behavior identical.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ToolSchemas = void 0;
exports.warmupBackendAtStartup = warmupBackendAtStartup;
exports.normalizeInputBuffer = normalizeInputBuffer;
exports.handleGenerateImage = handleGenerateImage;
exports.handleCrop = handleCrop;
exports.handleMask = handleMask;
exports.handleZoomIn = handleZoomIn;
exports.handleInpaint = handleInpaint;
exports.handleOutpaint = handleOutpaint;
exports.handleRefine = handleRefine;
const path_1 = __importDefault(require("path"));
const fs_1 = __importDefault(require("fs"));
const net_1 = __importDefault(require("net"));
const url_1 = require("url");
const core_bundle_mjs_1 = require("../core-bundle.mjs");
const pngMetadata_js_1 = require("../helpers/pngMetadata.js");
const drawThingsService_js_1 = require("../services/drawThingsService.js");
const modelOverlays_js_1 = require("../services/modelOverlays.js");
// Global debug toggle
const DEBUG_MODE = true;
(0, core_bundle_mjs_1.ensureLogsDir)();
const logsDir = (0, core_bundle_mjs_1.getLogsDir)();
try {
const line = `${localTimestamp()} - paths: logsDir=${logsDir}\n`;
fs_1.default.appendFileSync(path_1.default.join(logsDir, (0, core_bundle_mjs_1.getPluginLogFilename)()), line);
}
catch { }
const logFile = path_1.default.join(logsDir, (0, core_bundle_mjs_1.getPluginLogFilename)());
// LM Studio only – no client resolver switching
function resolvePreferredLocale() {
const envPref = process.env.LOG_LOCALE;
const lc = envPref || process.env.LC_ALL || process.env.LC_TIME || process.env.LANG;
if (!lc)
return undefined;
const cleaned = String(lc).split(".")[0].replace(/_/g, "-");
return cleaned || undefined;
}
function localTimestamp() {
const opts = {
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
hour12: false,
timeZoneName: "short",
};
const loc = resolvePreferredLocale();
try {
return new Date().toLocaleString(loc, opts);
}
catch {
const d = new Date();
const day = String(d.getDate()).padStart(2, "0");
const month = String(d.getMonth() + 1).padStart(2, "0");
const year = d.getFullYear();
const hh = String(d.getHours()).padStart(2, "0");
const mm = String(d.getMinutes()).padStart(2, "0");
const ss = String(d.getSeconds()).padStart(2, "0");
return `${day}/${month}/${year} ${hh}:${mm}:${ss}`;
}
}
function log(message) {
const timestamp = localTimestamp();
const line = `${timestamp} - ${message}\n`;
try {
const dir = path_1.default.dirname(logFile);
if (!fs_1.default.existsSync(dir))
fs_1.default.mkdirSync(dir, { recursive: true });
fs_1.default.appendFileSync(logFile, line);
}
catch { }
console.log(line.trim());
}
async function logError(error) {
try {
const errorLogFile = path_1.default.join(logsDir, "error.log");
await fs_1.default.promises.mkdir(logsDir, { recursive: true }).catch(() => { });
const timestamp = localTimestamp();
const details = error instanceof Error
? `${error.message}\n${error.stack}`
: String(error);
const block = `${timestamp} - ERROR:\n${details}\n\n`;
await fs_1.default.promises.appendFile(errorLogFile, block);
if (DEBUG_MODE)
console.error(block);
}
catch { }
}
async function appendErrorRaw(raw, status) {
try {
const errorLogFile = path_1.default.join(logsDir, "error.log");
await fs_1.default.promises.mkdir(logsDir, { recursive: true }).catch(() => { });
const timestamp = localTimestamp();
const header = typeof status === "number"
? `BACKEND RAW (status ${status})`
: "BACKEND RAW";
const block = `${timestamp} - ${header}:\n${raw}\n\n`;
await fs_1.default.promises.appendFile(errorLogFile, block);
}
catch { }
}
function isoStamp() {
return new Date().toISOString().replace(/[:.]/g, "-");
}
// Compact timestamp for filenames, e.g. 20251115T232635722Z
function isoStampCompact() {
const d = new Date();
const year = d.getUTCFullYear();
const month = String(d.getUTCMonth() + 1).padStart(2, "0");
const day = String(d.getUTCDate()).padStart(2, "0");
const hours = String(d.getUTCHours()).padStart(2, "0");
const minutes = String(d.getUTCMinutes()).padStart(2, "0");
const seconds = String(d.getUTCSeconds()).padStart(2, "0");
const millis = String(d.getUTCMilliseconds()).padStart(3, "0");
return `${year}${month}${day}T${hours}${minutes}${seconds}${millis}Z`;
}
function encodeFileUrl(abs) {
return (0, url_1.pathToFileURL)(abs).toString();
}
function stripInternalToolKeys(obj) {
if (!obj || typeof obj !== "object")
return {};
const out = {};
for (const [k, v] of Object.entries(obj)) {
// Prevent user injection of internal/legacy knobs (e.g. _dt_*, _i2i_*).
if (k.startsWith("_"))
continue;
out[k] = v;
}
return out;
}
function isSupportedImageBuffer(buf) {
if (!buf || buf.length < 12)
return false;
if (buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a)
return true; // PNG
if (buf[0] === 0xff && buf[1] === 0xd8)
return true; // JPEG
if (buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50)
return true; // WEBP
return false;
}
function isPng(buf) {
return (buf &&
buf.length >= 8 &&
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a);
}
function normalizeSourceNotation(s) {
let t = String(s || "")
.trim()
.toLowerCase();
if (t === "a")
t = "a1";
if (t === "v")
t = "v1";
if (t === "p")
t = "p1";
if (t === "i")
t = "i1";
return t;
}
function parsePrefixedNotation(s) {
const t = normalizeSourceNotation(s);
const m = t.match(/^([avpi])\s*(\d+)$/);
if (!m)
return null;
const idx = Math.max(1, parseInt(m[2], 10));
const pool = m[1] === "a" ? "attachment" : m[1] === "v" ? "variant" : m[1] === "i" ? "image" : "picture";
return { pool, index: idx };
}
function parseDigitOnlyNotation(s) {
const t = String(s || "").trim();
const m = t.match(/^(\d+)$/);
if (!m)
return null;
return Math.max(1, parseInt(m[1], 10));
}
// A canvas value that isn't aN/vN/pN/iN notation is a scratchpad basename or an absolute path
// (see schemas.ts's SourceNotationOrPath for the accepted shapes) — mirrors generate-image's
// core/tools.ts inline resolution; deliberately local/non-shared, same as there. The MCP layer
// (src/mcp/index.ts) already resolves and containment-checks these against the bound scratchpad
// before this handler ever runs (made-for-bionic-core's resolveMcpSourceToken()) — by the time a
// canvas value reaches here it is either an aN/vN/iN/pN notation or an already-validated absolute
// path, so this is a plain existence check, not a security boundary.
async function resolveCanvasPathToken(rawCanvas, baseDir) {
const trimmed = rawCanvas.trim();
const candidate = path_1.default.isAbsolute(trimmed)
? trimmed
: baseDir && trimmed === path_1.default.basename(trimmed)
? path_1.default.join(baseDir, trimmed)
: null;
if (!candidate)
return null;
const exists = await fs_1.default.promises.stat(candidate).then((s) => s.isFile()).catch(() => false);
return exists ? candidate : null;
}
// Pictures from third-party sources (e.g. find_image) may only carry a `sourceUrl` pointing at an
// external reference, with no locally materialized `filename` in this scratchpad — a Draw Things
// project blob (project://...sqlite3#id, resolvable via resolveProjectUri, same as find_image's own
// tool), a path on an unmounted/external volume, or a source this codebase cannot fetch at all
// (Apple Photos, a page inside a multi-page document). Mirrors find_image's own concept: resolve
// Draw Things project references properly, and otherwise fall back to the locally cached preview
// JPEG rather than failing — a lower-resolution result beats a crash or a bare error.
async function resolvePictureBuffer(found, dir, pIndex) {
if (typeof found?.filename === "string" && found.filename) {
const abs = path_1.default.join(dir, found.filename);
if (await fs_1.default.promises.stat(abs).then((s) => s.isFile()).catch(() => false)) {
return fs_1.default.promises.readFile(abs);
}
}
const sourceUrl = typeof found?.sourceUrl === "string" ? found.sourceUrl : undefined;
if (sourceUrl && (0, core_bundle_mjs_1.isProjectUri)(sourceUrl)) {
const buf = await (0, core_bundle_mjs_1.resolveProjectUri)(sourceUrl).catch(() => null);
if (buf)
return buf;
}
else if (sourceUrl && path_1.default.isAbsolute(sourceUrl)) {
if (await fs_1.default.promises.stat(sourceUrl).then((s) => s.isFile()).catch(() => false)) {
return fs_1.default.promises.readFile(sourceUrl);
}
}
if (typeof found?.preview === "string" && found.preview) {
const previewAbs = path_1.default.join(dir, found.preview);
if (await fs_1.default.promises.stat(previewAbs).then((s) => s.isFile()).catch(() => false)) {
log(`[picture] p${pIndex}: original unavailable${sourceUrl ? ` (sourceUrl: ${sourceUrl})` : ""} — falling back to preview ${previewAbs}`);
return fs_1.default.promises.readFile(previewAbs);
}
}
const hint = sourceUrl ? ` (sourceUrl: ${sourceUrl})` : "";
throw new Error(`Picture p${pIndex} has no local file and no preview in this scratchpad${hint}. Re-run the tool that produced it so it materializes a local copy, or use a different canvas.`);
}
async function saveOriginalPng(sourceBuffer, preferredDir, preferredFileName, xmpParams) {
if (!preferredDir) {
throw new Error("No output directory resolved (LM Studio chat working directory missing).");
}
const dir = path_1.default.resolve(preferredDir);
await fs_1.default.promises.mkdir(dir, { recursive: true });
const name = preferredFileName && preferredFileName.endsWith(".png")
? preferredFileName
: preferredFileName
? `${preferredFileName}.png`
: `generated-image-${isoStampCompact()}.png`;
const abs = path_1.default.join(dir, name);
const embedMeta = xmpParams != null &&
/^(1|true|yes)$/i.test(String(process.env.EMBED_PNG_METADATA ?? "true").trim());
try {
if (!isSupportedImageBuffer(sourceBuffer)) {
const binName = name.replace(/\.png$/i, ".bin");
const binAbs = path_1.default.join(dir, binName);
await fs_1.default.promises.writeFile(binAbs, sourceBuffer);
const stat = await fs_1.default.promises.stat(binAbs);
return {
savedPath: binAbs,
fileName: binName,
size: stat.size,
mimeType: "application/octet-stream",
fileUrl: encodeFileUrl(binAbs),
};
}
let pngBuf;
if (isPng(sourceBuffer)) {
pngBuf = sourceBuffer;
}
else {
pngBuf = await (0, core_bundle_mjs_1.toPng)(sourceBuffer);
}
if (embedMeta) {
pngBuf = (0, pngMetadata_js_1.injectXmpIntoBuffer)(pngBuf, xmpParams);
}
await fs_1.default.promises.writeFile(abs, pngBuf);
}
catch {
await fs_1.default.promises.writeFile(abs, sourceBuffer);
}
const stat = await fs_1.default.promises.stat(abs);
return {
savedPath: abs,
fileName: name,
size: stat.size,
mimeType: "image/png",
fileUrl: encodeFileUrl(abs),
};
}
// Legacy TinyPreviewOptions and buildAndSavePreview removed.
// Use generatePreviewFromBuffer() from media-promotion-core/image.js instead.
let lastPreviewRef = null;
let lastOriginalRef = null;
// Per-chat image tracking (LM Studio) - stores i-value and path for proper lookup
const LAST_IMAGES_BY_LM_CHAT = {};
// PHASE 4: Sticky mode removed - mode is now effectively required when sources exist
const LAST_CONSUMED_ATTACHMENT_ID_BY_LM_CHAT = {};
/**
* Get current connection settings from process.env (set by toolsProvider)
* Defaults to hardcoded settings if env vars are not set.
*/
function getCurrentConnectionSettings() {
return (0, core_bundle_mjs_1.getEngineConnectionDefaults)({
host: process.env.DRAW_THINGS_HOST,
httpPort: process.env.DRAW_THINGS_HTTP_PORT
? parseInt(process.env.DRAW_THINGS_HTTP_PORT, 10)
: undefined,
grpcPort: process.env.DRAW_THINGS_GRPC_PORT
? parseInt(process.env.DRAW_THINGS_GRPC_PORT, 10)
: undefined,
});
}
// Backend service: Draw Things only
const drawthingsService = new drawThingsService_js_1.DrawThingsService(core_bundle_mjs_1.engineConnectionDefaults.http?.baseUrl || "http://127.0.0.1:7860", core_bundle_mjs_1.engineConnectionDefaults.sharedSecret || undefined);
let imageService = drawthingsService;
async function ensureBackendReady(opts) {
// Draw Things backend only - always proceed with connection check
// Use config-aware connection settings (reads from process.env)
const conn = getCurrentConnectionSettings();
const httpBaseUrl = conn.http?.baseUrl ||
`http://${conn.http?.host || "127.0.0.1"}:${conn.http?.port || 7860}`;
const resolvedGrpc = (conn.grpc?.target || `127.0.0.1:7859`).replace(/^grpc:\/\//i, "");
const [host, portStr] = (() => {
const lastColon = resolvedGrpc.lastIndexOf(":");
if (lastColon > -1)
return [
resolvedGrpc.slice(0, lastColon),
resolvedGrpc.slice(lastColon + 1),
];
return [resolvedGrpc, String(7859)];
})();
const portNum = parseInt(portStr, 10);
const tcpReachable = (h, p, timeoutMs) => new Promise((resolve) => {
try {
const socket = net_1.default.connect({ host: h, port: p });
const onOk = () => {
cleanup();
resolve(true);
};
const onErr = () => {
cleanup();
resolve(false);
};
const timer = setTimeout(() => onErr(), timeoutMs);
const cleanup = () => {
try {
clearTimeout(timer);
}
catch { }
try {
socket.destroy();
}
catch { }
};
socket.once("connect", onOk);
socket.once("error", onErr);
}
catch {
resolve(false);
}
});
// requireGrpc=true (edit-mode-only callers, e.g. zoom-in's Restore-Pass):
// HTTP can never serve generateImageEdit, so don't even probe/fall back to it.
const transport = opts?.requireGrpc === true ? "grpc" : (conn.transport || "auto");
const wantGrpc = transport === "grpc" || transport === "auto";
const wantHttp = transport === "http" || transport === "auto";
const grpcOk = wantGrpc ? await tcpReachable(host, portNum, 1200) : false;
// probe HTTP only when desired
const httpProbeHost = (() => {
try {
const u = new URL(httpBaseUrl);
return u.hostname || "127.0.0.1";
}
catch {
return "127.0.0.1";
}
})();
const httpProbePort = (() => {
try {
const u = new URL(httpBaseUrl);
return Number(u.port) || 7860;
}
catch {
return 7860;
}
})();
const httpOk = wantHttp
? await tcpReachable(httpProbeHost, httpProbePort, 1200)
: false;
const httpDesc = (() => {
try {
const u = new URL(httpBaseUrl);
return `${u.protocol}//${u.hostname}:${u.port || 80}`;
}
catch {
return httpBaseUrl;
}
})();
log([
"Attempting to connect to Draw Things API at:",
` grpc://${host}:${portNum} - ${grpcOk ? "OK" : "UNAVAILABLE"}`,
` ${httpDesc} - ${httpOk ? "OK" : "UNAVAILABLE"}`,
"",
"Starting service...",
"",
].join("\n"));
let usedTransport = null;
if (grpcOk) {
// Map defaults to expected gRPC envs for downstream service compatibility
try {
if (conn.grpc?.target)
process.env.DRAWTHINGS_GRPC_TARGET = conn.grpc.target;
if (conn.grpc?.service)
process.env.DRAWTHINGS_GRPC_SERVICE = conn.grpc.service;
if (conn.grpc?.compression)
process.env.DRAWTHINGS_GRPC_COMPRESSION = conn.grpc.compression;
if (conn.grpc?.acceptEncoding)
process.env.DRAWTHINGS_GRPC_ACCEPT_ENCODING = conn.grpc
.acceptEncoding;
if (conn.grpc?.protoPath)
process.env.DRAWTHINGS_GRPC_PROTO = conn.grpc.protoPath;
if (conn.sharedSecret)
process.env.DRAWTHINGS_SHARED_SECRET = conn.sharedSecret;
}
catch { }
try {
const mod = await import("../services/drawThingsGrpcService.js");
const GrpcCtor = mod?.DrawThingsGrpcService;
if (typeof GrpcCtor !== "function")
throw new Error("DrawThingsGrpcService not exported");
const grpcSvc = new GrpcCtor(`${host}:${portNum}`);
const ok = await grpcSvc.checkApiConnection();
if (ok) {
imageService = grpcSvc;
usedTransport = "grpc";
// Startup-only: log if SOLL models/LoRAs exist on the gRPC server.
// Non-blocking by design; it helps diagnose silent fallback behavior.
try {
const client = grpcSvc?.client;
if (client) {
const bn = (s) => {
try {
return path_1.default.basename(String(s || "").trim());
}
catch {
return "";
}
};
const { MODEL_PRESET_TO_CAPABILITY_KEY, selectAutoModel, checkModeSupport, } = await import("../core-bundle.mjs");
const { getModelRequiredFiles } = await import("../services/modelOverlays.js");
const { defaultParams: defaultT2I } = await import("../core-bundle.mjs");
const { defaultParamsImg2Img: defaultI2I } = await import("../core-bundle.mjs");
const { defaultParamsEdit: defaultEdit } = await import("../core-bundle.mjs");
const requiredModels = new Set();
const requiredLoras = new Set();
const optionalLoras = new Set();
// Defaults (used when model preset is "auto" or when no overlay is applied)
if (defaultT2I?.model)
requiredModels.add(bn(defaultT2I.model));
if (defaultI2I?.model)
requiredModels.add(bn(defaultI2I.model));
if (defaultEdit?.model)
requiredModels.add(bn(defaultEdit.model));
// Default LoRAs are treated as optional to avoid hard assumptions.
for (const d of [defaultT2I, defaultI2I, defaultEdit]) {
const ls = Array.isArray(d?.loras)
? d.loras
: [];
for (const l of ls) {
const f = bn(l?.file);
if (f)
optionalLoras.add(f);
}
}
const toolModes = ["text2image", "image2image", "edit", "text2video", "image2video"];
const toOverlayMode = (m) => m === "text2image"
? "txt2img"
: m === "image2image"
? "img2img"
: m === "text2video"
? "txt2vid"
: m === "image2video"
? "img2vid"
: "edit";
const presetKeys = Object.keys(MODEL_PRESET_TO_CAPABILITY_KEY || {});
// Overlay SOLL files (models + LoRAs)
for (const preset of presetKeys) {
for (const tm of toolModes) {
const supported = checkModeSupport(preset, tm);
if (!supported?.supported)
continue;
const files = getModelRequiredFiles(preset, toOverlayMode(tm));
for (const fRaw of files) {
const f = bn(fRaw);
if (!f)
continue;
if (/lora/i.test(f))
requiredLoras.add(f);
else
requiredModels.add(f);
}
}
}
// Custom Configs: Not checked at warmup (requires config access via toolsProvider).
// Hard-fail happens per-request in the gRPC backend if model/LoRA is missing.
// Explicit log of auto resolution
const autoMap = toolModes.map((m) => `${m}→${selectAutoModel(m)}`);
log(`[startup] auto preset resolution: ${autoMap.join(", ")}`);
const allToCheck = [
...Array.from(requiredModels),
...Array.from(requiredLoras),
...Array.from(optionalLoras),
];
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const ex = await (0, core_bundle_mjs_1.checkDrawThingsGrpcFilesExist)({
client,
sharedSecret: sharedSecret || undefined,
files: allToCheck,
});
if (!ex.usedFilesExist) {
log("[startup] gRPC asset preflight skipped (FilesExist RPC unavailable or failed).");
}
else {
const missingSet = new Set(ex.missing);
const missingModels = Array.from(requiredModels).filter((f) => missingSet.has(f));
const missingReqLoras = Array.from(requiredLoras).filter((f) => missingSet.has(f));
const missingOptLoras = Array.from(optionalLoras).filter((f) => missingSet.has(f));
if (missingModels.length === 0 && missingReqLoras.length === 0) {
log(`[startup] gRPC asset preflight OK: required models=${requiredModels.size}, required LoRAs=${requiredLoras.size}`);
}
else {
if (missingModels.length) {
log(`[startup] gRPC asset preflight MISSING models: ${missingModels.join(", ")}`);
}
if (missingReqLoras.length) {
log(`[startup] gRPC asset preflight MISSING required LoRAs: ${missingReqLoras.join(", ")}`);
}
}
if (missingOptLoras.length) {
log(`[startup] gRPC asset preflight (optional) missing LoRAs: ${missingOptLoras.join(", ")}`);
}
}
}
}
catch (e) {
log(`[startup] gRPC asset preflight warning: ${e?.message || String(e)}`);
}
}
else {
// fall back to HTTP if desired and reachable
if (httpOk && (transport === "auto" || transport === "http")) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
else {
console.error("Draw Things gRPC reachable but not ready; no HTTP fallback available.");
}
}
}
catch (e) {
console.error(`Draw Things gRPC init failed: ${e?.message || String(e)}.`);
// prefer HTTP fallback on init error
if (httpOk && (transport === "auto" || transport === "http")) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
}
}
else if (httpOk) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
else {
imageService = drawthingsService; // not connected yet
}
globalThis.__DT_SELECTED_TRANSPORT__ = usedTransport;
try {
const isApiConnected = await imageService.checkApiConnection();
const t = globalThis.__DT_SELECTED_TRANSPORT__;
const suffix = t === "grpc" ? " - gRPC" : t === "http" ? " - HTTP" : "";
if (isApiConnected) {
log(`Connected to Draw Things API${suffix}.`);
if (t === "grpc") {
const sec = imageService?.currentSecurity ?? globalThis.__DT_GRPC_TLS_SELECTED__ ?? "unknown";
log(`[gRPC] security: ${sec}`);
}
}
else
log(`Failed to connect to Draw Things API.`);
}
catch { }
}
/**
* Startup warmup entrypoint.
* Invoked from the Tools Provider during plugin initialization so the backend probe
* (and gRPC model/LoRA preflight logging) happens before the first tool call.
*/
async function warmupBackendAtStartup() {
await ensureBackendReady();
}
// Utility: read last audit prompt and mode for context
async function getLastAuditPromptAndMode() {
try {
const p = path_1.default.join(logsDir, "generate-image-plugin.audit.jsonl");
const txt = await fs_1.default.promises.readFile(p, "utf8").catch(() => "");
if (!txt)
return null;
const chunks = txt
.split(/\n\s*\n/g)
.map((s) => s.trim())
.filter((s) => s.length > 0);
for (let i = chunks.length - 1; i >= 0; i--) {
const s = chunks[i];
try {
const obj = JSON.parse(s);
if (obj && typeof obj === "object") {
// Read prompt from output.prompt_used (what was actually used)
const prompt = typeof obj.output?.prompt_used === "string"
? obj.output.prompt_used
: undefined;
const mode = typeof obj.mode === "string" ? obj.mode : undefined;
return { prompt, mode };
}
}
catch { }
}
return null;
}
catch {
return null;
}
}
async function getLMConversationFilePath(chatId) {
try {
const home = (0, core_bundle_mjs_1.findLMStudioHome)();
const convDir = path_1.default.join(home, "conversations");
if (!fs_1.default.existsSync(convDir))
return null;
if (chatId) {
const p = path_1.default.join(convDir, `${chatId}.conversation.json`);
return (await fs_1.default.promises
.stat(p)
.then((s) => (s.isFile() ? p : null))
.catch(() => null));
}
const entries = await fs_1.default.promises
.readdir(convDir)
.catch(() => []);
const convFiles = entries
.filter((f) => f.endsWith(".conversation.json"))
.map((f) => path_1.default.join(convDir, f));
if (convFiles.length === 0)
return null;
const withTimes = (await Promise.all(convFiles.map(async (p) => {
try {
const s = await fs_1.default.promises.stat(p);
return s.isFile() ? { p, t: s.mtimeMs } : null;
}
catch {
return null;
}
}))).filter(Boolean);
if (withTimes.length === 0)
return null;
withTimes.sort((a, b) => b.t - a.t);
return withTimes[0].p;
}
catch {
return null;
}
}
async function getLastVariantGroupFromLMConversation(chatId) {
try {
const convPath = await getLMConversationFilePath(chatId || undefined);
if (!convPath)
return null;
const text = await fs_1.default.promises.readFile(convPath, "utf8");
const byBase = new Map();
const reOrig = /file:\/\/[\S)"']+\/(images|working-directories\/\d+)\/(image-[^\/]*)-i(\d+)\.png/gi;
let m;
while ((m = reOrig.exec(text)) != null) {
try {
const basePlus = m[2];
const iNum = parseInt(m[3], 10);
const urlStr = m[0].match(/file:\/\/[^^\s)"']+/i)?.[0];
if (!urlStr)
continue;
let absPath = null;
try {
absPath = (0, url_1.fileURLToPath)(urlStr);
}
catch {
absPath = null;
}
if (!absPath)
continue;
const g = byBase.get(basePlus) || {
lastIndex: m.index,
variants: new Set(),
originals: new Map(),
};
g.lastIndex = Math.max(g.lastIndex, m.index);
g.variants.add(iNum);
g.originals.set(iNum, absPath);
byBase.set(basePlus, g);
}
catch { }
}
const rePrev = /file:\/\/[\S)"']+\/(images\/previews|working-directories\/\d+)\/(preview-image-[^\/]*)-i(\d+)\.(jpg|jpeg|webp)/gi;
while ((m = rePrev.exec(text)) != null) {
try {
const nameWithPreview = m[2];
const iNum = parseInt(m[3], 10);
const urlStr = m[0]
.replace(/\/images\/previews\//i, "/images/")
.replace(/\/working-directories\/(\d+)\//i, "/working-directories/$1/")
.replace(/preview-/, "")
.replace(/\.(jpg|jpeg|webp)$/i, ".png");
let absPath = null;
try {
const urlOnly = urlStr.match(/file:\/\/[^^\s)"']+/i)?.[0];
absPath = urlOnly ? (0, url_1.fileURLToPath)(urlOnly) : null;
}
catch {
absPath = null;
}
if (!absPath)
continue;
const originalBase = nameWithPreview.replace(/^preview-/, "");
const g = byBase.get(originalBase) || {
lastIndex: m.index,
variants: new Set(),
originals: new Map(),
};
g.lastIndex = Math.max(g.lastIndex, m.index);
g.variants.add(iNum);
g.originals.set(iNum, absPath);
byBase.set(originalBase, g);
}
catch { }
}
if (byBase.size === 0)
return null;
const best = Array.from(byBase.entries())
.map(([base, g]) => ({ base, g }))
.sort((a, b) => b.g.lastIndex - a.g.lastIndex)[0];
if (!best)
return null;
const out = [];
for (let v = 1; v <= 3; v++) {
const p = best.g.originals.get(v);
if (!p)
continue;
const exists = await fs_1.default.promises
.stat(p)
.then((s) => s.isFile())
.catch(() => false);
if (exists)
out.push(p);
}
return out.length > 0 ? out : null;
}
catch {
return null;
}
}
// Decide which event is the most recent in the conversation file.
// Returns:
// - "image" when the latest reference to a generated image is i1 (original or preview)
// - "attachment" when the latest user image attachment appears after the last image
// - null when neither could be detected
async function getLastEventTypeFromLMConversation(chatId) {
try {
const convPath = await getLMConversationFilePath(chatId || undefined);
if (!convPath)
return null;
const text = await fs_1.default.promises.readFile(convPath, "utf8");
let lastVariantIdx = -1;
let m;
const reOrig = /file:\/\/[\S)"']+\/(images|working-directories\/\d+)\/(image-[^\/]*)-i(\d+)\.png/gi;
while ((m = reOrig.exec(text)) != null) {
const iNum = parseInt(m[3], 10);
if (iNum === 1)
lastVariantIdx = Math.max(lastVariantIdx, m.index);
}
const rePrev = /file:\/\/[\S)"']+\/(images\/previews|working-directories\/\d+)\/(preview-image-[^\/]*)-i(\d+)\.(jpg|jpeg|webp)/gi;
while ((m = rePrev.exec(text)) != null) {
const iNum = parseInt(m[3], 10);
if (iNum === 1)
lastVariantIdx = Math.max(lastVariantIdx, m.index);
}
let lastAttachmentIdx = -1;
const reAtt = /"(fileIdentifier|identifier)"\s*:\s*"([^"\n]+\.(png|jpg|jpeg|webp|gif|bmp|tif|tiff|heic))"/gi;
while ((m = reAtt.exec(text)) != null) {
lastAttachmentIdx = Math.max(lastAttachmentIdx, m.index);
}
if (lastVariantIdx < 0 && lastAttachmentIdx < 0)
return null;
return lastVariantIdx > lastAttachmentIdx ? "image" : "attachment";
}
catch {
return null;
}
}
/**
* Unified input normalization for all image sources (Attachments, Variants, Pictures).
* Applies in order:
* 1. Adopt target aspect ratio (if user specified output dimensions)
* 2. Sum constraint (w + h <= targetSum)
* 3. Alignment (multiples of 64)
* 4. Minimum dimension (256px)
* 5. Convert to PNG
*/
async function normalizeInputBuffer(buf, opts) {
const prefix = opts?.logPrefix || "[normalize]";
const size = await (0, core_bundle_mjs_1.getSize)(buf);
let w = size.width || 0;
let h = size.height || 0;
const align = core_bundle_mjs_1.drawthingsLimits.align;
const minDim = core_bundle_mjs_1.drawthingsLimits.minDim;
const targetSum = opts?.targetSumOverride ?? core_bundle_mjs_1.drawthingsLimits.targetSum;
const origW = w;
const origH = h;
const origFmt = isPng(buf) ? "png" : undefined;
const origBytes = buf.byteLength;
let reason = "unchanged";
// 1. Adopt target aspect ratio when both requested dims are given.
// The adjusted image must match the OUTPUT format (e.g. landscape)
// rather than preserving the source image's aspect ratio.
// resizeCoverToPng() then uniformly scales + centre-crops the original
// to fill these dimensions without distortion.
// Subsequent steps (sum clamp, 64-alignment, minDim) refine the size.
const hasReqW = typeof opts?.requestedRawW === "number" &&
Number.isFinite(opts.requestedRawW) &&
opts.requestedRawW > 0;
const hasReqH = typeof opts?.requestedRawH === "number" &&
Number.isFinite(opts.requestedRawH) &&
opts.requestedRawH > 0;
if (hasReqW && hasReqH) {
w = opts.requestedRawW;
h = opts.requestedRawH;
reason = "clamped_to_requested_raw";
log(`${prefix} adopting target dimensions: ${origW}x${origH} → ${w}x${h}`);
}
// 2. Sum constraint: w + h <= targetSum (skipped for zoom profile)
const currentSum = w + h;
if (!opts?.skipSumConstraint && currentSum > targetSum) {
const aspect = w / Math.max(1, h);
const rawH = targetSum / (aspect + 1);
const rawW = aspect * rawH;
// Build candidates from floor/round/ceil on each axis, plus aspect-derived pairs.
// Independent rounding (round×round) can distort the aspect ratio when both
// axes happen to align to different multiples. The candidate selection below
// picks the pair that best preserves the target aspect ratio while satisfying
// the sum constraint and 64-alignment.
const wFloor = Math.max(align, Math.floor(rawW / align) * align);
const wRound = Math.max(align, Math.round(rawW / align) * align);
const wCeil = Math.max(align, Math.ceil(rawW / align) * align);
const hFloor = Math.max(align, Math.floor(rawH / align) * align);
const hRound = Math.max(align, Math.round(rawH / align) * align);
const hCeil = Math.max(align, Math.ceil(rawH / align) * align);
const sumCandidates = [];
for (const cw of [wFloor, wRound, wCeil]) {
for (const ch of [hFloor, hRound, hCeil]) {
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
}
// Aspect-derived pairs: derive one axis from the other
for (const cw of [wFloor, wRound]) {
const ch = Math.max(align, Math.round(cw / aspect / align) * align);
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
for (const ch of [hFloor, hRound]) {
const cw = Math.max(align, Math.round(ch * aspect / align) * align);
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
if (sumCandidates.length > 0) {
sumCandidates.sort((a, b) => {
const ae = Math.abs(a.cw / Math.max(1, a.ch) - aspect);
const be = Math.abs(b.cw / Math.max(1, b.ch) - aspect);
if (ae !== be)
return ae - be;
return (Math.abs(a.cw - rawW) + Math.abs(a.ch - rawH)) -
(Math.abs(b.cw - rawW) + Math.abs(b.ch - rawH));
});
w = sumCandidates[0].cw;
h = sumCandidates[0].ch;
}
else {
// Fallback: reduce by align increments (should not happen within normal limits)
let fbW = wFloor;
let fbH = hFloor;
while (fbW + fbH > targetSum && (fbW > align || fbH > align)) {
if (fbW >= fbH)
fbW = Math.max(align, fbW - align);
else
fbH = Math.max(align, fbH - align);
}
w = fbW;
h = fbH;
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 3. Alignment: round to multiples of 64 while preserving aspect ratio.
// Independent floor on each axis would distort the aspect (e.g. 523×697 → 512×640).
// Instead, pick the (floor,round) / (round,floor) / (floor,floor) / (round,round)
// combination that best preserves the original aspect ratio.
if (w % align !== 0 || h % align !== 0) {
const aspect = w / Math.max(1, h);
const wFloor = Math.max(align, Math.floor(w / align) * align);
const wRound = Math.max(align, Math.round(w / align) * align);
const hFloor = Math.max(align, Math.floor(h / align) * align);
const hRound = Math.max(align, Math.round(h / align) * align);
const candidates = [
{ cw: wFloor, ch: hFloor },
{ cw: wFloor, ch: hRound },
{ cw: wRound, ch: hFloor },
{ cw: wRound, ch: hRound },
];
// Also try deriving one axis from the other to hit exact aspect multiples.
for (const wBase of [wFloor, wRound]) {
const hDerived = Math.max(align, Math.round(wBase / aspect / align) * align);
candidates.push({ cw: wBase, ch: hDerived });
}
for (const hBase of [hFloor, hRound]) {
const wDerived = Math.max(align, Math.round(hBase * aspect / align) * align);
candidates.push({ cw: wDerived, ch: hBase });
}
// Filter valid candidates (within targetSum, ≥ align), then pick by lowest aspect error.
const valid = candidates.filter((c) => c.cw >= align && c.ch >= align && c.cw + c.ch <= targetSum);
if (valid.length > 0) {
valid.sort((a, b) => {
const ae = Math.abs(a.cw / Math.max(1, a.ch) - aspect);
const be = Math.abs(b.cw / Math.max(1, b.ch) - aspect);
if (ae !== be)
return ae - be;
// Tie-break: prefer closer to original pixel count.
const ad = Math.abs(a.cw - w) + Math.abs(a.ch - h);
const bd = Math.abs(b.cw - w) + Math.abs(b.ch - h);
return ad - bd;
});
w = valid[0].cw;
h = valid[0].ch;
}
else {
// Hard fallback: independent floor (should not happen within normal limits).
w = Math.max(align, Math.floor(w / align) * align);
h = Math.max(align, Math.floor(h / align) * align);
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 4. Minimum dimension: upscale only if needed to satisfy minDim
if (w < minDim || h < minDim) {
const scale = minDim / Math.min(w, h);
w = Math.round(w * scale);
h = Math.round(h * scale);
// Re-align after upscale
w = Math.max(align, Math.floor(w / align) * align);
h = Math.max(align, Math.floor(h / align) * align);
// Ensure sum constraint still met after minDim upscale
while (w + h > targetSum && (w > minDim || h > minDim)) {
if (w > h)
w = Math.max(minDim, w - align);
else
h = Math.max(minDim, h - align);
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 5. Resize and/or convert to PNG
let outBuf;
if (w !== origW || h !== origH) {
outBuf = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, w, h);
if (reason === "unchanged")
reason = "normalized_to_constraints";
log(`${prefix} dimension normalization: ${origW}x${origH} → ${w}x${h} (sum=${w + h})`);
}
else {
if (isPng(buf)) {
outBuf = buf;
}
else {
outBuf = await (0, core_bundle_mjs_1.toPng)(buf);
if (reason === "unchanged")
reason = "converted_to_png";
}
}
return {
buf: outBuf,
preprocess: {
original: {
width: origW,
height: origH,
format: origFmt,
bytes: origBytes,
},
adjusted: {
width: w,
height: h,
format: "png",
bytes: outBuf.byteLength,
},
reason,
},
normalizedLongSide: w !== origW || h !== origH ? Math.max(w, h) : undefined,
};
}
const ALLOWED_GEN_INPUT_KEYS = [
"prompt",
"width",
"height",
"imageFormat",
"quality",
"variants",
"canvas",
"moodboard",
];
async function handleGenerateImage(pluginParams, onProgress, _internal) {
await ensureBackendReady().catch((e) => {
log(`[startup] ensureBackendReady failed: ${String(e)}`);
});
try {
const rawIncoming = pluginParams || {};
// zoom/upscale sources may supply width/height up to ZOOM_TOOL_MAX_DIM (2048),
// which exceeds the standard schema limit (1536). Strip them before schema
// validation and re-inject after, so the rest of the pipeline sees them.
const isZoomLikeSource = _internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale";
const rawForValidation = isZoomLikeSource
? (({ width: _w, height: _h, ...rest }) => rest)(rawIncoming)
: rawIncoming;
const parsed = core_bundle_mjs_1.GenerateToolParamsSchemaMinimalStrict.safeParse(rawForValidation);
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid generate_image parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Re-inject zoom/upscale dims after schema validation
if (isZoomLikeSource) {
if (rawIncoming.width !== undefined)
input.width = rawIncoming.width;
if (rawIncoming.height !== undefined)
input.height = rawIncoming.height;
}
// ─────────────────────────────────────────────────────────────────────────
// HARD LIMIT CHECK: Reject requests exceeding maxWidth/maxHeight immediately.
// No silent clamping – explicit error with clear guidance.
// Skipped for zoom-like sources (canvas:zoom-in / canvas:upscale): input.width/height
// represent requestedRaw (the final post-processing target), not the backend input.
// The zoom-pass pipeline handles any requestedRaw size via SeedVR2 + post-processing.
// ─────────────────────────────────────────────────────────────────────────
if (!isZoomLikeSource) {
const reqW = input.width;
const reqH = input.height;
const maxW = core_bundle_mjs_1.drawthingsLimits.maxWidth;
const maxH = core_bundle_mjs_1.drawthingsLimits.maxHeight;
if (typeof reqW === "number" && reqW > maxW) {
log(`[validation] REJECTED: width ${reqW} exceeds maxWidth ${maxW}`);
return {
content: [
{
type: "text",
text: `Invalid width: ${reqW}px exceeds maximum allowed width of ${maxW}px. Please use width ≤ ${maxW}.`,
},
],
isError: true,
};
}
if (typeof reqH === "number" && reqH > maxH) {
log(`[validation] REJECTED: height ${reqH} exceeds maxHeight ${maxH}`);
return {
content: [
{
type: "text",
text: `Invalid height: ${reqH}px exceeds maximum allowed height of ${maxH}px. Please use height ≤ ${maxH}.`,
},
],
isError: true,
};
}
}
// Preserve the user-requested mode for logging/audit.
// "edit" is a variant of image2image with different defaults and (future) multi-source support.
const requestedMode = input.mode;
const isEditMode = requestedMode === "edit";
log(`generate_image input: ${JSON.stringify(input)}`);
try {
void (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
}
catch { }
// Validate model/mode compatibility early
const modelPreset = input.model || "auto";
const modeForValidation = requestedMode || "text2image"; // default mode if not specified
// Import capability check and custom configs (dynamic to avoid circular deps at module load)
const { checkModeSupport, checkModeSupportWithCustom, selectAutoModel, detectImageModelCapabilities, getCapabilityKeyForPreset, } = await import("../core-bundle.mjs");
const { getAvailableCustomCombinations, getCustomPreset } = await import("../services/customConfigsLoader.js");
// Use extended check that includes Custom Configs info
const modeCheck = checkModeSupportWithCustom(modelPreset, modeForValidation, getAvailableCustomCombinations);
if (!modeCheck.supported) {
log(`[validation] mode/model incompatible: ${modeCheck.reason}`);
return {
content: [
{
type: "text",
text: modeCheck.reason,
},
],
isError: true,
};
}
// NOTE: "model=auto" means: do not apply an overlay; backend uses mode-specific defaults.
// We still compute an "effectiveModelPreset" for capability logic (e.g. edit-mode limits),
// but logging should reflect the engine model that will actually be used.
const effectiveModelPreset = modelPreset === "auto" ? selectAutoModel(modeForValidation) : modelPreset;
// Resolve actual .ckpt filename for logging
const modeForFilename = modeForValidation === "edit"
? "edit"
: modeForValidation === "image2image"
? "img2img"
: modeForValidation === "text2video"
? "txt2vid"
: modeForValidation === "image2video"
? "img2vid"
: "txt2img";
// If the user did not pick a model (or explicitly picked "auto"), the backend will use
// the per-mode defaultParams*.model value (no overlay). Log that to avoid confusion.
let engineDefaultModel = null;
if (modelPreset === "auto") {
try {
if (modeForFilename === "txt2img") {
const { defaultParams } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParams?.model === "string"
? defaultParams.model
: null;
}
else if (modeForFilename === "img2img") {
const { defaultParamsImg2Img } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParamsImg2Img?.model === "string"
? defaultParamsImg2Img.model
: null;
}
else if (modeForFilename === "txt2vid") {
engineDefaultModel = null;
}
else if (modeForFilename === "img2vid") {
engineDefaultModel = null;
}
else {
const { defaultParamsEdit } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParamsEdit?.model === "string"
? defaultParamsEdit.model
: null;
}
}
catch { }
}
const customPresetModel = (() => {
if (!(0, modelOverlays_js_1.getModelFilename)(effectiveModelPreset, modeForFilename)) {
const raw = getCustomPreset(`${modeForValidation}.${effectiveModelPreset}`)?.params?.model;
if (typeof raw === "string" && raw.trim())
return path_1.default.basename(raw.trim());
}
return null;
})();
const effectiveModelFilename = (engineDefaultModel ? path_1.default.basename(engineDefaultModel) : null) ||
(0, modelOverlays_js_1.getModelFilename)(effectiveModelPreset, modeForFilename) ||
customPresetModel ||
effectiveModelPreset;
log(`[validation] model=${modelPreset} → engineModel=${effectiveModelFilename} mode=${modeForValidation}`);
// Draw Things backend only
const svc = imageService;
const resolvedName = "drawthings";
const requestedVariants = input.variants;
const usedVariants = typeof requestedVariants === "number"
? Math.max(1, Math.min(4, Math.round(requestedVariants)))
: 1;
if (typeof requestedVariants === "number" &&
requestedVariants !== usedVariants) {
log(`variants: requested=${requestedVariants} used=${usedVariants}`);
}
log(`generate_image: using backend='${resolvedName}'`);
const mode = input.mode ||
"text2image";
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas !== undefined && requestedMode === undefined) {
return {
content: [
{
type: "text",
text: "A reference image (`canvas`) was provided but `mode` was not specified. Please re-run and set `mode` to one of: `image2image` (use the reference as a base), `edit` (edit / inpaint the reference), or `image2video` (animate the reference image).",
},
],
isError: true,
};
}
const rawMoodboard = Array.isArray(input.moodboard)
? input.moodboard
: undefined;
const moodboardNotations = (rawMoodboard || [])
.filter((x) => typeof x === "string")
.map((x) => String(x));
let result;
let effectiveMode = "text2image";
let sourceTag = null;
let sourceVariantUsed = undefined;
let sourceKind = undefined;
let sourceOriginAbs = undefined;
let sourceOriginalName = undefined; // Real original filename (e.g., "Katze.png")
let hasFreshAttachment = false;
let isAttachmentSource = false;
// Track reference metadata for summary (used in edit mode multi-reference)
let usedReferenceMeta = [];
// Track per-reference preprocessing (normalization) metadata for audit
let usedReferencePreprocess = [];
let currentLmChatId = null;
let currentLmWorkingDir = null;
let lmResolverConfidence = undefined;
let lmResolverReason = undefined;
let stickyScope = "none";
let sourceFileName = undefined;
let lmCrosscheckInfo = undefined;
let sourcePreprocess;
let normalizedToLongSide = undefined;
// requestedRaw: what the user asked for (or the original source size when user omitted).
// requestedEffective: the internally used aligned/clamped size (multiples of 64).
// Backend may still run at a different internal size (e.g. i2i normalization); we track that separately.
let requestedRawW = undefined;
let requestedRawH = undefined;
let requestedEffectiveW = undefined;
let requestedEffectiveH = undefined;
let _dtNeedsUpscaler = false; // propagated from i2i block; read in post-processing
let zoomPassRan = false; // set to true when zoom-pass pipeline completes
let pendingAudit2 = null;
let auditMaskBuf = undefined; // post-resize mask PNG; read by audit block
const auditRequestId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
if (resolvedName === "drawthings") {
let srcBuf;
// NOTE: chatWdForContext is computed dynamically to ensure it uses
// the latest currentLmChatId/currentLmWorkingDir after resolution.
const getChatWdForContext = () => currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : null);
// V2: Returns all images with stable i-index for proper lookup
const getAllImagesForContext = async () => {
const chatWdForContext = getChatWdForContext();
// Primary: chat working directory state file (contains ALL images, not just latest generation)
try {
if (chatWdForContext) {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForContext);
if (st && Array.isArray(st.images) && st.images.length > 0) {
const images = [...st.images]
.filter((v) => v && typeof v.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0));
const result = images.map((v) => ({
i: v.i || 1,
path: path_1.default.join(chatWdForContext, v.filename),
}));
if (result.length > 0)
return result;
}
}
}
catch { }
// Fallback: in-process memory (only contains latest generation, not all images)
// NOTE: This is less reliable than state file but useful before state is written
try {
if (currentLmChatId) {
const mem = LAST_IMAGES_BY_LM_CHAT[currentLmChatId];
if (Array.isArray(mem) && mem.length > 0)
return mem;
}
}
catch { }
// Fallback: legacy heuristic from SSOT conversation file (returns paths only, assume i=1..n)
try {
const group = await getLastVariantGroupFromLMConversation(currentLmChatId || undefined);
if (Array.isArray(group) && group.length > 0) {
return group.map((p, i) => ({ i: i + 1, path: p }));
}
}
catch { }
return null;
};
// Try to resolve current LM Studio chat for scoping
if (_internal?.chatId) {
// A delegating caller (zoom-in/inpaint/outpaint/refine) already resolved its own chat
// context before calling us — reuse it verbatim instead of re-deriving it here. Re-deriving
// would redo a TTL-gated (60s, see activeChatContext.ts) lookup that can expire mid-pipeline
// for slow, multi-stage renders (Restore-Pass + SeedVR2, model loads, ...) and silently fall
// back to the wrong working directory (~/.lmstudio/working-directories/<chatId> instead of
// the actual bound scratchpad) even when a valid chatId is still found.
currentLmChatId = _internal.chatId;
currentLmWorkingDir = _internal.workingDir ?? null;
lmResolverConfidence = "high";
lmResolverReason = "delegated_context";
log(`[chatId] resolved via delegated_context: chatId=${currentLmChatId} workingDir=${currentLmWorkingDir || "null"}`);
}
else {
try {
// Prefer the generator-provided context (deterministic within the current turn/tool-call loop)
const active = (0, core_bundle_mjs_1.getActiveChatContext)();
if (active &&
typeof active.chatId === "string" &&
/^\d+$/.test(active.chatId)) {
currentLmChatId = active.chatId;
currentLmWorkingDir = active.workingDir;
lmResolverConfidence = "high";
lmResolverReason = `active_context${active.requestId ? `:${active.requestId}` : ""}`;
}
}
catch { }
try {
if (currentLmChatId) {
// already resolved via active context
log(`[chatId] resolved via active_context: chatId=${currentLmChatId} workingDir=${currentLmWorkingDir || "null"}`);
}
else {
// Fallback: filesystem heuristic (picks newest *.conversation.json)
console.warn("[generate_image] No deterministic chat context available – falling back to filesystem heuristic (newest conversation file). " +
"This may happen if the tool is called outside a normal Generator turn or if context TTL (60s) expired.");
const lm = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)({
requireRecentMtimeSec: 600,
});
log(`[chatId] fallback heuristic result: ok=${lm?.ok} chatId=${lm?.chatId || "null"} reason=${lm?.reason || "unknown"}`);
if (lm?.ok) {
currentLmChatId = lm.chatId;
lmResolverConfidence = lm.confidence;
lmResolverReason = lm.reason;
}
}
}
catch { }
}
// ========================================================================
// PHASE 3: Global error rules (before source resolution)
// ========================================================================
// Count available sources from state
let stateAttachmentCount = 0;
let stateImageCount = 0;
let statePictureCount = 0;
const chatWdForValidation = getChatWdForContext();
if (chatWdForValidation) {
try {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForValidation);
stateAttachmentCount = Array.isArray(st.attachments)
? st.attachments.length
: 0;
stateImageCount = Array.isArray(st.images)
? st.images.length
: 0;
statePictureCount = Array.isArray(st.pictures)
? st.pictures.length
: 0;
}
catch (e) {
log(`[phase3] failed to read state for source counts: ${String(e)}`);
}
}
const totalSourcesInState = stateAttachmentCount + stateImageCount + statePictureCount;
log(`[phase3] sources in state: attachments=${stateAttachmentCount}, images=${stateImageCount}, pictures=${statePictureCount}, total=${totalSourcesInState}`);
const loadSourceState = async () => {
const chatWd = getChatWdForContext();
let attachments = [];
let pictures = [];
let variants = [];
try {
if (chatWd) {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
attachments = Array.isArray(st?.attachments) ? st.attachments : [];
pictures = Array.isArray(st?.pictures) ? st.pictures : [];
if (Array.isArray(st?.variants)) {
variants = st.variants
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({
v: typeof v.v === "number" ? v.v : 1,
path: path_1.default.join(chatWd, v.filename),
}));
}
}
}
catch (e) {
log(`[state] failed to read attachments/pictures/variants: ${String(e)}`);
}
let images = [];
try {
images = (await getAllImagesForContext()) || [];
}
catch (e) {
log(`[state] failed to enumerate images: ${String(e)}`);
}
return { chatWd, attachments, pictures, images, variants };
};
const resolveNotation = (notationRaw, ctx) => {
const pref = parsePrefixedNotation(notationRaw);
if (pref) {
const n = `${pref.pool === "attachment"
? "a"
: pref.pool === "variant"
? "v"
: pref.pool === "image"
? "i"
: "p"}${pref.index}`;
return { pool: pref.pool, index: pref.index, notation: n };
}
const digit = parseDigitOnlyNotation(notationRaw);
if (digit != null) {
const pools = [];
if (ctx.attachments.length > 0)
pools.push("attachment");
if (ctx.variants.length > 0)
pools.push("variant");
if (ctx.images.length > 0)
pools.push("image");
if (ctx.pictures.length > 0)
pools.push("picture");
if (pools.length === 0) {
throw new Error("No sources available");
}
if (pools.length > 1) {
const abbrev = pools
.map((p) => p === "attachment" ? "a" : p === "variant" ? "v" : p === "image" ? "i" : "p")
.join("/");
throw new Error(`Ambiguous: use prefix (${abbrev})`);
}
const only = pools[0];
const n = `${only === "attachment" ? "a" : only === "variant" ? "v" : only === "image" ? "i" : "p"}${digit}`;
return { pool: only, index: digit, notation: n };
}
throw new Error(`Invalid source notation: ${String(notationRaw || "").trim()}`);
};
const { chatWd, attachments, pictures, images, variants } = await loadSourceState();
const totalSources = attachments.length + variants.length + images.length + pictures.length;
const autoSelectSingleSource = () => {
if (attachments.length === 1 &&
variants.length === 0 &&
images.length === 0 &&
pictures.length === 0) {
const a = typeof attachments[0]?.a === "number" ? attachments[0].a : 1;
return { pool: "attachment", index: a, notation: `a${a}` };
}
if (variants.length === 1 &&
attachments.length === 0 &&
images.length === 0 &&
pictures.length === 0) {
const v = typeof variants[0]?.v === "number" ? variants[0].v : 1;
return { pool: "variant", index: v, notation: `v${v}` };
}
if (images.length === 1 &&
attachments.length === 0 &&
variants.length === 0 &&
pictures.length === 0) {
const i = typeof images[0]?.i === "number" ? images[0].i : 1;
return { pool: "image", index: i, notation: `i${i}` };
}
if (pictures.length === 1 &&
attachments.length === 0 &&
variants.length === 0 &&
images.length === 0) {
const p = typeof pictures[0]?.p === "number" ? pictures[0].p : 1;
return { pool: "picture", index: p, notation: `p${p}` };
}
throw new Error("Ambiguous source – specify canvas explicitly");
};
let resolvedCanvas = null;
if (mode === "text2image" || mode === "text2video") {
if (rawCanvas) {
log(`[info] canvas ignored for mode='${mode}': ${rawCanvas}`);
}
}
else {
if (rawCanvas) {
resolvedCanvas = resolveNotation(rawCanvas, {
attachments,
variants,
pictures,
images,
});
}
else {
if (totalSources === 0) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
if (totalSources === 1) {
resolvedCanvas = autoSelectSingleSource();
log(`[canvas] auto-resolved to ${resolvedCanvas.notation}`);
}
else {
return {
content: [
{
type: "text",
text: "Ambiguous source – specify canvas explicitly (e.g., canvas='a1' or canvas='v1' or canvas='p1').",
},
],
isError: true,
};
}
}
}
const loadBufferForSel = async (sel) => {
if (sel.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: sel.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${sel.index} not found.`);
}
return {
buf: lm.buffer,
originPath: typeof lm.originalPath === "string"
? lm.originalPath
: undefined,
originalName: typeof lm.originalName === "string"
? lm.originalName
: undefined,
};
}
if (sel.pool === "variant") {
const found = variants.find((v) => v.v === sel.index);
if (!found) {
const available = variants.map((v) => `v${v.v}`).join(", ") || "(none)";
throw new Error(`Variant v${sel.index} not found. Available: ${available}`);
}
return {
buf: await fs_1.default.promises.readFile(found.path),
originPath: found.path,
};
}
if (sel.pool === "image") {
const found = images.find((img) => img.i === sel.index);
if (!found) {
const available = images.map((img) => `i${img.i}`).join(", ") || "(none)";
throw new Error(`Image i${sel.index} not found. Available: ${available}`);
}
return {
buf: await fs_1.default.promises.readFile(found.path),
originPath: found.path,
};
}
// picture
const found = pictures.find((p) => typeof p?.p === "number" && p.p === sel.index);
if (!found) {
const available = pictures
.map((p) => (typeof p?.p === "number" ? `p${p.p}` : null))
.filter(Boolean)
.join(", ") || "(none)";
throw new Error(`Picture p${sel.index} not found. Available: ${available}`);
}
if (!chatWd)
throw new Error("No working directory resolved for pictures.");
const abs = path_1.default.join(chatWd, String(found.filename || ""));
const exists = await fs_1.default.promises
.stat(abs)
.then((s) => s.isFile())
.catch(() => false);
if (!exists) {
throw new Error(`Picture file missing: ${abs}`);
}
return { buf: await fs_1.default.promises.readFile(abs), originPath: abs };
};
// Resolve canvas buffer for image2image/edit/image2video
if (mode === "image2image" || mode === "edit" || mode === "image2video") {
if (!resolvedCanvas) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
// Set unconditionally, BEFORE the fallible lookup below: effectiveMode drives which
// top-level render branch runs (image2image/edit vs. plain text2image). It must never
// depend on whether the redundant loadBufferForSel() re-lookup below succeeds — a caller
// with a presuppliedSourceBuf (zoom-in/inpaint/outpaint/refine) must stay on the
// image2image/edit render path even if that lookup fails, or its already-loaded real
// source buffer gets silently ignored in favor of a pure-prompt (hallucinated) render.
effectiveMode = mode;
try {
const loaded = await loadBufferForSel(resolvedCanvas);
srcBuf = loaded.buf;
sourceTag = `canvas:${resolvedCanvas.notation}`;
sourceKind = resolvedCanvas.pool;
sourceOriginAbs = loaded.originPath;
sourceOriginalName = loaded.originalName;
sourceFileName = loaded.originPath
? path_1.default.basename(loaded.originPath)
: undefined;
if (resolvedCanvas.pool === "image") {
sourceVariantUsed = resolvedCanvas.index;
}
// Persist last canvas selection so the orchestrator can do smarter vision promotion.
try {
if (chatWd) {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
st.lastCanvasNotation = resolvedCanvas.notation;
st.lastCanvasAt = localTimestamp();
await (0, core_bundle_mjs_1.writeStateAtomic)(chatWd, st);
}
}
catch (e) {
log(`[state] failed to persist lastCanvasNotation: ${String(e)}`);
}
}
catch (e) {
// Callers with a presupplied buffer (zoom-in/inpaint/outpaint/refine) already loaded
// the source themselves — this lookup here only re-derives audit metadata (sourceKind/
// sourceFileName/...) from a SEPARATE, independently-resolved chatWd/state read, and its
// failure must not discard an already-valid buffer the caller is about to supply anyway.
if (!_internal?.presuppliedSourceBuf) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
log(`[canvas] resolution failed but a presupplied buffer is available — continuing without audit metadata: ${String(e?.message || e)}`);
}
}
// _internal override: pre-cropped buffer bypasses canvas loading
if (_internal?.presuppliedSourceBuf) {
srcBuf = _internal.presuppliedSourceBuf;
if (_internal.sourceTag)
sourceTag = _internal.sourceTag;
}
// Audit source override: lets callers (e.g. handleZoomIn) report the
// original pre-crop source instead of the presupplied post-processed buffer.
if (_internal?.auditSourceOverride) {
const aso = _internal.auditSourceOverride;
if (aso.sourceKind !== undefined)
sourceKind = aso.sourceKind;
if (aso.sourceFileName !== undefined)
sourceFileName = aso.sourceFileName;
if (aso.sourceOriginalName !== undefined)
sourceOriginalName = aso.sourceOriginalName;
if (aso.sourceOriginAbs !== undefined)
sourceOriginAbs = aso.sourceOriginAbs;
}
// Track mask separately so it can be scaled to match adjusted canvas dims
let effectiveMaskBuf = _internal?.maskBuf;
// Resolve moodboard selections (edit mode, or image2image via gRPC)
// Note: HTTP does not support moodboard for image2image; only gRPC does.
const resolvedMoodboard = [];
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
const moodboardAllowedForI2I = selectedTransport === "grpc";
if ((mode === "edit" || (mode === "image2image" && moodboardAllowedForI2I)) &&
moodboardNotations.length > 0) {
const seen = new Set();
if (resolvedCanvas) {
seen.add(`${resolvedCanvas.pool}:${resolvedCanvas.index}`);
}
for (const nRaw of moodboardNotations) {
const sel = resolveNotation(nRaw, {
attachments,
variants,
pictures,
images,
});
const key = `${sel.pool}:${sel.index}`;
if (seen.has(key))
continue;
seen.add(key);
resolvedMoodboard.push(sel);
}
log(`[${mode}] moodboard resolved: ${resolvedMoodboard.map((s) => s.notation).join(", ") || "(none)"}`);
}
else if (mode === "image2image" && moodboardNotations.length > 0 && !moodboardAllowedForI2I) {
// Warn user that moodboard is ignored for image2image via HTTP
log(`[image2image] WARNING: moodboard ignored - requires gRPC transport. Using single canvas only.`);
}
// Stash for the edit/image2image multi-ref block below
const resolvedCanvasSel = resolvedCanvas;
const resolvedMoodboardSel = resolvedMoodboard;
// Multi-reference mode: edit always, image2image only via gRPC
const isMultiReference = (mode === "edit" || (mode === "image2image" && moodboardAllowedForI2I)) &&
resolvedMoodboardSel.length > 0;
if (mode === "text2image" || mode === "text2video") {
effectiveMode = mode;
log(`effective mode: ${effectiveMode}`);
// If the user provided non-aligned sizes, round to the effective (backend-safe) multiples of 64,
// but keep the raw values for audit + final postprocess resize.
let serviceInputForT2I = stripInternalToolKeys(input);
if (mode === "text2video") {
serviceInputForT2I._dt_video_mode = "txt2vid";
}
try {
if (resolvedName === "drawthings") {
const rawW = input?.width;
const rawH = input?.height;
const hasW = typeof rawW === "number" && Number.isFinite(rawW);
const hasH = typeof rawH === "number" && Number.isFinite(rawH);
if (hasW)
requestedRawW = Math.max(1, Math.round(rawW));
if (hasH)
requestedRawH = Math.max(1, Math.round(rawH));
if (hasW || hasH) {
const align = core_bundle_mjs_1.drawthingsLimits.align;
const maxW = core_bundle_mjs_1.drawthingsLimits.maxWidth;
const maxH = core_bundle_mjs_1.drawthingsLimits.maxHeight;
const floorTo = (v, step) => Math.floor(v / step) * step;
const clamp = (v, min, max) => Math.min(max, Math.max(min, v));
// requestedEffective must be backend-safe: aligned + never exceed render limits.
// Use floor alignment (never rounds up beyond user's raw request).
if (hasW)
requestedEffectiveW = clamp(Math.max(align, floorTo(requestedRawW, align)), align, maxW);
if (hasH)
requestedEffectiveH = clamp(Math.max(align, floorTo(requestedRawH, align)), align, maxH);
if (typeof requestedEffectiveW === "number")
serviceInputForT2I.width = requestedEffectiveW;
if (typeof requestedEffectiveH === "number")
serviceInputForT2I.height = requestedEffectiveH;
// Deterministic contract: when raw dims are known, provide them + upscaler decision.
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
serviceInputForT2I._dt_requested_raw_w = requestedRawW;
serviceInputForT2I._dt_requested_raw_h = requestedRawH;
serviceInputForT2I._dt_needs_upscaler =
requestedRawW > maxW || requestedRawH > maxH;
}
log(`[t2i] requested: raw=${requestedRawW || "-"}x${requestedRawH || "-"} effective=${requestedEffectiveW || "-"}x${requestedEffectiveH || "-"}`);
}
}
}
catch (e) {
log(`[t2i] failed to compute effective size: ${String(e)}`);
}
result = await svc.generateImage(serviceInputForT2I, onProgress);
// Reconstruct render_target from service metadata if not already set
// (handles imageFormat/quality shorthand + custom config scenarios)
if ((typeof requestedRawW !== "number" ||
typeof requestedRawH !== "number") &&
result?.metadata?.requested_dimensions) {
const reqDims = result.metadata.requested_dimensions;
if (typeof reqDims.width === "number" &&
typeof reqDims.height === "number") {
requestedRawW = reqDims.width;
requestedRawH = reqDims.height;
requestedEffectiveW = reqDims.width;
requestedEffectiveH = reqDims.height;
log(`[t2i] reconstructed render_target from service: ${requestedRawW}x${requestedRawH}`);
}
}
}
else {
if (!srcBuf) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
// Resolve imageFormat → explicit width/height for i2i/edit/image2video
// so that normalizeInputBuffer and render_target use the correct aspect ratio.
{
const fmt = input?.imageFormat;
const hasW = typeof input?.width === "number" && Number.isFinite(input.width);
const hasH = typeof input?.height === "number" && Number.isFinite(input.height);
if (fmt && !hasW && !hasH) {
const formatDims = {
square: { w: 1024, h: 1024 },
landscape: { w: 1024, h: 768 },
portrait: { w: 768, h: 1024 },
"16:9": { w: 1024, h: 576 },
};
const dims = formatDims[fmt];
if (dims) {
input.width = dims.w;
input.height = dims.h;
log(`[i2i/edit] resolved imageFormat "${fmt}" → ${dims.w}x${dims.h}`);
}
}
}
// UNIFIED: Use normalizeInputBuffer for all i2i/edit input preprocessing
try {
if (resolvedName === "drawthings") {
const userOutW = input?.width;
const userOutH = input?.height;
const isZoomProfile = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
const isRefineProfile = !isEditMode && _internal?.sourceTag === "canvas:refine";
const normalized = await normalizeInputBuffer(srcBuf, {
requestedRawW: typeof userOutW === "number" &&
Number.isFinite(userOutW) &&
userOutW > 0
? userOutW
: undefined,
requestedRawH: typeof userOutH === "number" &&
Number.isFinite(userOutH) &&
userOutH > 0
? userOutH
: undefined,
logPrefix: "[i2i]",
targetSumOverride: isZoomProfile
? core_bundle_mjs_1.drawthingsLimits.targetSumZoom
: isRefineProfile
? core_bundle_mjs_1.drawthingsLimits.targetSumRefine
: undefined,
});
srcBuf = normalized.buf;
sourcePreprocess = normalized.preprocess;
if (typeof normalized.normalizedLongSide === "number") {
normalizedToLongSide = normalized.normalizedLongSide;
}
// Scale mask to match adjusted canvas dims
if (effectiveMaskBuf) {
const adjW = normalized.preprocess.adjusted.width;
const adjH = normalized.preprocess.adjusted.height;
const origW = normalized.preprocess.original.width;
const origH = normalized.preprocess.original.height;
if (adjW && adjH && (adjW !== origW || adjH !== origH)) {
effectiveMaskBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(effectiveMaskBuf, adjW, adjH);
}
}
auditMaskBuf = effectiveMaskBuf;
}
}
catch (e) {
const errMsg = `i2i normalization error: ${String(e)}`;
log(errMsg);
return {
content: [
{
type: "text",
text: `Image2Image setup failed. Error: ${String(e)}`,
},
],
isError: true,
};
}
log(`effective mode: ${mode === "image2video" ? "image2video" : mode} (source=${sourceTag || "unknown"}${sourceFileName ? ", file=" + sourceFileName : ""})`);
// Backend input must not accept internal knobs from user.
let serviceInputForI2I = stripInternalToolKeys(input);
// Only applies when we're actually running an i2i call.
if (mode === "image2video") {
serviceInputForI2I._dt_video_mode = "img2vid";
}
else if (isEditMode) {
if (_internal?.sourceTag === "canvas:inpaint") {
serviceInputForI2I._dt_i2i_profile = "inpaint";
}
else if (_internal?.sourceTag === "canvas:outpaint") {
serviceInputForI2I._dt_i2i_profile = "outpaint";
}
else {
serviceInputForI2I._dt_i2i_profile = "edit";
}
}
else {
serviceInputForI2I._dt_i2i_profile = (_internal?.sourceTag === "canvas:zoom-in" || _internal?.sourceTag === "canvas:upscale") ? "zoom" : _internal?.sourceTag === "canvas:refine" ? "refine" : "img2img";
}
// Derive requestedRaw + requestedEffective.
// - requestedRaw is the user request when provided; otherwise the ORIGINAL source dimensions.
// - requestedEffective is the aligned/clamped size we treat as the effective target.
// Backend internal processing size (i2i normalization) may still differ.
try {
if (resolvedName === "drawthings") {
const userW = input?.width;
const userH = input?.height;
const hasUserW = typeof userW === "number" && Number.isFinite(userW);
const hasUserH = typeof userH === "number" && Number.isFinite(userH);
{
const limits = isEditMode
? core_bundle_mjs_1.drawthingsEditLimits
: core_bundle_mjs_1.drawthingsLimits;
const align = limits.align;
const minDim = limits.minDim;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));
const roundTo = (v, step) => Math.round(v / step) * step;
const floorTo = (v, step) => Math.floor(v / step) * step;
const ceilTo = (v, step) => Math.ceil(v / step) * step;
const chooseEgalized = (origW, origH) => {
const aspect = origW / Math.max(1, origH);
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
const candidates = [];
const add = (w, h) => {
if (!Number.isFinite(w) || !Number.isFinite(h))
return;
w = Math.round(w);
h = Math.round(h);
if (w <= 0 || h <= 0)
return;
if (w % align !== 0 || h % align !== 0)
return;
if (w < minAligned || h < minAligned)
return;
if (w > maxAlignedW || h > maxAlignedH)
return;
candidates.push({ w, h });
};
// 1) Near-original multiples for W/H
const wFloor = clamp(floorTo(origW, align), align, maxAlignedW);
const wCeil = clamp(ceilTo(origW, align), align, maxAlignedW);
const hFloor = clamp(floorTo(origH, align), align, maxAlignedH);
const hCeil = clamp(ceilTo(origH, align), align, maxAlignedH);
// Try deriving H from W candidates (preserve aspect as best we can)
for (const w0 of [wFloor, wCeil, minAligned]) {
const h0 = roundTo(w0 / aspect, align);
add(w0, h0);
}
// Try deriving W from H candidates
for (const h0 of [hFloor, hCeil, minAligned]) {
const w0 = roundTo(h0 * aspect, align);
add(w0, h0);
}
// Small neighborhood search around rounded H to capture exact-aspect pairs
// (e.g., 300x200 -> 384x256 preserves 1.5 exactly).
for (const hBase of [hFloor, hCeil, minAligned]) {
for (const dh of [-2, -1, 0, 1, 2]) {
const h0 = hBase + dh * align;
const w0 = roundTo(h0 * aspect, align);
add(w0, h0);
}
}
if (candidates.length === 0) {
// Hard fallback: clamp + align independently (aspect may drift)
const w0 = clamp(roundTo(origW, align), minAligned, maxAlignedW);
const h0 = clamp(roundTo(origH, align), minAligned, maxAlignedH);
return { w: w0, h: h0 };
}
// Pick candidate minimizing aspect error, then size delta
candidates.sort((a, b) => {
const ae = Math.abs(a.w / Math.max(1, a.h) - aspect);
const be = Math.abs(b.w / Math.max(1, b.h) - aspect);
if (ae !== be)
return ae - be;
const ad = Math.abs(a.w - origW) + Math.abs(a.h - origH);
const bd = Math.abs(b.w - origW) + Math.abs(b.h - origH);
return ad - bd;
});
return candidates[0];
};
// Prefer explicit user-provided OUT size when both provided.
// Otherwise derive OUT size from ORIGINAL (pre-normalization) source.
const origW0 = sourcePreprocess?.original?.width;
const origH0 = sourcePreprocess?.original?.height;
const origAspect = typeof origW0 === "number" &&
Number.isFinite(origW0) &&
typeof origH0 === "number" &&
Number.isFinite(origH0) &&
origH0 > 0
? origW0 / origH0
: undefined;
// Fall back to current (normalized) buffer size only if original is missing.
const fallbackSz = typeof origW0 === "number" && typeof origH0 === "number"
? null
: await (0, core_bundle_mjs_1.getSize)(srcBuf);
let baseW;
let baseH;
if (hasUserW && hasUserH) {
baseW = Math.round(userW);
baseH = Math.round(userH);
}
else if (!hasUserW && !hasUserH) {
baseW =
typeof origW0 === "number" && Number.isFinite(origW0)
? origW0
: fallbackSz?.width;
baseH =
typeof origH0 === "number" && Number.isFinite(origH0)
? origH0
: fallbackSz?.height;
}
else if (hasUserW && !hasUserH) {
baseW = Math.round(userW);
if (typeof origAspect === "number") {
baseH = Math.max(1, Math.round(baseW / origAspect));
}
}
else if (!hasUserW && hasUserH) {
baseH = Math.round(userH);
if (typeof origAspect === "number") {
baseW = Math.max(1, Math.round(baseH * origAspect));
}
}
if (typeof baseW === "number" && typeof baseH === "number") {
// requestedRaw: user request when present; else original source size.
if (hasUserW)
requestedRawW = Math.max(1, Math.round(userW));
if (hasUserH)
requestedRawH = Math.max(1, Math.round(userH));
if (!hasUserW && !hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
else if (hasUserW && !hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
else if (!hasUserW && hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
// requestedEffective: if user provided BOTH dims, round each independently to align.
// Otherwise, preserve aspect as closely as possible.
if (hasUserW && hasUserH) {
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
// Zoom/upscale profile: DT backend renders at input image dims, not config dims.
// Skip maxW/maxH clamp so requestedEffective = adjusted (= requestedRaw for zoom).
const isZoomProfileEffective = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
const wEff = isZoomProfileEffective
? Math.max(minAligned, roundTo(baseW, align))
: clamp(roundTo(baseW, align), minAligned, maxAlignedW);
const hEff = isZoomProfileEffective
? Math.max(minAligned, roundTo(baseH, align))
: clamp(roundTo(baseH, align), minAligned, maxAlignedH);
requestedEffectiveW = wEff;
requestedEffectiveH = hEff;
}
else {
const eg = chooseEgalized(baseW, baseH);
requestedEffectiveW = eg.w;
requestedEffectiveH = eg.h;
}
log(`[i2i/edit] requested: raw=${requestedRawW || "-"}x${requestedRawH || "-"} effective=${requestedEffectiveW || "-"}x${requestedEffectiveH || "-"} (user provided: ${hasUserW ? "w" : "-"}${hasUserH ? "h" : "-"})`);
// CRITICAL: Pass requestedEffective dimensions to the backend!
// The backend must generate at requested_effective size, not at
// the normalized source (adjusted) size.
if (typeof requestedEffectiveW === "number" &&
typeof requestedEffectiveH === "number") {
serviceInputForI2I.width = requestedEffectiveW;
serviceInputForI2I.height = requestedEffectiveH;
log(`[i2i/edit] set serviceInputForI2I dimensions to effective: ${requestedEffectiveW}x${requestedEffectiveH}`);
}
// Deterministic contract: always provide raw dims + upscaler decision for Draw Things i2i/edit.
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
serviceInputForI2I._dt_requested_raw_w =
requestedRawW;
serviceInputForI2I._dt_requested_raw_h =
requestedRawH;
// needs_upscaler: true when requestedEffective exceeds adjusted AND the
// scale-factor (rawW/adjW) meets the method-specific threshold.
// Below the threshold, Jimp alone handles the upscale; no upscaler/zoom-pass.
const _adjW = sourcePreprocess?.adjusted?.width ?? 0;
const _adjH = sourcePreprocess?.adjusted?.height ?? 0;
const _scaleFactor = _adjW > 0 && _adjH > 0
? Math.max(requestedRawW / _adjW, requestedRawH / _adjH)
: 0;
const _minThreshold = core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass"
? core_bundle_mjs_1.drawthingsLimits.minUpscaleFactorZoomPass
: core_bundle_mjs_1.drawthingsLimits.minUpscaleFactorUpscaler;
const _exceedsAdjusted = (typeof requestedEffectiveW === "number" && requestedEffectiveW > _adjW) ||
(typeof requestedEffectiveH === "number" && requestedEffectiveH > _adjH);
const _dt_needs_upscaler = _exceedsAdjusted && _scaleFactor >= _minThreshold;
_dtNeedsUpscaler = _dt_needs_upscaler;
serviceInputForI2I._dt_needs_upscaler = _dt_needs_upscaler;
// When zoom-pass will run, Pass 1 generates at adjusted size.
// Zoom-pass handles all upscaling; edit model only needs canvas resolution.
// Exception: canvas:zoom-in / canvas:upscale always run SeedVR2 at target dims — no override.
const _isZoomToolCall = _internal?.sourceTag === "canvas:zoom-in" || _internal?.sourceTag === "canvas:upscale";
if (_dt_needs_upscaler && core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass" && _adjW > 0 && _adjH > 0 && !_isZoomToolCall) {
serviceInputForI2I.width = _adjW;
serviceInputForI2I.height = _adjH;
}
}
else {
throw new Error("Invariant failed: requested_raw dims missing for drawthings i2i/edit");
}
}
}
}
}
catch (e) {
log(`[i2i] failed to derive width/height from source: ${String(e)}`);
}
// SAFETY NET: Ensure serviceInputForI2I dimensions ALWAYS respect backend limits.
// This catches edge cases where the main calculation block was skipped or failed.
// Zoom/upscale profile is excluded: DT backend ignores config dims for i2i and
// renders at input image dimensions, which may legitimately exceed 1536.
{
const isZoomProfileSafety = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
if (!isZoomProfileSafety) {
const limits = isEditMode ? core_bundle_mjs_1.drawthingsEditLimits : core_bundle_mjs_1.drawthingsLimits;
const align = limits.align;
const minDim = limits.minDim;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));
const roundTo = (v, step) => Math.round(v / step) * step;
const inW = serviceInputForI2I.width;
const inH = serviceInputForI2I.height;
if (typeof inW === "number" && Number.isFinite(inW)) {
const sanitized = clamp(roundTo(inW, align), minAligned, maxAlignedW);
if (sanitized !== inW) {
log(`[i2i] SAFETY: sanitized width ${inW} → ${sanitized}`);
serviceInputForI2I.width = sanitized;
}
}
if (typeof inH === "number" && Number.isFinite(inH)) {
const sanitized = clamp(roundTo(inH, align), minAligned, maxAlignedH);
if (sanitized !== inH) {
log(`[i2i] SAFETY: sanitized height ${inH} → ${sanitized}`);
serviceInputForI2I.height = sanitized;
}
}
}
}
// Edit mode requires gRPC backend (HTTP does not support edit mode at all)
if (isEditMode && typeof svc.generateImageEdit !== "function") {
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
// If no backend is connected at all, prefer the generic backend error.
// Otherwise the message is misleading (it implies HTTP is active).
if (!selectedTransport) {
log(`[edit] ERROR: edit mode requested but no Draw Things backend is connected`);
return {
content: [
{
type: "text",
text: "Failed to generate image: backend error",
},
],
isError: true,
};
}
if (selectedTransport === "http") {
log(`[edit] ERROR: HTTP backend does not support edit mode (generateImageEdit not available)`);
return {
content: [
{
type: "text",
text: `Edit mode is not supported via HTTP. Edit mode requires the Draw Things gRPC backend. Use mode='image2image' instead, or switch to gRPC.`,
},
],
isError: true,
};
}
// Defensive fallback: transport says gRPC but method is missing.
log(`[edit] ERROR: gRPC transport selected but edit mode is unavailable (generateImageEdit missing)`);
return {
content: [
{ type: "text", text: "Failed to generate image: backend error" },
],
isError: true,
};
}
// Multi-reference edit/image2image mode: collect additional buffers and call generateImageEdit
if (isMultiReference && typeof svc.generateImageEdit === "function") {
log(`[${mode}] resolving multi-reference sources...`);
// ─────────────────────────────────────────────────────────────────
// PHASE 2: No auto-fill. Only explicitly selected sources are used.
// ─────────────────────────────────────────────────────────────────
// Get model capabilities for limit checking
// Use edit or image2image limits based on mode
const capKey = getCapabilityKeyForPreset(effectiveModelPreset);
const imageCaps = capKey
? detectImageModelCapabilities(capKey)
: null;
const maxRefs = mode === "edit"
? (imageCaps?.edit?.maxReferenceImages ?? 4)
: (imageCaps?.image2image?.maxReferenceImages ?? 1);
log(`[${mode}] model=${effectiveModelPreset}, maxReferenceImages=${maxRefs}`);
// Get available attachments/variants/pictures count (for existence validation)
const chatWd = getChatWdForContext();
let availableAttachmentCount = 0;
let availableVariantCount = 0;
let availablePictureCount = 0;
if (chatWd) {
try {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
availableAttachmentCount = Array.isArray(st.attachments)
? st.attachments.length
: 0;
availableVariantCount = Array.isArray(st.variants)
? st.variants.length
: 0;
availablePictureCount = Array.isArray(st.pictures)
? st.pictures.length
: 0;
log(`[${mode}] available: ${availableAttachmentCount} attachments, ${availableVariantCount} variants, ${availablePictureCount} pictures`);
}
catch (e) {
log(`[${mode}] failed to read state: ${String(e)}`);
}
}
// Use resolved canvas + moodboard (no legacy sourceAttachment/sourceVariant)
const canvasSel = resolvedCanvasSel;
const moodboardSel = resolvedMoodboardSel;
// ─────────────────────────────────────────────────────────────────
// LIMIT VALIDATION: Check total references against model capabilities
// ─────────────────────────────────────────────────────────────────
const totalRequested = (canvasSel ? 1 : 0) + (moodboardSel?.length || 0);
log(`[${mode}] total references requested: ${totalRequested}, limit: ${maxRefs}`);
if (totalRequested > maxRefs) {
// Build a detailed error message
const details = [
canvasSel ? `canvas=${canvasSel.notation}` : "no canvas",
moodboardSel && moodboardSel.length > 0
? `moodboard=[${moodboardSel.map((s) => s.notation).join(",")}]`
: "",
]
.filter((s) => s)
.join(", ");
return {
content: [
{
type: "text",
text: `Model '${effectiveModelPreset}' supports max ${maxRefs} reference images in ${mode} mode.\n` +
`Requested: ${totalRequested} (${details}).\n\n` +
`Please make an explicit selection:\n` +
`- Use 'canvas' to specify the priority image (e.g., canvas="a1" or canvas="v2" or canvas="p3")\n` +
`- Use 'moodboard' to add reference images (e.g., moodboard=["a2","v1","p4"])`,
},
],
isError: true,
};
}
// ─────────────────────────────────────────────────────────────────
const referenceBuffers = [];
const referenceMetadata = [];
const referencePreprocess = [];
// Get user-requested output dimensions for capping input size
const userOutW = input?.width;
const userOutH = input?.height;
const editRequestedRawW = typeof userOutW === "number" &&
Number.isFinite(userOutW) &&
userOutW > 0
? userOutW
: undefined;
const editRequestedRawH = typeof userOutH === "number" &&
Number.isFinite(userOutH) &&
userOutH > 0
? userOutH
: undefined;
const pushReference = async (sel, isCanvas) => {
const loaded = await loadBufferForSel(sel);
// UNIFIED: Use normalizeInputBuffer for all edit/image2image multi-ref inputs
// Only canvas adopts the requested output AR; moodboard preserves its native AR.
const normalized = await normalizeInputBuffer(loaded.buf, {
requestedRawW: isCanvas ? editRequestedRawW : undefined,
requestedRawH: isCanvas ? editRequestedRawH : undefined,
logPrefix: `[${mode}:${isCanvas ? "canvas" : "moodboard"}]`,
});
referenceBuffers.push(normalized.buf);
referenceMetadata.push({
type: sel.pool,
index: sel.index,
isCanvas,
originPath: loaded.originPath,
originalName: loaded.originalName,
});
referencePreprocess.push({
type: sel.pool,
index: sel.index,
role: isCanvas ? "canvas" : "moodboard",
originPath: loaded.originPath,
originalName: loaded.originalName,
preprocess: normalized.preprocess,
});
};
// 1. Resolve Canvas first
if (canvasSel) {
try {
await pushReference(canvasSel, true);
log(`[${mode}] canvas resolved: ${canvasSel.notation}`);
}
catch (e) {
return {
content: [
{
type: "text",
text: `Canvas ${canvasSel.notation} not found: ${String(e?.message || e)}`,
},
],
isError: true,
};
}
}
// 2. Resolve Moodboard selections
for (const sel of moodboardSel || []) {
try {
await pushReference(sel, false);
log(`[${mode}] moodboard resolved: ${sel.notation}`);
}
catch (e) {
return {
content: [
{
type: "text",
text: `Moodboard ${sel.notation} not found: ${String(e?.message || e)}`,
},
],
isError: true,
};
}
}
// If no canvas was explicitly or auto-selected but we have references, use first as canvas
if (!resolvedCanvasSel && referenceBuffers.length > 0) {
referenceMetadata[0].isCanvas = true;
log(`[${mode}] auto-selected first reference as canvas: ${referenceMetadata[0].type} ${referenceMetadata[0].index}`);
}
// Fallback: if still no references, use the already-resolved srcBuf
if (referenceBuffers.length === 0 && srcBuf) {
// UNIFIED: Use normalizeInputBuffer
const normalized = await normalizeInputBuffer(srcBuf, {
requestedRawW: editRequestedRawW,
requestedRawH: editRequestedRawH,
logPrefix: `[${mode}:fallback-canvas]`,
});
referenceBuffers.push(normalized.buf);
referenceMetadata.push({
type: sourceKind || "image",
index: sourceVariantUsed || 1,
isCanvas: true,
});
referencePreprocess.push({
type: sourceKind || "image",
index: sourceVariantUsed || 1,
role: "canvas",
preprocess: normalized.preprocess,
});
log(`[${mode}] fallback: using single source as canvas`);
}
// Copy metadata for summary (outside this block)
usedReferenceMeta = [...referenceMetadata];
usedReferencePreprocess = [...referencePreprocess];
log(`[${mode}] calling generateImageEdit with ${referenceBuffers.length} references`);
result = await svc.generateImageEdit(serviceInputForI2I, referenceBuffers, onProgress, effectiveMaskBuf);
// Update sourceTag for audit
sourceTag = `${mode}:refs=${referenceBuffers.length}`;
}
else {
// Single-reference path (original behavior)
// Populate usedReferenceMeta for consistency in summary
if (srcBuf && (sourceKind || effectiveMode === "image2image")) {
usedReferenceMeta = [
{
type: sourceKind || "image",
index: sourceVariantUsed || 1,
isCanvas: true, // Single reference is always canvas
},
];
}
result = await svc.generateImageImg2Img(serviceInputForI2I, srcBuf, onProgress, effectiveMaskBuf);
// Reconstruct render_target from service metadata if not already set
// (fallback for edge cases where core logic was bypassed)
if ((typeof requestedRawW !== "number" ||
typeof requestedRawH !== "number") &&
result?.metadata?.requested_dimensions) {
const reqDims = result.metadata.requested_dimensions;
if (typeof reqDims.width === "number" &&
typeof reqDims.height === "number") {
requestedRawW = reqDims.width;
requestedRawH = reqDims.height;
requestedEffectiveW = reqDims.width;
requestedEffectiveH = reqDims.height;
log(`[i2i/edit] reconstructed render_target from service: ${requestedRawW}x${requestedRawH}`);
}
}
}
}
}
if (result.isError || result.error) {
const statusRaw = result.status;
let statusNum = undefined;
if (typeof statusRaw === "number" && Number.isFinite(statusRaw))
statusNum = statusRaw;
else if (typeof statusRaw === "string") {
const p = parseInt(statusRaw, 10);
if (Number.isFinite(p))
statusNum = p;
}
const raw = result.errorMessage ||
result.error ||
"unknown error";
const codeText = typeof statusNum === "number" ? `status ${statusNum}` : "backend error";
await logError(new Error(`Failed to generate image: ${codeText}`));
await appendErrorRaw(typeof raw === "string" ? raw : String(raw), statusNum);
const snippet = (() => {
try {
const s = String(raw);
return s.length > 500 ? s.slice(0, 500) + "…" : s;
}
catch {
return "";
}
})();
return {
content: [
{ type: "text", text: `Failed to generate image: ${codeText}` },
...(snippet
? [{ type: "text", text: `Details: ${snippet}` }]
: []),
],
isError: true,
};
}
let buffers = [];
if (result.images &&
Array.isArray(result.images) &&
result.images.length > 0) {
for (const img of result.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
buffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (result.imageBuffer &&
Buffer.isBuffer(result.imageBuffer)) {
buffers.push(result.imageBuffer);
}
else if (result.imageData) {
const data = result.imageData;
if (Buffer.isBuffer(data))
buffers.push(data);
else if (typeof data === "string") {
const b64 = data.startsWith("data:") ? data.split(",")[1] : data;
buffers.push(Buffer.from(b64, "base64"));
}
}
else if (result.imagePath) {
try {
const abs = path_1.default.resolve(result.imagePath);
buffers.push(await fs_1.default.promises.readFile(abs));
}
catch (e) {
log(`Failed to read returned imagePath: ${result.imagePath}: ${String(e)}`);
}
}
if (buffers.length === 0)
throw new Error("No valid image data returned");
// Measure Pass-1 backend output before zoom-pass can replace buffers.
let backendReturnedW;
let backendReturnedH;
try {
if (buffers[0]) {
const meta0 = await (0, core_bundle_mjs_1.getSize)(buffers[0]);
backendReturnedW = meta0.width;
backendReturnedH = meta0.height;
}
}
catch { }
// ── ZOOM-PASS PIPELINE (Schritte 3-7) ────────────────────────────────────
if (imageService?.name === "drawthings" &&
core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass" &&
_dtNeedsUpscaler &&
(effectiveMode === "image2image" || effectiveMode === "edit") &&
_internal?.sourceTag !== "canvas:zoom-in" &&
_internal?.sourceTag !== "canvas:upscale" &&
typeof requestedRawW === "number" &&
typeof requestedRawH === "number" &&
buffers.length > 0) {
try {
const rawSum = requestedRawW + requestedRawH;
const targetSumZoom = core_bundle_mjs_1.drawthingsLimits.targetSumZoom;
const align = core_bundle_mjs_1.drawthingsLimits.align;
// Step 4: Jimp Pre-Resize — build Zoom-Pass Canvas
let zoomCanvasW;
let zoomCanvasH;
let zoomCanvasBuf;
if (rawSum <= targetSumZoom) {
zoomCanvasW = requestedRawW;
zoomCanvasH = requestedRawH;
zoomCanvasBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(buffers[0], zoomCanvasW, zoomCanvasH);
}
else {
// Canvas = AR-preserving scale to targetSumZoom, 64-aligned
// Use candidate selection (floor/round/ceil on each axis) to minimise AR error.
const aspect = requestedRawW / requestedRawH;
const hRaw = targetSumZoom / (1 + aspect);
const wRaw = targetSumZoom - hRaw;
const snaps = (v) => [
Math.max(align, Math.floor(v / align) * align),
Math.max(align, Math.round(v / align) * align),
Math.max(align, Math.ceil(v / align) * align),
];
const candidates = [];
for (const wSnap of snaps(wRaw)) {
for (const hSnap of snaps(hRaw)) {
if (wSnap + hSnap <= targetSumZoom)
candidates.push({ w: wSnap, h: hSnap, arErr: Math.abs(wSnap / hSnap - aspect) });
}
}
candidates.sort((a, b) => a.arErr - b.arErr || (b.w + b.h) - (a.w + a.h));
const best = candidates[0] ?? { w: Math.max(align, Math.floor(wRaw / align) * align), h: Math.max(align, Math.floor(hRaw / align) * align) };
zoomCanvasW = best.w;
zoomCanvasH = best.h;
const arExpected = requestedRawW / requestedRawH;
const arActual = zoomCanvasW / zoomCanvasH;
if (Math.abs(arActual - arExpected) > 0.01)
log(`[AR-CHECK] zoom-canvas: ${zoomCanvasW}x${zoomCanvasH} AR=${arActual.toFixed(4)} vs raw ${requestedRawW}x${requestedRawH} AR=${arExpected.toFixed(4)} err=${Math.abs(arActual - arExpected).toFixed(4)}`);
zoomCanvasBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(buffers[0], zoomCanvasW, zoomCanvasH);
}
// Step 5: Pass 2 — SeedVR2 call
const zoomOnProgress = onProgress
? (step, total, msg) => {
if (step === -1) {
onProgress(-1, total, msg ? `Zoom ${msg}` : "Zoom");
}
else {
const t = typeof total === "number" ? total : undefined;
if (t && t > 0) {
onProgress(-1, total, `Zoom Step ${step}/${t} (${Math.round((step / (t + 1)) * 100)}%)`);
}
else {
onProgress(-1, total, `Zoom Step ${step}...`);
}
}
}
: undefined;
// Only pass runtime overrides — the service fills defaultParamsZoom as baseDefaults itself.
const zoomParams = {
width: zoomCanvasW,
height: zoomCanvasH,
prompt: typeof input.prompt === "string" ? input.prompt : "",
_dt_i2i_profile: "zoom",
_dt_needs_upscaler: false,
_dt_requested_raw_w: zoomCanvasW,
_dt_requested_raw_h: zoomCanvasH,
};
const zoomStartMs = Date.now();
const zoomResult = await svc.generateImageImg2Img(zoomParams, zoomCanvasBuf, zoomOnProgress);
const zoomInferenceMs = Date.now() - zoomStartMs;
if (zoomResult.isError || zoomResult.error) {
throw new Error(`Zoom-Pass failed: ${zoomResult.errorMessage ?? zoomResult.error ?? "unknown"}`);
}
// Extract Pass-2 buffers (same pattern as main buffer extraction above)
let zoomBuffers = [];
if (Array.isArray(zoomResult.images) && zoomResult.images.length > 0) {
for (const img of zoomResult.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
zoomBuffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (Buffer.isBuffer(zoomResult.imageBuffer)) {
zoomBuffers.push(zoomResult.imageBuffer);
}
else if (zoomResult.imageData) {
const d = zoomResult.imageData;
if (Buffer.isBuffer(d))
zoomBuffers.push(d);
else if (typeof d === "string") {
const b64 = d.startsWith("data:") ? d.split(",")[1] : d;
zoomBuffers.push(Buffer.from(b64, "base64"));
}
}
if (zoomBuffers.length === 0)
throw new Error("Zoom-Pass returned no image data");
// Measure Pass-2 backend output
let zoomBackendW;
let zoomBackendH;
try {
const zm = await (0, core_bundle_mjs_1.getSize)(zoomBuffers[0]);
zoomBackendW = zm.width;
zoomBackendH = zm.height;
}
catch { }
const zoomMeta = zoomResult?.metadata ?? {};
// Step 6: Post-Processing Stage 2
if (rawSum <= targetSumZoom) {
buffers = zoomBuffers;
}
else {
const resized = [];
for (const zb of zoomBuffers) {
resized.push(await (0, core_bundle_mjs_1.resizeFillToPng)(zb, requestedRawW, requestedRawH));
}
buffers = resized;
}
// Step 7: Write Pass-2 audit entry
zoomPassRan = true;
try {
const audit2 = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: resolvedName, mode: "zoom", requestId: auditRequestId });
if (currentLmChatId)
audit2.setChatId(currentLmChatId);
const zoomUserReq = {};
if (input?.mode)
zoomUserReq.mode = input.mode;
if (input?.prompt)
zoomUserReq.prompt = input.prompt;
if (input?.canvas)
zoomUserReq.canvas = input.canvas;
audit2.setUserRequest(zoomUserReq);
audit2.setRenderTarget({
requested_raw: { width: requestedRawW, height: requestedRawH },
requested_effective: { width: zoomCanvasW, height: zoomCanvasH },
needs_upscaler: true,
});
audit2.setInputs({
canvas: {
original: backendReturnedW !== undefined && backendReturnedH !== undefined
? { width: backendReturnedW, height: backendReturnedH }
: { width: requestedRawW, height: requestedRawH },
adjusted: { width: zoomCanvasW, height: zoomCanvasH },
},
});
const zoomOutput = {};
if (zoomBackendW !== undefined && zoomBackendH !== undefined) {
zoomOutput.backend_returned = { width: zoomBackendW, height: zoomBackendH };
}
zoomOutput.post_processed = { width: requestedRawW, height: requestedRawH };
zoomOutput.inference_time_ms = zoomInferenceMs;
if (typeof zoomMeta.model === "string" && zoomMeta.model.trim()) {
zoomOutput.model_used = path_1.default.basename(zoomMeta.model);
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.model === "string") {
zoomOutput.model_used = core_bundle_mjs_1.defaultParamsZoom.model;
}
if (typeof zoomMeta.steps_used === "number") {
zoomOutput.steps_used = zoomMeta.steps_used;
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.steps === "number") {
zoomOutput.steps_used = core_bundle_mjs_1.defaultParamsZoom.steps;
}
if (typeof zoomMeta.sampler_used === "string" && zoomMeta.sampler_used.trim()) {
zoomOutput.sampler_used = zoomMeta.sampler_used;
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.sampler === "string") {
zoomOutput.sampler_used = core_bundle_mjs_1.defaultParamsZoom.sampler;
}
audit2.setOutput(zoomOutput);
pendingAudit2 = audit2;
}
catch (auditErr) {
log(`zoom-pass audit write failed: ${String(auditErr)}`);
}
log(`zoom-pass: canvas=${zoomCanvasW}x${zoomCanvasH} backend=${zoomBackendW ?? "?"}x${zoomBackendH ?? "?"} -> final=${requestedRawW}x${requestedRawH} (${zoomInferenceMs}ms)`);
}
catch (e) {
throw new Error(`Zoom-Pass pipeline failed: ${String(e?.message || e)}`);
}
}
// ── END ZOOM-PASS PIPELINE ────────────────────────────────────────────────
const userReqW = (() => {
const v = input?.width;
return typeof v === "number" && Number.isFinite(v)
? Math.max(1, Math.round(v))
: undefined;
})();
const userReqH = (() => {
const v = input?.height;
return typeof v === "number" && Number.isFinite(v)
? Math.max(1, Math.round(v))
: undefined;
})();
if (!zoomPassRan &&
imageService?.name === "drawthings" &&
(effectiveMode === "image2image" || effectiveMode === "edit") &&
sourcePreprocess &&
(sourcePreprocess.reason === "normalized_to_constraints" ||
sourcePreprocess.reason === "clamped_to_requested_raw") &&
sourcePreprocess.original?.width &&
sourcePreprocess.original?.height &&
sourcePreprocess.adjusted?.width &&
sourcePreprocess.adjusted?.height &&
(sourcePreprocess.original.width !== sourcePreprocess.adjusted.width ||
sourcePreprocess.original.height !== sourcePreprocess.adjusted.height)) {
try {
let targetW = typeof requestedEffectiveW === "number" &&
Number.isFinite(requestedEffectiveW)
? Math.max(1, Math.round(requestedEffectiveW))
: Math.max(1, Math.round(sourcePreprocess.original.width));
let targetH = typeof requestedEffectiveH === "number" &&
Number.isFinite(requestedEffectiveH)
? Math.max(1, Math.round(requestedEffectiveH))
: Math.max(1, Math.round(sourcePreprocess.original.height));
try {
const limits = effectiveMode === "edit" ? core_bundle_mjs_1.drawthingsEditLimits : core_bundle_mjs_1.drawthingsLimits;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const s = Math.min(maxW / targetW, maxH / targetH, 1);
if (s < 1) {
targetW = Math.max(1, Math.round(targetW * s));
targetH = Math.max(1, Math.round(targetH * s));
}
}
catch { }
const resizedBuffers = [];
for (const buf of buffers) {
const r = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, targetW, targetH);
resizedBuffers.push(r);
}
buffers = resizedBuffers;
const restoreTargetLabel = typeof requestedEffectiveW === "number" &&
typeof requestedEffectiveH === "number"
? "requested effective size"
: "original source size";
log(`postprocess: restored generated image(s) to ${restoreTargetLabel} ${targetW}x${targetH} (from normalized ${sourcePreprocess.adjusted.width}x${sourcePreprocess.adjusted.height})`);
}
catch (e) {
log(`postprocess restore-to-original-size failed: ${String(e)}`);
}
}
// Final step: if we have a raw size target (either explicit user size, or derived from the
// attached source when user omitted width/height), resize output back to that exact size.
// Only absent when user provided no size AND there is no attached source (pure defaults).
try {
const finalRawW = typeof requestedRawW === "number" && Number.isFinite(requestedRawW)
? Math.max(1, Math.round(requestedRawW))
: undefined;
const finalRawH = typeof requestedRawH === "number" && Number.isFinite(requestedRawH)
? Math.max(1, Math.round(requestedRawH))
: undefined;
if (finalRawW && finalRawH) {
const metaCur = buffers[0] ? await (0, core_bundle_mjs_1.getSize)(buffers[0]) : null;
const curW = metaCur?.width;
const curH = metaCur?.height;
if (curW !== finalRawW || curH !== finalRawH) {
const resizedFinal = [];
for (const buf of buffers) {
const r = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, finalRawW, finalRawH);
resizedFinal.push(r);
}
buffers = resizedFinal;
log(`postprocess: adjusted final generated image(s) to requested raw size ${finalRawW}x${finalRawH}`);
}
}
}
catch (e) {
log(`postprocess final-resize-to-user-request failed: ${String(e)}`);
}
let postProcessedW;
let postProcessedH;
try {
if (zoomPassRan) {
// Pass 1 post-processed = Pass 1 backend output (before zoom-pass replaced buffers).
postProcessedW = backendReturnedW;
postProcessedH = backendReturnedH;
}
else if (buffers[0]) {
const metaF = await (0, core_bundle_mjs_1.getSize)(buffers[0]);
postProcessedW = metaF.width;
postProcessedH = metaF.height;
}
}
catch { }
const saveOriginal = core_bundle_mjs_1.generateRuntimeDefaults.saveOriginal;
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw != null
? /^(1|true|yes)$/i.test(String(envPreviewRaw).trim())
: true;
log(`preview toggle: PREVIEW_IN_CHAT='${envPreviewRaw}' -> ${previewInChat}`);
const promptStr = typeof input.prompt === "string" ? input.prompt : "";
const alt = promptStr.trim()
? `Generated image: ${promptStr.slice(0, 80)}`
: "Generated image";
const savedFiles = [];
// Primary storage: write directly into the active LM Studio chat working directory.
// Fail-fast if chatId could not be resolved.
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
throw new Error("Failed to resolve LM Studio chat working directory (chatId missing).");
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Read current state to get nextImageI for FORTLAUFENDE (continuous) image numbering
// This ensures i1, i2, i3... across multiple generation runs (not resetting to i1 each time)
const currentState = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const baseImageI = Math.max(1, currentState.counters.nextImageI ?? 1);
log(`image numbering: starting at i${baseImageI} (nextImageI from state)`);
const baseStamp = isoStampCompact();
const imageRecordsForState = [];
// ── VIDEO PATH ────────────────────────────────────────────────────────
const numFramesMeta = result?.metadata?.num_frames;
// Require at least 3 buffers: [discarded-first] + [≥1 real frame] + [discarded-last].
// Without this guard, slice(1, buffers.length - 1) returns an empty array when the
// gRPC response only carried a preview/fallback buffer despite num_frames > 1.
// Mode guard: some official defaults carry wrong numFrames > 1 for non-video models.
const isVideoResult = (effectiveMode === "text2video" || effectiveMode === "image2video") &&
typeof numFramesMeta === "number" && numFramesMeta > 1 && buffers.length >= 3;
// ── BUILD XMP PARAMS (shared by video + image paths) ─────────────────
const resMeta = result?.metadata ?? {};
const xmpSourcePaths = [];
if (sourceOriginAbs)
xmpSourcePaths.push(sourceOriginAbs);
for (const r of usedReferenceMeta ?? []) {
if (r.originPath && !xmpSourcePaths.includes(r.originPath)) {
xmpSourcePaths.push(r.originPath);
}
}
const xmpLorasUsed = Array.isArray(resMeta.loras_used)
? resMeta.loras_used
: [];
const xmpParams = {
...(typeof input.prompt === "string" && input.prompt ? { prompt: input.prompt } : {}),
...(typeof resMeta.model === "string" && resMeta.model
? { model: path_1.default.basename(resMeta.model) }
: effectiveModelFilename
? { model: path_1.default.basename(effectiveModelFilename) }
: {}),
...(typeof resMeta.width === "number" ? { width: resMeta.width } : {}),
...(typeof resMeta.height === "number" ? { height: resMeta.height } : {}),
...(typeof resMeta.steps_used === "number" ? { steps: resMeta.steps_used } : {}),
...(typeof resMeta.seed === "number" ? { seed: resMeta.seed } : {}),
...(typeof resMeta.seed_mode === "string" && resMeta.seed_mode ? { seedMode: resMeta.seed_mode } : {}),
...(typeof resMeta.sampler_used === "string" && resMeta.sampler_used ? { sampler: resMeta.sampler_used } : {}),
...(typeof resMeta.guidance_scale_used === "number" ? { guidanceScale: resMeta.guidance_scale_used } : {}),
...(typeof resMeta.strength_used === "number" ? { strength: resMeta.strength_used } : {}),
...(typeof resMeta.shift_used === "number" ? { shift: resMeta.shift_used } : {}),
...(xmpLorasUsed.length > 0 ? { loras: xmpLorasUsed.map((f) => ({ file: f })) } : {}),
...(xmpSourcePaths.length > 0 ? { sources: xmpSourcePaths } : {}),
mode: effectiveMode,
...(_internal?.cropMeta ? { crop: { left: _internal.cropMeta.left, top: _internal.cropMeta.top, right: _internal.cropMeta.right, bottom: _internal.cropMeta.bottom } } : {}),
...(_internal?.cropMeta?.cropSource ? { cropSource: _internal.cropMeta.cropSource } : {}),
};
// ── END XMP PARAMS ────────────────────────────────────────────────────
let videoFrames = [];
let videoPngSaved = null;
if (isVideoResult) {
const videoFps = typeof result?.metadata?.fps === "number"
? result.metadata.fps
: 24;
// Trim: discard first frame (confirmed) and last frame (pending verification).
// Roadmap trim rule: slice(1, buffers.length - 1) == slice(1, num_frames + 1)
videoFrames = buffers.slice(1, buffers.length - 1);
const videoImageI = baseImageI;
const videoBaseName = `image-${baseStamp}-i${videoImageI}`;
const lastFrame = videoFrames[videoFrames.length - 1];
// PNG: canonical for state / VP / i2i / lastOriginalRef
videoPngSaved = await saveOriginalPng(lastFrame, primaryOutDir, `${videoBaseName}.png`, { ...xmpParams, isVideoFrame: true });
try {
log(`saved original (video last-frame): ${videoPngSaved.savedPath} (${videoPngSaved.size} bytes) [i${videoImageI}]`);
}
catch { }
imageRecordsForState.push({
filename: `${videoBaseName}.png`,
preview: `preview-${videoBaseName}.jpg`,
i: videoImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/generate_image`,
});
// MOV: goes into savedFiles so originalLinksText shows .mov. On failure, falls back to PNG.
try {
onProgress?.(-1, undefined, "Assembling video...");
const { assembleVideo } = await import("../helpers/videoAssembler.js");
const audioChunks = result?.audioBuffers;
const audioRaw = audioChunks && audioChunks.length > 0
? Buffer.concat(audioChunks)
: undefined;
const audioSampleRateRaw = (0, core_bundle_mjs_1.getAudioSampleRateForModel)(effectiveModelFilename);
if (audioSampleRateRaw === undefined) {
log(`[video] ERROR: no audioSampleRate registered for model '${effectiveModelFilename}' — falling back to 48 000 Hz`);
}
const audioSampleRate = audioSampleRateRaw ?? 48_000;
const movBuffer = await assembleVideo(videoFrames, videoFps, audioRaw, audioSampleRate);
const movFileName = `${videoBaseName}.mov`;
const movPath = path_1.default.join(primaryOutDir, movFileName);
await fs_1.default.promises.writeFile(movPath, movBuffer);
const movUrl = encodeFileUrl(movPath);
savedFiles.push({ savedPath: movPath, fileUrl: movUrl, size: movBuffer.length, fileName: movFileName });
log(`saved video: ${movPath} (${movBuffer.length} bytes) [i${videoImageI}]`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`video assembly failed (i${videoImageI}): ${msg}`);
savedFiles.push(videoPngSaved);
}
}
// ── END VIDEO PATH ────────────────────────────────────────────────────
if (!isVideoResult) {
for (let i = 0; i < buffers.length; i++) {
const buf = buffers[i];
const imageI = baseImageI + i; // Fortlaufende Nummerierung
const baseName = `image-${baseStamp}-i${imageI}`;
const s = await saveOriginalPng(buf, primaryOutDir, `${baseName}.png`, xmpParams);
try {
log(`saved original: ${s.savedPath} (${s.size} bytes) [i${imageI}]`);
}
catch { }
savedFiles.push(s);
// Track for state update later
imageRecordsForState.push({
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: imageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/generate_image`,
});
}
}
const firstSaved = savedFiles[0];
// Video: lastOriginalRef must point to PNG (not MOV) so follow-up i2i/canvas loads work.
lastOriginalRef =
isVideoResult && videoPngSaved
? { path: videoPngSaved.savedPath, url: videoPngSaved.fileUrl }
: { path: firstSaved.savedPath, url: firstSaved.fileUrl };
// Policy: generate JPEG previews only (unified for attachments + variants)
// Use PreviewSpec from VARIANT_FULL_CONFIG with central generatePreviewFromBuffer()
// VARIANT_FULL_CONFIG.preview uses maxSum: 1536 for proper sizing
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const previews = [];
if (isVideoResult) {
// Generate one JPEG preview from the last trimmed video frame
const previewFrame = videoFrames[videoFrames.length - 1];
const videoImageI = baseImageI;
const videoBaseName = `image-${baseStamp}-i${videoImageI}`;
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(previewFrame, primaryOutDir, videoPngSaved.fileName, variantPreviewSpec, { customFilename: `preview-${videoBaseName}.jpg` });
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`video preview saved (i${videoImageI}): ${previewFilePath} ${p.width}x${p.height} ${p.data.length} bytes ok=true`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`video preview build failed (i${videoImageI}): ${msg} spec=${JSON.stringify(variantPreviewSpec)}`);
}
}
if (!isVideoResult) {
for (let i = 0; i < buffers.length; i++) {
const buf = buffers[i];
const imageI = baseImageI + i; // Use fortlaufende i-Nummer for logging
try {
if (!isSupportedImageBuffer(buf)) {
try {
const magic = Buffer.from(buf.slice(0, 12) || []).toString("hex");
log(`preview skip: unsupported buffer signature (i${imageI}) magic=${magic}`);
}
catch { }
continue;
}
// Use central generatePreviewFromBuffer() with correct maxSum/maxWidth constraints
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(buf, primaryOutDir, savedFiles[i].fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`preview saved (i${imageI}): ${previewFilePath} ${p.width}x${p.height} ${p.data.length} bytes ok=true`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`preview build failed (i${imageI}): ${msg} spec=${JSON.stringify(variantPreviewSpec)}`);
}
}
}
if (previews.length === 0) {
log(`previews built: count=0 (no preview created)`);
}
else {
log(`previews built: count=${previews.length}`);
}
if (previews.length > 0) {
const firstPreview = previews[0];
lastPreviewRef = {
path: firstPreview.filePath,
url: firstPreview.fileUrl,
mimeType: firstPreview.mimeType,
width: firstPreview.width,
height: firstPreview.height,
};
// Track per-chat last images and clear pending sentinel after any generation
try {
if (currentLmChatId) {
// Store with i-values for proper lookup in getAllImagesForContext
LAST_IMAGES_BY_LM_CHAT[currentLmChatId] =
imageRecordsForState.map((ir, idx) => ({
i: ir.i,
path: savedFiles[idx].savedPath,
}));
}
}
catch { }
// Update chat_media_state.json with new images (append, rolling window in orchestrator)
// This ensures the State has the correct i-numbers and nextImageI is incremented
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`state updated: appended ${imageRecordsForState.length} images, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`state update failed (non-fatal): ${String(e.message)}`);
}
}
const explicit = {};
for (const k of ALLOWED_GEN_INPUT_KEYS) {
if (Object.prototype.hasOwnProperty.call(input, k) &&
input[k] !== undefined) {
explicit[k] = input[k];
}
}
const inferenceMs = result?.metadata?.inference_time_ms;
const meta = (result?.metadata || {});
const effWidth = typeof postProcessedW === "number"
? Math.round(postProcessedW)
: typeof meta.width === "number"
? Math.round(meta.width)
: undefined;
const effHeight = typeof postProcessedH === "number"
? Math.round(postProcessedH)
: typeof meta.height === "number"
? Math.round(meta.height)
: undefined;
const imgFmtRaw = typeof meta.image_format === "string"
? String(meta.image_format)
: undefined;
const effQuality = typeof meta.quality === "string" ? String(meta.quality) : undefined;
const effSteps = typeof meta.steps === "number" ? Math.round(meta.steps) : undefined;
const summary = {
width: effWidth,
height: effHeight,
image_format: imgFmtRaw,
quality: effQuality,
...(typeof effSteps === "number" ? { steps: effSteps } : {}),
backend: resolvedName,
mode_effective: effectiveMode,
source: sourceTag || undefined,
...(typeof sourceVariantUsed === "number"
? { source_variant_used: sourceVariantUsed }
: {}),
...(typeof normalizedToLongSide === "number"
? { normalized_to_long_side: normalizedToLongSide }
: {}),
images_generated: isVideoResult ? 1 : buffers.length,
// Reference sources used (for edit mode / img2img)
...(usedReferenceMeta && usedReferenceMeta.length > 0
? {
references_used: (() => {
const moodboardCount = usedReferenceMeta.filter((r) => !r.isCanvas).length;
const moodboardWeight = moodboardCount > 0 ? 1.0 / moodboardCount : 0;
return usedReferenceMeta.map((r) => ({
source: r.type,
index: r.index,
role: r.isCanvas ? "canvas" : "moodboard",
...(r.originPath ? { source_originAbs: r.originPath } : {}),
...(r.originalName
? { source_originalName: r.originalName }
: {}),
...(!r.isCanvas && moodboardCount > 0
? { weight: moodboardWeight }
: {}),
}));
})(),
canvas_source: (() => {
const canvas = usedReferenceMeta.find((r) => r.isCanvas);
return canvas
? `${canvas.type === "attachment" ? "a" : "v"}${canvas.index}`
: null;
})(),
}
: {}),
files: {
original: firstSaved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
...(typeof inferenceMs === "number"
? { inference_time_ms: inferenceMs }
: {}),
};
// Backfill originalName for audit wherever we have originAbs.
// Rationale: originAbs is the stable attachment identity; originalName may be missing
// in some resolver paths unless explicitly persisted in chat_media_state.json.
try {
const chatWdForAudit = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : null);
if (chatWdForAudit) {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForAudit);
const attachments = Array.isArray(st?.attachments)
? st.attachments
: [];
const originalNameByOriginAbs = new Map();
const originalNameByFilename = new Map();
for (const a of attachments) {
if (!a || typeof a !== "object")
continue;
const oa = typeof a.originAbs === "string" && a.originAbs.trim()
? String(a.originAbs)
: null;
const fn = typeof a.filename === "string" && a.filename.trim()
? String(a.filename)
: typeof a.origin === "string" && a.origin.trim()
? String(a.origin)
: null;
const on = typeof a.originalName === "string" && a.originalName.trim()
? String(a.originalName)
: null;
if (oa && on)
originalNameByOriginAbs.set(oa, on);
if (fn && on)
originalNameByFilename.set(fn, on);
}
if (sourceKind === "attachment" &&
sourceOriginAbs &&
!sourceOriginalName) {
sourceOriginalName =
originalNameByOriginAbs.get(sourceOriginAbs) ||
originalNameByFilename.get(path_1.default.basename(sourceOriginAbs));
}
if (Array.isArray(usedReferenceMeta) && usedReferenceMeta.length > 0) {
usedReferenceMeta = usedReferenceMeta.map((r) => {
if (r &&
r.type === "attachment" &&
r.originPath &&
typeof r.originPath === "string" &&
!r.originalName) {
const filled = originalNameByOriginAbs.get(r.originPath) ||
originalNameByFilename.get(path_1.default.basename(r.originPath));
return filled ? { ...r, originalName: filled } : r;
}
return r;
});
}
}
}
catch { }
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginals = savedFiles.map((s) => httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(s.fileName, httpBase, currentLmChatId || undefined)
: "");
// Build preview URLs pointing to preview-* files in chat working directory
const httpPreviews = savedFiles.map((_s, i) => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[i]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
});
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({
backend: resolvedName,
mode: effectiveMode,
requestId: auditRequestId,
});
// Metadata
if (currentLmChatId)
audit.setChatId(currentLmChatId);
// === USER REQUEST (what the user sent) ===
const userRequest = {};
if (input?.prompt)
userRequest.prompt = input.prompt;
if (input?.mode)
userRequest.mode = input.mode;
if (input?.canvas)
userRequest.canvas = input.canvas;
if (input?.moodboard)
userRequest.moodboard = input.moodboard;
if (input?.model)
userRequest.model = input.model;
if (input?.width)
userRequest.width = input.width;
if (input?.height)
userRequest.height = input.height;
if (typeof input?.seed === "number") {
userRequest.seed = input.seed;
}
if (Object.prototype.hasOwnProperty.call(input, "seed_mode")) {
userRequest.seed_mode = input.seed_mode;
}
else if (Object.prototype.hasOwnProperty.call(input, "seedMode")) {
userRequest.seed_mode = input.seedMode;
}
if (input?.imageFormat)
userRequest.imageFormat = input.imageFormat;
if (input?.quality)
userRequest.quality = input.quality;
if (input?.variants)
userRequest.variants = input.variants;
audit.setUserRequest(userRequest);
// === RENDER TARGET (Step 0+1) ===
const renderTarget = {};
if (typeof requestedRawW === "number" ||
typeof requestedRawH === "number") {
renderTarget.requested_raw = {
...(typeof requestedRawW === "number"
? { width: requestedRawW }
: {}),
...(typeof requestedRawH === "number"
? { height: requestedRawH }
: {}),
};
}
if (typeof requestedEffectiveW === "number" ||
typeof requestedEffectiveH === "number") {
renderTarget.requested_effective = {
...(typeof requestedEffectiveW === "number"
? { width: requestedEffectiveW }
: {}),
...(typeof requestedEffectiveH === "number"
? { height: requestedEffectiveH }
: {}),
};
}
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
renderTarget.needs_upscaler = _dtNeedsUpscaler;
}
if (Object.keys(renderTarget).length > 0) {
audit.setRenderTarget(renderTarget);
}
// === INPUTS (Step 2: Canvas + Moodboard) ===
const inputs = {};
// Canvas from sourcePreprocess (single-image i2i) or usedReferencePreprocess
if (sourcePreprocess && sourcePreprocess.original) {
inputs.canvas = {
notation: sourceTag || undefined,
source_type: sourceKind || undefined,
file_name: sourceFileName || undefined,
original_name: sourceOriginalName || undefined,
origin_path: sourceOriginAbs || undefined,
original: _internal?.auditSourceOverride?.originalDims ?? {
width: sourcePreprocess.original.width,
height: sourcePreprocess.original.height,
bytes: sourcePreprocess.original.bytes,
},
adjusted: sourcePreprocess.adjusted
? {
width: sourcePreprocess.adjusted.width,
height: sourcePreprocess.adjusted.height,
bytes: sourcePreprocess.adjusted.bytes,
}
: undefined,
};
}
// Multi-reference (edit mode): usedReferencePreprocess
if (usedReferencePreprocess && usedReferencePreprocess.length > 0) {
const canvasRef = usedReferencePreprocess.find((r) => r.role === "canvas");
const moodboardRefs = usedReferencePreprocess.filter((r) => r.role === "moodboard");
const moodboardCount = moodboardRefs.length;
if (canvasRef) {
const notation = `${canvasRef.type === "attachment"
? "a"
: canvasRef.type === "image"
? "i"
: "p"}${canvasRef.index}`;
inputs.canvas = {
notation,
source_type: canvasRef.type,
original_name: canvasRef.originalName || undefined,
origin_path: canvasRef.originPath || undefined,
original: canvasRef.preprocess?.original,
adjusted: canvasRef.preprocess?.adjusted,
};
}
if (moodboardRefs.length > 0) {
inputs.moodboard = moodboardRefs.map((r) => {
const notation = `${r.type === "attachment" ? "a" : r.type === "image" ? "i" : "p"}${r.index}`;
return {
notation,
source_type: r.type,
original_name: r.originalName || undefined,
origin_path: r.originPath || undefined,
original: r.preprocess?.original,
adjusted: r.preprocess?.adjusted,
weight: moodboardCount > 0 ? 1.0 / moodboardCount : undefined,
};
});
}
}
// Mask info (inpaint / outpaint) — uses auditMaskBuf (post-resize, actual dims sent)
if (auditMaskBuf && auditMaskBuf.length >= 24) {
const mb = auditMaskBuf;
// Read PNG dimensions from IHDR (bytes 16–23, big-endian uint32)
const maskW = mb.readUInt32BE(16);
const maskH = mb.readUInt32BE(20);
inputs.mask = {
bytes: mb.length,
width: maskW,
height: maskH,
...((_internal?.cropMeta) && { crop_percent: {
left: _internal.cropMeta.left,
top: _internal.cropMeta.top,
right: _internal.cropMeta.right,
bottom: _internal.cropMeta.bottom,
} }),
};
}
if (Object.keys(inputs).length > 0) {
audit.setInputs(inputs);
}
// === OUTPUT (Step 3+4) ===
const output = {};
// Backend returned dimensions
if (typeof backendReturnedW === "number" &&
typeof backendReturnedH === "number") {
output.backend_returned = {
width: backendReturnedW,
height: backendReturnedH,
};
}
// Post-processed dimensions
if (typeof effWidth === "number" && typeof effHeight === "number") {
output.post_processed = { width: effWidth, height: effHeight };
}
// Inference time
if (typeof inferenceMs === "number") {
output.inference_time_ms = inferenceMs;
}
// Model used
if (typeof meta.model === "string" && meta.model.trim()) {
output.model_used = meta.model;
}
// Model origin + presets
// Read overlay info from service result metadata (authoritative source)
try {
const meta = result?.metadata || {};
// overlay_source and overlay_preset come from the service layer
if (meta.overlay_source) {
output.overlay_source = meta.overlay_source;
}
if (meta.overlay_preset) {
output.overlay_preset = meta.overlay_preset;
}
if (typeof meta.defaults_used === "string" && meta.defaults_used.trim()) {
output.defaults_used = meta.defaults_used;
}
if (typeof meta.overlay_lookup_mode === "string" &&
meta.overlay_lookup_mode.trim()) {
output.overlay_lookup_mode = meta.overlay_lookup_mode;
}
if (typeof meta.i2i_profile === "string" && meta.i2i_profile.trim()) {
output.i2i_profile = meta.i2i_profile;
}
if (typeof meta.strength_used === "number" && Number.isFinite(meta.strength_used)) {
output.strength_used = meta.strength_used;
}
if (typeof meta.steps_used === "number" && Number.isFinite(meta.steps_used)) {
output.steps_used = meta.steps_used;
}
if (typeof meta.sampler_used === "string" && meta.sampler_used.trim()) {
output.sampler_used = meta.sampler_used;
}
if (typeof meta.guidance_scale_used === "number" &&
Number.isFinite(meta.guidance_scale_used)) {
output.guidance_scale_used = meta.guidance_scale_used;
}
if (typeof meta.shift_used === "number" && Number.isFinite(meta.shift_used)) {
output.shift_used = meta.shift_used;
}
if (typeof meta.resolution_dependent_shift_used === "boolean") {
output.resolution_dependent_shift_used = meta.resolution_dependent_shift_used;
}
if (typeof meta.compression_artifacts_used === "string") {
output.compression_artifacts_used = meta.compression_artifacts_used;
}
if (typeof meta.compression_artifacts_quality_used === "number" && Number.isFinite(meta.compression_artifacts_quality_used)) {
output.compression_artifacts_quality_used = meta.compression_artifacts_quality_used;
}
// Model used (from service - the actual model that was sent to Draw Things)
if (typeof meta.model === "string" && meta.model.trim()) {
output.model_used = path_1.default.basename(meta.model);
}
// LoRAs used (from service - actual LoRA files validated and sent)
if (Array.isArray(meta.loras_used) && meta.loras_used.length > 0) {
output.loras_used = meta.loras_used.map((f) => path_1.default.basename(f));
}
// Seed used (from service - effective seed after defaults/overlays)
if (typeof meta.seed === "number" && Number.isFinite(meta.seed)) {
output.seed = meta.seed;
}
if (typeof meta.seed_mode === "string" && meta.seed_mode.trim()) {
output.seed_mode = meta.seed_mode;
}
if (typeof meta.seed_source === "string" && meta.seed_source.trim()) {
output.seed_source = meta.seed_source;
}
if (typeof meta.seed_mode_source === "string" &&
meta.seed_mode_source.trim()) {
output.seed_mode_source = meta.seed_mode_source;
}
}
catch { }
// Prompt used (prefer backend's prompt_used, fallback to user input)
const metaPrompt = result?.metadata?.prompt_used;
const promptUsed = typeof metaPrompt === "string" && metaPrompt.trim()
? metaPrompt
: typeof input?.prompt === "string"
? input.prompt
: undefined;
if (promptUsed) {
output.prompt_used = promptUsed;
}
// Prompt origin
const backendOriginRaw = result?.metadata?.prompt_origin;
const userPromptInput = typeof input?.prompt === "string"
? String(input.prompt).trim()
: "";
if (backendOriginRaw === "user" || userPromptInput) {
output.prompt_origin = "user";
}
else {
output.prompt_origin = "default";
}
// Images (saved files)
if (Array.isArray(savedFiles) && savedFiles.length > 0) {
output.images = savedFiles.map((s, i) => {
const pv = previews[i];
const iMatch = /[-]i(\d+)\.(png|jpe?g|webp)$/i.exec(s?.fileName || "");
const idx = iMatch ? parseInt(iMatch[1], 10) : i + 1;
return {
i: idx,
path: s.savedPath,
url: s.fileUrl,
bytes: s.size,
...(httpOriginals[i]
? { http_url: httpOriginals[i] }
: {}),
...(pv
? { preview_path: pv.filePath, preview_url: pv.fileUrl }
: {}),
...(httpPreviews[i]
? { http_preview_url: httpPreviews[i] }
: {}),
};
});
}
audit.setOutput(output);
await audit.write();
if (pendingAudit2) {
try {
await pendingAudit2.write();
}
catch (e2) {
log(`zoom-pass audit2 write failed: ${String(e2)}`);
}
pendingAudit2 = null;
}
}
catch (e) {
log(`audit logging error: ${String(e)}`);
}
try {
const p = process.env.HTTP_SERVER_PORT;
log(`[httpServer] generate_image: external server ${httpBase ? "healthy" : "unavailable"}${p ? ` (port=${p})` : ""}.`);
}
catch { }
const extractStableVariantV = (fileName) => {
try {
const m = /[-]i(\d+)\.(png|jpe?g|webp|mov)$/i.exec(String(fileName || "")) ||
/-v(\d+)\.(png|jpe?g|webp|mov)$/i.exec(String(fileName || ""));
if (!m)
return undefined;
const n = parseInt(m[1], 10);
return Number.isFinite(n) && n > 0 ? n : undefined;
}
catch {
return undefined;
}
};
const variantLinksText = savedFiles.length > 0
? savedFiles
.map((s, i) => {
const stableV = extractStableVariantV(s.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1;
const httpPreviewUrl = httpPreviews[i];
const fallback = previews[i]?.fileUrl || s.fileUrl;
const url = httpPreviewUrl ? httpPreviewUrl : fallback;
return `Preview i${stableV}: ${url}`;
})
.join(" | ")
: "";
// Originals are saved directly to chat working directory.
const originalLinksText = savedFiles.length > 0
? savedFiles
.map((s, i) => {
const stableV = extractStableVariantV(s.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1;
const httpUrl = httpOriginals[i];
const url = httpUrl ? httpUrl : s.fileUrl;
return `Original i${stableV}: ${url}`;
})
.join(" | ")
: "";
const note = typeof requestedVariants === "number" &&
requestedVariants !== usedVariants
? `Note: variants=${requestedVariants} was clamped to ${usedVariants}.`
: null;
try {
const invMs = result?.metadata?.inference_time_ms;
const wLog = typeof effWidth === "number"
? effWidth
: typeof postProcessedW === "number"
? postProcessedW
: typeof meta.width === "number"
? Math.round(meta.width)
: undefined;
const hLog = typeof effHeight === "number"
? effHeight
: typeof postProcessedH === "number"
? postProcessedH
: typeof meta.height === "number"
? Math.round(meta.height)
: undefined;
log(`generation summary: backend=${resolvedName} mode=${effectiveMode} width=${typeof wLog === "number" ? wLog : "-"} height=${typeof hLog === "number" ? hLog : "-"} previews=${previews.length} variants=${buffers.length} inferenceMs=${invMs ?? "-"}`);
}
catch { }
const reviewHint = "Carefully examine the preview and comment on how well it matches your prompt. Do not assume it does.";
const { files: _files, ...summaryNoFilesBase } = summary;
const summaryNoFiles = await (async () => {
const modelUsedBasename = typeof meta.model === "string" && meta.model.trim()
? path_1.default.basename(meta.model)
: undefined;
if (!modelUsedBasename)
return summaryNoFilesBase;
let matchingPresets;
const overlaySourceFromMeta = meta.overlay_source;
const overlayPresetFromMeta = meta.overlay_preset;
if (overlayPresetFromMeta && overlaySourceFromMeta) {
const dotIdx = overlayPresetFromMeta.indexOf(".");
const presetMode = dotIdx >= 0 ? overlayPresetFromMeta.slice(0, dotIdx) : undefined;
const presetModelId = dotIdx >= 0 ? overlayPresetFromMeta.slice(dotIdx + 1) : undefined;
matchingPresets = [{
mode: presetMode,
modelId: presetModelId,
preset: overlayPresetFromMeta,
overlaySource: overlaySourceFromMeta,
...(overlaySourceFromMeta === "custom" ? { customConfig: overlayPresetFromMeta } : {}),
}];
}
else if (overlaySourceFromMeta === "modelOverlay") {
try {
const { resolveImageModelInfoFromModelUsed } = await import("../helpers/imageModelMeta.js");
const info = resolveImageModelInfoFromModelUsed(modelUsedBasename, {
mode: effectiveMode,
});
const overlayOnly = (info?.presets || []).filter((p) => p.overlaySource === "modelOverlay");
if (overlayOnly.length > 0)
matchingPresets = overlayOnly;
}
catch { }
}
return {
...summaryNoFilesBase,
model_used: modelUsedBasename,
...(matchingPresets ? { model_presets: matchingPresets } : {}),
};
})();
if (previewInChat && previews.length > 0) {
// The preview files were already written to the primary chat working directory.
// Reuse those file names to avoid duplicates.
const imageContents = previews.map((p, i) => {
const fname = String(p.fileName || "");
const stableV = extractStableVariantV(fname) ||
extractStableVariantV(savedFiles[i]?.fileName) ||
i + 1;
return {
type: "image",
fileName: String(savedFiles[i]?.fileName || fname),
mimeType: p.mimeType,
markdown: `})`,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
};
});
return {
content: [
...(note ? [{ type: "text", text: note }] : []),
...imageContents,
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
...(note ? [{ type: "text", text: note }] : []),
...(() => {
if (httpOriginals.length > 0 && httpOriginals[0]) {
// PREVIEW_IN_CHAT = false: simplified response without inline previews.
// Markdown will be injected by orchestrator after tool call.
// Still provide a hint so the model reviews the output quality.
return httpOriginals.map((u, i) => ({
type: "text",
text: `${isVideoResult ? "Video" : "Image"} i${extractStableVariantV(savedFiles[i]?.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1} successfully generated.`,
$hint: reviewHint,
}));
}
const count = typeof usedVariants === "number" && usedVariants > 0
? usedVariants
: 1;
return Array.from({ length: count }, (_, i) => ({
type: "text",
text: `${isVideoResult ? "Video" : "Image"} i${extractStableVariantV(savedFiles[i]?.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1} successfully generated.`,
$hint: reviewHint,
}));
})(),
...(variantLinksText
? [{ type: "text", text: variantLinksText }]
: []),
...(originalLinksText
? [{ type: "text", text: originalLinksText }]
: []),
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`generate_image error: ${error instanceof Error ? error.message : String(error)}`);
await logError(error);
return {
content: [
{ type: "text", text: "Failed to generate image: backend error" },
],
isError: true,
};
}
}
exports.ToolSchemas = {
generate_image: core_bundle_mjs_1.GenerateToolParamsShapeMinimal,
crop: core_bundle_mjs_1.CropToolParamsShape,
mask: core_bundle_mjs_1.CropToolParamsShape,
zoom_in: core_bundle_mjs_1.ZoomInToolParamsShape,
inpaint: core_bundle_mjs_1.InpaintToolParamsShape,
outpaint: core_bundle_mjs_1.OutpaintToolParamsShape,
refine: core_bundle_mjs_1.RefineToolParamsShape,
};
// ─────────────────────────────────────────────────────────────────────────────
// applyCropFormat: compute final crop percentages from explicit values + imageFormat.
// resolveBoxOverride: resolve a per-box crop side value for multi-region detect.
// override — the raw field value from input: scalar (applies to all boxes) |
// array (parallel to detectLabel, null entry = use detectDefault) | undefined.
// index — the box index within the detectLabel array.
// detectDefault — the percent value returned by resolveDetectionCrop for this side.
// Returns the value to pass into applyCropFormat for this specific box (or undefined = let
// applyCropFormat treat it as "unset", which will then fall through to detectDefault via the
// caller setting the field when undefined).
function resolveBoxOverride(override, index, detectDefault) {
if (override === undefined)
return undefined; // no override → caller uses detectDefault
if (Array.isArray(override)) {
const entry = override[index];
if (entry === null || entry === undefined)
return undefined; // null = keep detection
return entry;
}
// scalar: same value for every box
return override;
}
// If imageFormat is set AND not all 4 sides are given, derive missing sides from AR.
// If imageFormat is NOT set, use explicit values only (0 for unset sides).
// If all 4 sides are explicitly given, imageFormat is ignored.
// Each crop side accepts number (→ %) or string like "10", "10%", "10 %", "120px", "120 px".
// Returns { left, right, top, bottom } as percentages, or an error string.
// ─────────────────────────────────────────────────────────────────────────────
function parseCropSide(raw) {
if (raw === undefined)
return undefined;
if (typeof raw === "number") {
if (raw < 0)
return new Error(`Invalid crop value ${raw}: must be >= 0.`);
return { value: raw, unit: "%" };
}
const m = String(raw).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(px|%)?$/i);
if (!m)
return new Error(`Invalid crop value "${raw}": expected a number optionally followed by "%" or "px".`);
const num = parseFloat(m[1]);
if (num < 0)
return new Error(`Invalid crop value "${raw}": must be >= 0.`);
const unitStr = (m[2] ?? "").toLowerCase();
return { value: num, unit: unitStr === "px" ? "px" : "%" };
}
async function applyCropFormat(srcBuf, input) {
const pLeft = parseCropSide(input.cropLeft);
const pRight = parseCropSide(input.cropRight);
const pTop = parseCropSide(input.cropTop);
const pBottom = parseCropSide(input.cropBottom);
for (const [name, p] of [["cropLeft", pLeft], ["cropRight", pRight], ["cropTop", pTop], ["cropBottom", pBottom]]) {
if (p instanceof Error)
return p.message;
}
const leftSet = pLeft !== undefined;
const rightSet = pRight !== undefined;
const topSet = pTop !== undefined;
const bottomSet = pBottom !== undefined;
const hasPx = pLeft?.unit === "px" || pRight?.unit === "px" ||
pTop?.unit === "px" || pBottom?.unit === "px";
const needsImageDims = hasPx || !!input.imageFormat || input.frameAdjust !== undefined;
let imgW = 0, imgH = 0;
if (needsImageDims) {
const size = await (0, core_bundle_mjs_1.getSize)(srcBuf);
imgW = size.width;
imgH = size.height;
}
function toPct(p, dim) {
if (!p)
return 0;
return p.unit === "px" ? (dim > 0 ? (p.value / dim) * 100 : 0) : p.value;
}
let left = toPct(pLeft, imgW);
let right = toPct(pRight, imgW);
let top = toPct(pTop, imgH);
let bottom = toPct(pBottom, imgH);
if (!(leftSet && rightSet && topSet && bottomSet) && input.imageFormat) {
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetRatio = ratioMap[input.imageFormat]; // W/H
// imgW, imgH already computed above (needsImageDims covers this branch)
const hFixed = leftSet && rightSet;
const vFixed = topSet && bottomSet;
const cropW_px = imgW * (1 - left / 100 - right / 100);
const cropH_px = imgH * (1 - top / 100 - bottom / 100);
if (hFixed) {
const cropH_needed = cropW_px / targetRatio;
if (topSet) {
bottom = Math.max(0, (imgH * (1 - top / 100) - cropH_needed) / imgH * 100);
}
else if (bottomSet) {
top = Math.max(0, (imgH * (1 - bottom / 100) - cropH_needed) / imgH * 100);
}
else {
const margin = Math.max(0, (imgH - cropH_needed) / 2 / imgH * 100);
top = margin;
bottom = margin;
}
}
else if (vFixed) {
const cropW_needed = cropH_px * targetRatio;
if (leftSet) {
right = Math.max(0, (imgW * (1 - left / 100) - cropW_needed) / imgW * 100);
}
else if (rightSet) {
left = Math.max(0, (imgW * (1 - right / 100) - cropW_needed) / imgW * 100);
}
else {
const margin = Math.max(0, (imgW - cropW_needed) / 2 / imgW * 100);
left = margin;
right = margin;
}
}
else {
const currentRatio = cropW_px / cropH_px;
if (Math.abs(currentRatio - targetRatio) > 1e-6) {
if (currentRatio > targetRatio) {
const cropW_needed = cropH_px * targetRatio;
if (leftSet) {
right = Math.max(0, (imgW * (1 - left / 100) - cropW_needed) / imgW * 100);
}
else if (rightSet) {
left = Math.max(0, (imgW * (1 - right / 100) - cropW_needed) / imgW * 100);
}
else {
const margin = Math.max(0, (imgW - cropW_needed) / 2 / imgW * 100);
left = margin;
right = margin;
}
}
else {
const cropH_needed = cropW_px / targetRatio;
if (topSet) {
bottom = Math.max(0, (imgH * (1 - top / 100) - cropH_needed) / imgH * 100);
}
else if (bottomSet) {
top = Math.max(0, (imgH * (1 - bottom / 100) - cropH_needed) / imgH * 100);
}
else {
const margin = Math.max(0, (imgH - cropH_needed) / 2 / imgH * 100);
top = margin;
bottom = margin;
}
}
}
}
if (left + right >= 100 || top + bottom >= 100) {
return `imageFormat '${input.imageFormat}' cannot be achieved with the given crop values.`;
}
}
else if (input.imageFormat) {
// All 4 sides are set (e.g. from detectLabel) — apply symmetric AR expansion,
// matching the zoom-in algorithm: always expand the shorter dimension outward.
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetRatio = ratioMap[input.imageFormat];
if (targetRatio !== undefined) {
const regionW = imgW * (1 - left / 100 - right / 100);
const regionH = imgH * (1 - top / 100 - bottom / 100);
const regionAR = regionW / regionH;
if (Math.abs(regionAR - targetRatio) > 0.005) {
if (regionAR < targetRatio) {
// Region too tall → expand width symmetrically.
const totalExtraPct = ((regionH * targetRatio - regionW) / imgW) * 100;
const halfExtra = totalExtraPct / 2;
const leftReduction = Math.min(left, halfExtra);
const rightReduction = Math.min(right, halfExtra + (halfExtra - leftReduction));
const leftReduction2 = Math.min(left, halfExtra + (halfExtra - rightReduction));
left = Math.max(0, left - leftReduction2);
right = Math.max(0, right - rightReduction);
}
else {
// Region too wide → expand height symmetrically.
const totalExtraPct = ((regionW / targetRatio - regionH) / imgH) * 100;
const halfExtra = totalExtraPct / 2;
const topReduction = Math.min(top, halfExtra);
const bottomReduction = Math.min(bottom, halfExtra + (halfExtra - topReduction));
const topReduction2 = Math.min(top, halfExtra + (halfExtra - bottomReduction));
top = Math.max(0, top - topReduction2);
bottom = Math.max(0, bottom - bottomReduction);
}
}
}
}
// Apply frameAdjust to the final crop region — always the last step.
// Each side is clamped independently at 0 when the expansion overshoots the image boundary.
if (input.frameAdjust !== undefined) {
const cropW_px = imgW * (1 - left / 100 - right / 100);
const cropH_px = imgH * (1 - top / 100 - bottom / 100);
const cropDiag = Math.hypot(cropW_px, cropH_px);
let d_px;
if (typeof input.frameAdjust === "string") {
const m = String(input.frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (!m)
return `Invalid frameAdjust value: "${input.frameAdjust}"`;
const val = parseFloat(m[1]);
d_px = m[2]?.toLowerCase() === "px" ? val : (val / 100) * cropDiag;
}
else {
d_px = (input.frameAdjust / 100) * cropDiag;
}
const dLR = (d_px / imgW) * 100;
const dTB = (d_px / imgH) * 100;
left = Math.max(0, left - dLR);
right = Math.max(0, right - dLR);
top = Math.max(0, top - dTB);
bottom = Math.max(0, bottom - dTB);
}
if (left + right >= 100) {
return `cropLeft + cropRight must not cover the full image width.`;
}
if (top + bottom >= 100) {
return `cropTop + cropBottom must not cover the full image height.`;
}
return { left, right, top, bottom };
}
// ─────────────────────────────────────────────────────────────────────────────
// findDetectRecordForCanvas: locate the detect_object record relevant to rawCanvas — either
// rawCanvas IS the detect_object's own iN result, or rawCanvas is the ORIGINAL source and some
// later detect_object record references it via detectSource (latest wins, case-insensitive).
// Shared by expandDetectIndices/resolveDetectionCrop's own lookups and autoSelectDetectLabel().
// ─────────────────────────────────────────────────────────────────────────────
function findDetectRecordForCanvas(imageRecords, rawCanvas) {
if (!rawCanvas)
return undefined;
const pref = parsePrefixedNotation(rawCanvas);
if (pref?.pool === "image") {
const candidate = imageRecords.find((r) => r?.i === pref.index);
if (candidate && Array.isArray(candidate.detections))
return candidate;
}
const lc = rawCanvas.toLowerCase();
const candidates = imageRecords
.filter((r) => typeof r?.detectSource === "string" &&
r.detectSource.toLowerCase() === lc &&
Array.isArray(r.detections))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0));
return candidates[0];
}
// ─────────────────────────────────────────────────────────────────────────────
// autoSelectDetectLabel: when detectLabel is omitted and rawCanvas resolves to a prior
// detect_object record (rawCanvas itself, or the ORIGINAL source of a later detect_object call —
// see findDetectRecordForCanvas), auto-selects its label when exactly one distinct label exists
// (mirrors detectIndex's own default of 0). Throws when 2+ distinct labels exist, listing them.
// Returns the record's single label, or undefined (no-op — canvas has no/ambiguous-free detections).
// ─────────────────────────────────────────────────────────────────────────────
function autoSelectDetectLabel(imageRecords, rawCanvas) {
const detectRec = findDetectRecordForCanvas(imageRecords, rawCanvas);
if (!detectRec || !Array.isArray(detectRec.detections) || detectRec.detections.length === 0)
return undefined;
const distinctLabels = [...new Set(detectRec.detections.map((d) => d?.label).filter(Boolean))];
if (distinctLabels.length > 1) {
throw new Error(`Ambiguous — canvas '${rawCanvas}' contains ${detectRec.detections.length} detected regions. ` +
`Specify detectLabel to select one. Available labels: ${distinctLabels.join(", ")}.`);
}
return distinctLabels.length === 1 ? distinctLabels[0] : undefined;
}
// ─────────────────────────────────────────────────────────────────────────────
// expandDetectIndices: count how many detections match a single label in the
// detect_object record for rawCanvas. Returns [0..N-1], or [0] if only 1 match.
// Used for Option-A auto-expansion: single detectLabel + no explicit detectIndex
// → automatically iterate all instances of that label.
// ─────────────────────────────────────────────────────────────────────────────
function expandDetectIndices(imageRecords, rawCanvas, label) {
const detectRec = findDetectRecordForCanvas(imageRecords, rawCanvas);
if (!detectRec)
return [0];
const count = detectRec.detections.filter((d) => typeof d?.label === "string" && d.label.toLowerCase() === label.toLowerCase()).length;
return count > 1 ? Array.from({ length: count }, (_, i) => i) : [0];
}
// ─────────────────────────────────────────────────────────────────────────────
// ─────────────────────────────────────────────────────────────────────────────
// resolveDetectionCrop: find a detection by label in a prior detect_object record
// and return crop percentages, optionally adjusted by frameAdjust.
// ─────────────────────────────────────────────────────────────────────────────
function resolveDetectionCrop(imageRecords, rawCanvas, detectLabel, detectIndex, frameAdjust) {
// --- 1. find detect_object record ---
let detectRec;
let canvasWasDetectRecord = false;
let srcCanvas = rawCanvas;
if (rawCanvas) {
// Check if rawCanvas is itself an iN detect_object record
const pref = parsePrefixedNotation(rawCanvas);
if (pref?.pool === "image") {
const candidate = imageRecords.find((r) => r?.i === pref.index);
if (candidate && Array.isArray(candidate.detections)) {
detectRec = candidate;
canvasWasDetectRecord = true;
srcCanvas = typeof detectRec.detectSource === "string" ? detectRec.detectSource : undefined;
}
}
}
if (!detectRec && rawCanvas) {
// Search for detect_object record referencing rawCanvas as source
const lc = rawCanvas.toLowerCase();
const candidates = imageRecords
.filter((r) => typeof r?.detectSource === "string" &&
r.detectSource.toLowerCase() === lc &&
Array.isArray(r.detections))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0));
if (candidates.length > 0) {
detectRec = candidates[0];
canvasWasDetectRecord = false;
srcCanvas = rawCanvas;
}
}
if (!detectRec) {
return {
ok: false,
error: `No detect_object result found for canvas '${rawCanvas ?? "(auto)"}'. Run detect_object on the source image first.`,
};
}
// --- 2. find detection by label ---
const detections = detectRec.detections;
const matching = detections.filter((d) => typeof d?.label === "string" && d.label.toLowerCase() === detectLabel.toLowerCase());
if (matching.length === 0) {
const allLabels = [...new Set(detections.map((d) => d?.label).filter(Boolean))].join(", ");
return {
ok: false,
error: `Label '${detectLabel}' not found in detections. Available: ${allLabels || "(none)"}`,
};
}
if (detectIndex >= matching.length) {
return {
ok: false,
error: `detectIndex ${detectIndex} out of range for label '${detectLabel}' (${matching.length} detection(s)).`,
};
}
const det = matching[detectIndex];
// --- 3. apply frameAdjust or use pre-computed crop percentages ---
if (frameAdjust === undefined) {
const crop = det.crop;
return {
ok: true,
cropLeft: typeof crop?.cropLeft === "number" ? crop.cropLeft : 0,
cropRight: typeof crop?.cropRight === "number" ? crop.cropRight : 0,
cropTop: typeof crop?.cropTop === "number" ? crop.cropTop : 0,
cropBottom: typeof crop?.cropBottom === "number" ? crop.cropBottom : 0,
srcCanvas,
canvasWasDetectRecord,
};
}
// frameAdjust requires image dimensions stored on the detect record
const imgW = typeof detectRec.imageWidth === "number" ? detectRec.imageWidth : 0;
const imgH = typeof detectRec.imageHeight === "number" ? detectRec.imageHeight : 0;
if (imgW <= 0 || imgH <= 0) {
return {
ok: false,
error: `frameAdjust requires imageWidth/imageHeight in the detect record. Re-run detect_object to update the record.`,
};
}
// Bbox raw coordinates
const bbox = det.bbox;
const bx1 = typeof bbox?.x1 === "number" ? bbox.x1 : 0;
const by1 = typeof bbox?.y1 === "number" ? bbox.y1 : 0;
const bx2 = typeof bbox?.x2 === "number" ? bbox.x2 : imgW;
const by2 = typeof bbox?.y2 === "number" ? bbox.y2 : imgH;
// Parse frameAdjust → per-axis pixel deltas.
// %-value is relative to the respective bbox dimension (width for X, height
// for Y) so the total box grows by pct% in each dimension, preserving AR.
// px-value is an absolute margin applied uniformly on all four sides.
const bboxW = bx2 - bx1;
const bboxH = by2 - by1;
let dX_px;
let dY_px;
if (typeof frameAdjust === "string") {
const m = String(frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (!m) {
return { ok: false, error: `Invalid frameAdjust value: "${frameAdjust}"` };
}
const val = parseFloat(m[1]);
if (m[2]?.toLowerCase() === "px") {
dX_px = Math.round(val);
dY_px = Math.round(val);
}
else {
dX_px = Math.round((val / 100) * bboxW / 2);
dY_px = Math.round((val / 100) * bboxH / 2);
}
}
else {
dX_px = Math.round((frameAdjust / 100) * bboxW / 2);
dY_px = Math.round((frameAdjust / 100) * bboxH / 2);
}
// Apply adjustment. Clamp to [0, imgW] / [0, imgH] (inclusive — bbox
// coordinates are exclusive-end so x2=imgW is valid and must not become
// imgW-1, which would leave a spurious 1-pixel crop at a border edge).
let x1 = Math.max(0, bx1 - dX_px);
let x2 = Math.min(imgW, bx2 + dX_px);
let y1 = Math.max(0, by1 - dY_px);
let y2 = Math.min(imgH, by2 + dY_px);
// Ensure at least 1×1
if (x2 <= x1)
x2 = Math.min(imgW, x1 + 1);
if (y2 <= y1)
y2 = Math.min(imgH, y1 + 1);
// Convert back to crop percentages
return {
ok: true,
cropLeft: (x1 / imgW) * 100,
cropRight: ((imgW - x2) / imgW) * 100,
cropTop: (y1 / imgH) * 100,
cropBottom: ((imgH - y2) / imgH) * 100,
srcCanvas,
canvasWasDetectRecord,
};
}
// handleCrop: crop an image by percentage from each side, save as a new variant
// ─────────────────────────────────────────────────────────────────────────────
async function handleCrop(pluginParams, requestId) {
await ensureBackendReady().catch((e) => {
log(`[crop] ensureBackendReady failed: ${String(e)}`);
});
try {
const parsed = core_bundle_mjs_1.CropToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid crop parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
let currentLmChatId = null;
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
return {
content: [{ type: "text", text: "Failed to resolve LM Studio chat working directory." }],
isError: true,
};
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Load source state
const st = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const images = imageRecords
.filter((r) => r && typeof r.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0))
.map((r) => ({ i: r.i || 1, path: path_1.default.join(primaryOutDir, r.filename) }));
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variants = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(primaryOutDir, v.filename) }));
// Resolve canvas notation
let rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
let srcBuf;
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Runs before the
// detectLabel handling below, so cropLeft/cropRight/cropTop/cropBottom overrides (null = keep
// detection value) still apply.
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = [autoLabel];
}
// detectLabel: resolve crop coordinates from a prior detect_object run
if (Array.isArray(input.detectLabel) && input.detectLabel.length > 0) {
if (input.detectLabel.length > 1) {
return {
content: [{ type: "text", text: "crop supports only a single detectLabel. Use mask + inpaint for multi-region workflows." }],
isError: true,
};
}
const label = input.detectLabel[0];
const indices = Array.isArray(input.detectIndex) ? input.detectIndex : (input.detectIndex !== undefined ? [input.detectIndex] : []);
const detectIdx = indices[0] ?? 0;
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust is applied as the final step in applyCropFormat
);
if (!detectResult.ok) {
return {
content: [{ type: "text", text: detectResult.error }],
isError: true,
};
}
// Detect provides defaults; per-box overrides (scalar or array[0]) take precedence.
const oL = resolveBoxOverride(input.cropLeft, 0, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, 0, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, 0, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, 0, detectResult.cropBottom);
input.cropLeft = oL !== undefined ? oL : detectResult.cropLeft;
input.cropRight = oR !== undefined ? oR : detectResult.cropRight;
input.cropTop = oT !== undefined ? oT : detectResult.cropTop;
input.cropBottom = oB !== undefined ? oB : detectResult.cropBottom;
// frameAdjust is kept in input — applyCropFormat applies it last, clamping each side independently.
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
rawCanvas = detectResult.srcCanvas;
input.canvas = detectResult.srcCanvas;
}
}
// Auto-redirect: if canvas points to a detect_object result and no detectLabel is given, use detectSource
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const _autoRedirectPref = parsePrefixedNotation(rawCanvas);
if (_autoRedirectPref?.pool === "image") {
const _autoRedirectRec = imageRecords.find((r) => r?.i === _autoRedirectPref.index);
if (typeof _autoRedirectRec?.sourceTool === "string" &&
_autoRedirectRec.sourceTool.includes("detect_object") &&
typeof _autoRedirectRec.detectSource === "string") {
rawCanvas = _autoRedirectRec.detectSource;
input.canvas = _autoRedirectRec.detectSource;
}
}
}
// Auto-redirect: if canvas points to a mask result, crop the original unlined image instead.
// The CYAN annotation lines would otherwise appear at the crop boundary in the result.
if (rawCanvas) {
const _maskRedirectPref = parsePrefixedNotation(rawCanvas);
if (_maskRedirectPref?.pool === "image") {
const _maskRec = imageRecords.find((r) => r?.i === _maskRedirectPref.index);
if (typeof _maskRec?.sourceTool === "string" &&
_maskRec.sourceTool.includes("/mask") &&
typeof _maskRec.cropSource === "string") {
log(`[crop] canvas ${rawCanvas} is a mask result — redirecting to cropSource ${_maskRec.cropSource}`);
rawCanvas = _maskRec.cropSource;
input.canvas = _maskRec.cropSource;
}
}
}
try {
if (rawCanvas) {
const pref = parsePrefixedNotation(rawCanvas);
if (pref) {
if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${pref.index} not found.`);
}
srcBuf = lm.buffer;
}
else if (pref.pool === "variant") {
const found = variants.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else if (pref.pool === "image") {
const found = images.find((img) => img.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
srcBuf = await resolvePictureBuffer(found, primaryOutDir, pref.index);
}
}
else {
const pathToken = await resolveCanvasPathToken(rawCanvas, primaryOutDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
srcBuf = await fs_1.default.promises.readFile(pathToken);
}
}
else {
// Auto-select single source
const total = attachments.length + variants.length + images.length + pictures.length;
if (total === 0)
throw new Error("No source image available.");
if (total > 1)
throw new Error("Ambiguous source — specify canvas explicitly.");
if (attachments.length === 1) {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: typeof attachments[0]?.a === "number" ? attachments[0].a : 1,
});
if (!lm?.ok || !lm.buffer)
throw new Error("Attachment not found.");
srcBuf = lm.buffer;
}
else if (variants.length === 1) {
srcBuf = await fs_1.default.promises.readFile(variants[0].path);
}
else if (images.length === 1) {
srcBuf = await fs_1.default.promises.readFile(images[0].path);
}
else {
srcBuf = await fs_1.default.promises.readFile(path_1.default.join(primaryOutDir, String(pictures[0].filename || "")));
}
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
const cropResult = await applyCropFormat(srcBuf, input);
if (typeof cropResult === "string") {
return { content: [{ type: "text", text: cropResult }], isError: true };
}
const { left, right, top, bottom } = cropResult;
// Original dimensions for result reporting
const originalSize = await (0, core_bundle_mjs_1.getSize)(srcBuf);
const originalWidth = originalSize.width;
const originalHeight = originalSize.height;
// Apply crop
const croppedBuf = await (0, core_bundle_mjs_1.cropToPng)(srcBuf, { left, top, right, bottom });
// Measure cropped dimensions for XMP
const croppedSize = await (0, core_bundle_mjs_1.getSize)(croppedBuf);
// Build XMP params for crop (no diffusion — processing only)
const cropXmpParams = {
mode: "crop",
...(typeof croppedSize.width === "number" ? { width: croppedSize.width } : {}),
...(typeof croppedSize.height === "number" ? { height: croppedSize.height } : {}),
crop: { left, top, right, bottom },
...(typeof input.canvas === "string" ? { cropSource: input.canvas } : {}),
};
// Save as new image
const baseImageI = Math.max(1, st.counters?.nextImageI ?? 1);
const stamp = isoStampCompact();
const baseName = `image-${stamp}-i${baseImageI}`;
const saved = await saveOriginalPng(croppedBuf, primaryOutDir, `${baseName}.png`, cropXmpParams);
log(`[crop] saved: ${saved.savedPath} (${saved.size} bytes) [i${baseImageI}]`);
// Generate preview
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const imageRecordsForState = [
{
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: baseImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/crop`,
cropLeft: left,
cropTop: top,
cropRight: right,
cropBottom: bottom,
cropSource: typeof input.canvas === "string" ? input.canvas : undefined,
},
];
const previews = [];
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(croppedBuf, primaryOutDir, saved.fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`[crop] preview saved: ${previewFilePath} ${p.width}x${p.height}`);
}
catch (e) {
log(`[crop] preview failed: ${String(e)}`);
}
// Update state
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`[crop] state updated: appended i${baseImageI}, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`[crop] state update failed: ${String(e)}`);
}
// HTTP URLs
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginal = httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(saved.fileName, httpBase, currentLmChatId || undefined)
: "";
const httpPreview = (() => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[0]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
})();
// Audit log — effectiveRequestId is reused in the summary below, so a caller (the MCP
// adapter) can always name its HTML report after this exact audit requestId.
const effectiveRequestId = requestId ?? `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "crop", mode: "crop", requestId: effectiveRequestId });
if (currentLmChatId)
audit.setChatId(currentLmChatId);
const cropUserRequest = {};
if (input.canvas)
cropUserRequest.canvas = input.canvas;
cropUserRequest.cropLeft = left;
cropUserRequest.cropRight = right;
cropUserRequest.cropTop = top;
cropUserRequest.cropBottom = bottom;
audit.setUserRequest(cropUserRequest);
const auditOutput = {
i: baseImageI,
crop: { left, top, right, bottom },
images: [{
i: baseImageI,
path: saved.savedPath,
url: saved.fileUrl,
bytes: saved.size,
...(httpOriginal ? { http_url: httpOriginal } : {}),
...(previews[0] ? { preview_path: previews[0].filePath, preview_url: previews[0].fileUrl } : {}),
...(httpPreview ? { http_preview_url: httpPreview } : {}),
}],
};
audit.setOutput(auditOutput);
await audit.write();
}
catch (e) {
log(`[crop] audit logging error: ${String(e)}`);
}
// Assemble tool result — same structure as handleGenerateImage
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw === undefined
? true
: envPreviewRaw === "1" || envPreviewRaw.toLowerCase() === "true";
const summary = {
tool: "crop",
requestId: effectiveRequestId,
i: baseImageI,
originalWidth,
originalHeight,
crop: {
left: { pct: left, px: Math.round(left / 100 * originalWidth) },
right: { pct: right, px: Math.round(right / 100 * originalWidth) },
top: { pct: top, px: Math.round(top / 100 * originalHeight) },
bottom: { pct: bottom, px: Math.round(bottom / 100 * originalHeight) },
},
images_generated: 1,
files: {
original: saved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
};
const { files: _files, ...summaryNoFiles } = summary;
const fallbackPreviewUrl = previews[0]?.fileUrl || saved.fileUrl;
const variantLinksText = `Preview i${baseImageI}: ${httpPreview ? httpPreview : fallbackPreviewUrl}`;
const originalLinksText = `Original i${baseImageI}: ${httpOriginal ? httpOriginal : saved.fileUrl}`;
const reviewHint = "Carefully examine the preview and comment on how well the crop matches your intent.";
if (previewInChat && previews.length > 0) {
const fname = String(previews[0].fileName || "");
return {
content: [
{
type: "image",
fileName: saved.fileName,
mimeType: previews[0].mimeType,
markdown: ``,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
{
type: "text",
text: `Cropped Image i${baseImageI} successfully saved.`,
$hint: reviewHint,
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`[crop] error: ${error instanceof Error ? error.message : String(error)}`);
return {
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
isError: true,
};
}
}
// handleMask: draw a CYAN bounding box on the source image from crop percentages,
// save as a new image with crop metadata for use with inpaint / outpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleMask(pluginParams, requestId) {
await ensureBackendReady().catch((e) => {
log(`[mask] ensureBackendReady failed: ${String(e)}`);
});
try {
const parsed = core_bundle_mjs_1.CropToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid mask parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
let currentLmChatId = null;
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
return {
content: [{ type: "text", text: "Failed to resolve LM Studio chat working directory." }],
isError: true,
};
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Load source state
const st = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const images = imageRecords
.filter((r) => r && typeof r.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0))
.map((r) => ({ i: r.i || 1, path: path_1.default.join(primaryOutDir, r.filename) }));
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variants = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(primaryOutDir, v.filename) }));
// Resolve canvas notation
let rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
let srcBuf;
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Runs before the
// detectLabel handling below, so per-box crop overrides (null = keep detection value) still apply.
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = [autoLabel];
}
// detectLabel: resolve crop coordinates from a prior detect_object run
// Multi-region: draw one cyan box per label.
let resolvedMaskBoxes;
if (Array.isArray(input.detectLabel) && input.detectLabel.length > 0) {
let labels = input.detectLabel;
let indices = Array.isArray(input.detectIndex) ? input.detectIndex : (input.detectIndex !== undefined ? [input.detectIndex] : []);
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
const boxes = [];
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] ?? indices[0] ?? 0;
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step in applyCropFormat
);
if (!detectResult.ok) {
return {
content: [{ type: "text", text: detectResult.error }],
isError: true,
};
}
// Per-box overrides: scalar = same for all; array[li] non-null = override; null/missing = keep detect.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
// Build a temporary input object for applyCropFormat (single-box).
const boxInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas && li === 0) {
rawCanvas = detectResult.srcCanvas;
input.canvas = detectResult.srcCanvas;
}
boxes.push(boxInput); // resolved after srcBuf is available
}
resolvedMaskBoxes = boxes; // will be resolved via applyCropFormat per box below
// For single-label path compatibility: set input sides to first box so auto-redirect etc. still work.
input.cropLeft = undefined;
input.cropRight = undefined;
input.cropTop = undefined;
input.cropBottom = undefined;
input.frameAdjust = undefined;
}
// Auto-redirect: if canvas points to a detect_object result and no detectLabel is given, use detectSource
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const _autoRedirectPref = parsePrefixedNotation(rawCanvas);
if (_autoRedirectPref?.pool === "image") {
const _autoRedirectRec = imageRecords.find((r) => r?.i === _autoRedirectPref.index);
if (typeof _autoRedirectRec?.sourceTool === "string" &&
_autoRedirectRec.sourceTool.includes("detect_object") &&
typeof _autoRedirectRec.detectSource === "string") {
rawCanvas = _autoRedirectRec.detectSource;
input.canvas = _autoRedirectRec.detectSource;
}
}
}
try {
if (rawCanvas) {
const pref = parsePrefixedNotation(rawCanvas);
if (pref) {
if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${pref.index} not found.`);
}
srcBuf = lm.buffer;
}
else if (pref.pool === "variant") {
const found = variants.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else if (pref.pool === "image") {
const found = images.find((img) => img.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
srcBuf = await resolvePictureBuffer(found, primaryOutDir, pref.index);
}
}
else {
const pathToken = await resolveCanvasPathToken(rawCanvas, primaryOutDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
srcBuf = await fs_1.default.promises.readFile(pathToken);
}
}
else {
// Auto-select single source
const total = attachments.length + variants.length + images.length + pictures.length;
if (total === 0)
throw new Error("No source image available.");
if (total > 1)
throw new Error("Ambiguous source — specify canvas explicitly.");
if (attachments.length === 1) {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: typeof attachments[0]?.a === "number" ? attachments[0].a : 1,
});
if (!lm?.ok || !lm.buffer)
throw new Error("Attachment not found.");
srcBuf = lm.buffer;
}
else if (variants.length === 1) {
srcBuf = await fs_1.default.promises.readFile(variants[0].path);
}
else if (images.length === 1) {
srcBuf = await fs_1.default.promises.readFile(images[0].path);
}
else {
srcBuf = await fs_1.default.promises.readFile(path_1.default.join(primaryOutDir, String(pictures[0].filename || "")));
}
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
// Resolve multi-region box coordinates now that srcBuf is available.
let finalBoxes;
if (resolvedMaskBoxes) {
const resolved = [];
for (const boxInput of resolvedMaskBoxes) {
const r = await applyCropFormat(srcBuf, boxInput);
if (typeof r === "string")
return { content: [{ type: "text", text: r }], isError: true };
resolved.push(r);
}
finalBoxes = resolved;
}
else {
const cropResult = await applyCropFormat(srcBuf, input);
if (typeof cropResult === "string") {
return { content: [{ type: "text", text: cropResult }], isError: true };
}
finalBoxes = [cropResult];
}
// Original dimensions
const originalSize = await (0, core_bundle_mjs_1.getSize)(srcBuf);
const originalWidth = originalSize.width;
const originalHeight = originalSize.height;
// For state storage: use bounding union of all boxes (or single box).
const left = Math.min(...finalBoxes.map(b => b.left));
const right = Math.min(...finalBoxes.map(b => b.right));
const top = Math.min(...finalBoxes.map(b => b.top));
const bottom = Math.min(...finalBoxes.map(b => b.bottom));
// Draw CYAN bounding boxes on source image
const Jimp = (await import("jimp")).Jimp;
const img = await Jimp.read(srcBuf);
const cyanInt = (((0x44 & 0xff) << 24) | ((0x7d & 0xff) << 16) | ((0xf7 & 0xff) << 8) | (0xff & 0xff)) >>> 0;
const thickness = 2;
for (const box of finalBoxes) {
const x1 = Math.max(0, Math.min(originalWidth - 1, Math.round(box.left / 100 * originalWidth)));
const y1 = Math.max(0, Math.min(originalHeight - 1, Math.round(box.top / 100 * originalHeight)));
const x2 = Math.max(0, Math.min(originalWidth - 1, Math.round((100 - box.right) / 100 * originalWidth)));
const y2 = Math.max(0, Math.min(originalHeight - 1, Math.round((100 - box.bottom) / 100 * originalHeight)));
for (let t = 0; t < thickness; t++) {
for (let x = x1; x <= x2; x++) {
if (y1 + t < originalHeight)
img.setPixelColor(cyanInt, x, y1 + t);
if (y2 - t >= 0)
img.setPixelColor(cyanInt, x, y2 - t);
}
for (let y = y1; y <= y2; y++) {
if (x1 + t < originalWidth)
img.setPixelColor(cyanInt, x1 + t, y);
if (x2 - t >= 0)
img.setPixelColor(cyanInt, x2 - t, y);
}
}
}
const annotatedBuf = await img.getBuffer("image/png");
// Build XMP params (processing only, same mode as crop)
const maskXmpParams = {
mode: "crop",
...(typeof originalWidth === "number" ? { width: originalWidth } : {}),
...(typeof originalHeight === "number" ? { height: originalHeight } : {}),
crop: { left, top, right, bottom },
...(typeof input.canvas === "string" ? { cropSource: input.canvas } : {}),
};
// Save as new image
const baseImageI = Math.max(1, st.counters?.nextImageI ?? 1);
const stamp = isoStampCompact();
const baseName = `image-${stamp}-i${baseImageI}`;
const saved = await saveOriginalPng(annotatedBuf, primaryOutDir, `${baseName}.png`, maskXmpParams);
log(`[mask] saved: ${saved.savedPath} (${saved.size} bytes) [i${baseImageI}]`);
// Generate preview
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const imageRecordsForState = [
{
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: baseImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/mask`,
// Single-box scalar fields for backward compat (first box or the only box).
cropLeft: finalBoxes[0].left,
cropTop: finalBoxes[0].top,
cropRight: finalBoxes[0].right,
cropBottom: finalBoxes[0].bottom,
cropSource: typeof input.canvas === "string" ? input.canvas : undefined,
// Multi-box: always store full array so inpaint/outpaint can loop over all regions.
...(finalBoxes.length > 1 ? { bboxes: finalBoxes } : {}),
},
];
const previews = [];
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(annotatedBuf, primaryOutDir, saved.fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`[mask] preview saved: ${previewFilePath} ${p.width}x${p.height}`);
}
catch (e) {
log(`[mask] preview failed: ${String(e)}`);
}
// Update state
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`[mask] state updated: appended i${baseImageI}, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`[mask] state update failed: ${String(e)}`);
}
// HTTP URLs
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginal = httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(saved.fileName, httpBase, currentLmChatId || undefined)
: "";
const httpPreview = (() => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[0]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
})();
// Audit log — effectiveRequestId is reused in the summary below, so a caller (the MCP
// adapter) can always name its HTML report after this exact audit requestId.
const effectiveRequestId = requestId ?? `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "mask", mode: "crop", requestId: effectiveRequestId });
if (currentLmChatId)
audit.setChatId(currentLmChatId);
const maskUserRequest = {};
if (input.canvas)
maskUserRequest.canvas = input.canvas;
maskUserRequest.cropLeft = left;
maskUserRequest.cropRight = right;
maskUserRequest.cropTop = top;
maskUserRequest.cropBottom = bottom;
audit.setUserRequest(maskUserRequest);
const auditOutput = {
i: baseImageI,
crop: { left, top, right, bottom },
images: [{
i: baseImageI,
path: saved.savedPath,
url: saved.fileUrl,
bytes: saved.size,
...(httpOriginal ? { http_url: httpOriginal } : {}),
...(previews[0] ? { preview_path: previews[0].filePath, preview_url: previews[0].fileUrl } : {}),
...(httpPreview ? { http_preview_url: httpPreview } : {}),
}],
};
audit.setOutput(auditOutput);
await audit.write();
}
catch (e) {
log(`[mask] audit logging error: ${String(e)}`);
}
// Assemble tool result
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw === undefined
? true
: envPreviewRaw === "1" || envPreviewRaw.toLowerCase() === "true";
const summary = {
tool: "mask",
requestId: effectiveRequestId,
i: baseImageI,
originalWidth,
originalHeight,
crop: {
left: { pct: left, px: Math.round(left / 100 * originalWidth) },
right: { pct: right, px: Math.round(right / 100 * originalWidth) },
top: { pct: top, px: Math.round(top / 100 * originalHeight) },
bottom: { pct: bottom, px: Math.round(bottom / 100 * originalHeight) },
},
images_generated: 1,
files: {
original: saved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
};
const { files: _files, ...summaryNoFiles } = summary;
const fallbackPreviewUrl = previews[0]?.fileUrl || saved.fileUrl;
const variantLinksText = `Preview i${baseImageI}: ${httpPreview ? httpPreview : fallbackPreviewUrl}`;
const originalLinksText = `Original i${baseImageI}: ${httpOriginal ? httpOriginal : saved.fileUrl}`;
const reviewHint = "Carefully examine the preview and comment on how well the mask region matches your intent.";
if (previewInChat && previews.length > 0) {
const fname = String(previews[0].fileName || "");
return {
content: [
{
type: "image",
fileName: saved.fileName,
mimeType: previews[0].mimeType,
markdown: ``,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
{
type: "text",
text: `Mask Image i${baseImageI} successfully saved.`,
$hint: reviewHint,
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`[mask] error: ${error instanceof Error ? error.message : String(error)}`);
return {
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
isError: true,
};
}
}
// ─────────────────────────────────────────────────────────────────────────────
// handleZoomIn: crop + Draw Things edit + save result
// ─────────────────────────────────────────────────────────────────────────────
async function handleZoomIn(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.ZoomInToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid zoom-in parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Load canvas buffer and resolve crop metadata from state
let rawBuf;
let resolvedCropMeta;
// When canvas is an iN image, rawBuf IS already the target region — skip imgCropToPng.
let skipImgCrop = false;
// Source image dimensions for the render target (overrides origW/H when set).
let overrideZoomDims;
// Set when detectLabel resolved the crop — post-crop dims needed for render target.
let detectLabelWasUsed = false;
let currentLmChatId = null;
// Hoisted out of the try block below (not just block-scoped) so the final handleGenerateImage()
// call can pass this already-resolved context down instead of forcing a second, TTL-gated
// (60s) re-resolution that can expire during this function's own Restore-Pass/zoom render.
let currentLmWorkingDir = null;
let _zoomAuditSourceOverride = undefined;
try {
// Resolve chat context to find source
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const chatDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas && chatDir) {
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Must run before canvas
// pool resolution below, so the later detectLabel-driven resolveDetectionCrop path picks it up
// instead of silently falling back to the full original frame.
if ((!input.detectLabel || !String(input.detectLabel).trim()) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = autoLabel;
}
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
_zoomAuditSourceOverride = {
sourceKind: "attachment",
sourceOriginAbs: typeof lm.originalPath === "string" ? lm.originalPath : undefined,
sourceFileName: typeof lm.originalPath === "string" ? path_1.default.basename(lm.originalPath) : undefined,
sourceOriginalName: typeof lm.originalName === "string" ? lm.originalName : undefined,
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this attachment
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
const detectSrc = typeof found.detectSource === "string" ? found.detectSource : undefined;
// Mask results store crop coords on the annotated full-source image, not a pre-cropped file.
// For those, rawBuf must be reloaded from cropSource and skipImgCrop must stay false.
let isMaskCanvasResult = false;
if (hasCropFields) {
resolvedCropMeta = {
cropLeft: found.cropLeft,
cropTop: found.cropTop,
cropRight: found.cropRight,
cropBottom: found.cropBottom,
cropSource: found.cropSource,
};
if (typeof found.sourceTool === "string" && found.sourceTool.includes("/mask") &&
typeof found.cropSource === "string") {
isMaskCanvasResult = true;
const _srcRef = parsePrefixedNotation(found.cropSource);
if (_srcRef) {
if (_srcRef.pool === "attachment") {
const _lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: _srcRef.index });
if (_lm?.ok && _lm.buffer)
rawBuf = _lm.buffer;
}
else if (_srcRef.pool === "image") {
const _srcRec = imageRecords.find((r) => r?.i === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename)));
}
else if (_srcRef.pool === "variant") {
const _srcPath = variantPaths.find((v) => v.v === _srcRef.index)?.path;
if (_srcPath)
rawBuf = await fs_1.default.promises.readFile(_srcPath);
}
else {
const _srcRec = pictures.find((p) => p?.p === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename || "")));
}
}
}
}
else if (detectSrc) {
// detect_object result — reload rawBuf from the clean source (not the annotated image).
resolvedCropMeta = { cropLeft: 0, cropTop: 0, cropRight: 0, cropBottom: 0, cropSource: detectSrc };
const _srcRef = parsePrefixedNotation(detectSrc);
if (_srcRef) {
if (_srcRef.pool === "attachment") {
const _lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: _srcRef.index });
if (_lm?.ok && _lm.buffer)
rawBuf = _lm.buffer;
}
else if (_srcRef.pool === "image") {
const _srcRec = imageRecords.find((r) => r?.i === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename)));
}
else if (_srcRef.pool === "variant") {
const _srcPath = variantPaths.find((v) => v.v === _srcRef.index)?.path;
if (_srcPath)
rawBuf = await fs_1.default.promises.readFile(_srcPath);
}
else {
const _srcRec = pictures.find((p) => p?.p === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename || "")));
}
}
}
if (resolvedCropMeta && !isMaskCanvasResult) {
// rawBuf is already the target region (the iN file). Do not re-apply imgCropToPng.
skipImgCrop = true;
// Look up source image dimensions for the render target.
const sourceRef = (hasCropFields
? (typeof found.cropSource === "string" ? found.cropSource : undefined)
: detectSrc);
if (typeof sourceRef === "string") {
const srcPref = parsePrefixedNotation(sourceRef);
if (srcPref?.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcPref.index)?.path;
if (srcPath) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(srcPath));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
else if (srcPref?.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcPref.index);
if (srcRec?.filename) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename))));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
else if (srcPref?.pool === "attachment") {
const attRec = attachments.find((a) => a?.a === srcPref.index);
if (attRec?.width && attRec?.height) {
overrideZoomDims = { w: attRec.width, h: attRec.height };
}
else {
try {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: srcPref.index,
});
if (lm?.ok && lm.buffer) {
const srcSize = await (0, core_bundle_mjs_1.getSize)(lm.buffer);
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
const _oa = typeof lm.originalPath === "string" ? lm.originalPath : undefined;
_zoomAuditSourceOverride = {
sourceKind: "attachment",
sourceOriginAbs: _oa,
sourceFileName: _oa ? path_1.default.basename(_oa) : undefined,
sourceOriginalName: typeof lm.originalName === "string" ? lm.originalName : undefined,
originalDims: { width: srcSize.width, height: srcSize.height, bytes: lm.buffer.byteLength },
};
}
}
catch { }
}
}
else if (srcPref?.pool === "picture") {
const srcRec = pictures.find((p) => p?.p === srcPref.index);
if (srcRec?.filename) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename))));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
}
}
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
_zoomAuditSourceOverride = {
sourceKind: "variant",
sourceOriginAbs: found.path,
sourceFileName: path_1.default.basename(found.path),
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this variant
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
else {
// picture pool (pN)
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
_zoomAuditSourceOverride = {
sourceKind: "picture",
sourceOriginAbs: typeof found.filename === "string" ? path_1.default.join(chatDir, found.filename) : undefined,
sourceFileName: typeof found.filename === "string" ? found.filename : undefined,
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this picture
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
// detectLabel: resolve crop coordinates from a prior detect_object run
if (typeof input.detectLabel === "string" && input.detectLabel.trim()) {
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, input.detectLabel.trim(), typeof input.detectIndex === "number" ? Math.max(0, input.detectIndex) : 0, input.frameAdjust);
if (!detectResult.ok) {
throw new Error(detectResult.error);
}
resolvedCropMeta = {
cropLeft: detectResult.cropLeft,
cropRight: detectResult.cropRight,
cropTop: detectResult.cropTop,
cropBottom: detectResult.cropBottom,
};
skipImgCrop = false;
overrideZoomDims = undefined;
detectLabelWasUsed = true;
// If canvas was the iN detect result, reload rawBuf from the actual source
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
const srcRef = parsePrefixedNotation(detectResult.srcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: srcRef.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Source a${srcRef.index} not found.`);
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (!srcRec)
throw new Error(`Source i${srcRef.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (!srcPath)
throw new Error(`Source v${srcRef.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
// picture
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (!srcRec)
throw new Error(`Source p${srcRef.index} not found.`);
rawBuf = await resolvePictureBuffer(srcRec, chatDir, srcRef.index);
}
}
}
}
}
else {
// Use last attachment as fallback
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: input.canvas ? undefined : undefined,
explicitAttachmentSource: true,
});
if (!lm?.ok || !lm.buffer)
throw new Error("No source image available.");
rawBuf = lm.buffer;
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
// If no crop metadata was found (plain image, no prior crop), default to zero-crop
// (= full image). imgCropToPng with all-zero values is a no-op.
if (!resolvedCropMeta) {
resolvedCropMeta = { cropLeft: 0, cropTop: 0, cropRight: 0, cropBottom: 0 };
}
// Measure rawBuf dimensions (for variant/attachment paths: pre-crop source; for image path: the region itself).
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const origW = canvasSize.width;
const origH = canvasSize.height;
// Complete audit override with pre-crop dims for paths (attachment, variant, picture) that
// could not compute dims inside the try block.
if (_zoomAuditSourceOverride && !_zoomAuditSourceOverride.originalDims) {
_zoomAuditSourceOverride = {
..._zoomAuditSourceOverride,
originalDims: { width: origW, height: origH, bytes: rawBuf.byteLength },
};
}
// When imageFormat is specified and we are about to crop the source image,
// expand the crop region symmetrically so the region already has the target AR.
// This ensures normalizeInputBuffer can cover-resize without clipping the detection region.
// Always expand (never shrink) — the full selected region must remain visible.
if (input.imageFormat && resolvedCropMeta && !skipImgCrop) {
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetAR = ratioMap[input.imageFormat];
if (targetAR !== undefined) {
const cl = resolvedCropMeta.cropLeft ?? 0;
const cr = resolvedCropMeta.cropRight ?? 0;
const ct = resolvedCropMeta.cropTop ?? 0;
const cb = resolvedCropMeta.cropBottom ?? 0;
const regionW = origW * (1 - cl / 100 - cr / 100);
const regionH = origH * (1 - ct / 100 - cb / 100);
const regionAR = regionW / regionH;
if (Math.abs(regionAR - targetAR) > 0.005) {
if (regionAR < targetAR) {
// Region too tall → need to expand width.
// Try symmetric first; if one side hits the image boundary (crop=0),
// transfer the remainder to the opposite side.
const totalExtraPct = ((regionH * targetAR - regionW) / origW) * 100;
const halfExtra = totalExtraPct / 2;
const leftReduction = Math.min(cl, halfExtra);
const rightReduction = Math.min(cr, halfExtra + (halfExtra - leftReduction));
const leftReduction2 = Math.min(cl, halfExtra + (halfExtra - rightReduction));
resolvedCropMeta = {
...resolvedCropMeta,
cropLeft: Math.max(0, cl - leftReduction2),
cropRight: Math.max(0, cr - rightReduction),
};
}
else {
// Region too wide → need to expand height.
const totalExtraPct = ((regionW / targetAR - regionH) / origH) * 100;
const halfExtra = totalExtraPct / 2;
const topReduction = Math.min(ct, halfExtra);
const bottomReduction = Math.min(cb, halfExtra + (halfExtra - topReduction));
const topReduction2 = Math.min(ct, halfExtra + (halfExtra - bottomReduction));
resolvedCropMeta = {
...resolvedCropMeta,
cropTop: Math.max(0, ct - topReduction2),
cropBottom: Math.max(0, cb - bottomReduction),
};
}
}
}
}
// Apply frameAdjust to resolvedCropMeta for non-detect, non-skip paths.
// detectLabelWasUsed: frameAdjust was already applied inside resolveDetectionCrop.
// skipImgCrop: rawBuf is already the cropped region; can't re-crop from source here.
if (input.frameAdjust !== undefined && !detectLabelWasUsed && !skipImgCrop) {
const cl = resolvedCropMeta.cropLeft ?? 0;
const cr = resolvedCropMeta.cropRight ?? 0;
const ct = resolvedCropMeta.cropTop ?? 0;
const cb = resolvedCropMeta.cropBottom ?? 0;
const regionW = origW * (1 - cl / 100 - cr / 100);
const regionH = origH * (1 - ct / 100 - cb / 100);
const cropDiag = Math.hypot(regionW, regionH);
let d_px;
if (typeof input.frameAdjust === "string") {
const m = String(input.frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (m) {
const val = parseFloat(m[1]);
d_px = m[2]?.toLowerCase() === "px" ? val : (val / 100) * cropDiag;
}
else {
d_px = 0;
}
}
else {
d_px = (input.frameAdjust / 100) * cropDiag;
}
const dLR = (d_px / origW) * 100;
const dTB = (d_px / origH) * 100;
resolvedCropMeta = {
...resolvedCropMeta,
cropLeft: Math.max(0, cl - dLR),
cropRight: Math.max(0, cr - dLR),
cropTop: Math.max(0, ct - dTB),
cropBottom: Math.max(0, cb - dTB),
};
}
// Apply the crop from persisted metadata only when rawBuf is the full source (variant/attachment).
// For iN canvas (skipImgCrop=true) rawBuf is already the target region.
if (!skipImgCrop) {
rawBuf = await (0, core_bundle_mjs_1.cropToPng)(rawBuf, {
left: resolvedCropMeta.cropLeft ?? 0,
top: resolvedCropMeta.cropTop ?? 0,
right: resolvedCropMeta.cropRight ?? 0,
bottom: resolvedCropMeta.cropBottom ?? 0,
});
}
// ── RESTORE-PASS — edit-class quality pre-stage for small sources ─────────
// Restore is EDIT, not image2image. Must use generateImageEdit.
// If the source is below the restore threshold, run a fast edit pass to
// recover quality before the zoom-in pipeline consumes the buffer.
if (imageService?.name === "drawthings") {
if (typeof imageService.generateImageEdit !== "function") {
// handleZoomIn never warms up the backend itself before this point, so
// imageService may just be unprobed/stale — retry gRPC-only. HTTP is
// never a valid fallback here (it doesn't implement generateImageEdit).
await ensureBackendReady({ requireGrpc: true }).catch((e) => {
log(`[restore] ensureBackendReady(requireGrpc) failed: ${String(e?.message || e)}`);
});
}
if (typeof imageService.generateImageEdit !== "function") {
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
log(`[restore] ERROR: Draw Things gRPC is required for Restore-Pass but is unavailable ` +
`(imageService.name=${imageService?.name}, selectedTransport=${selectedTransport ?? "none"})`);
throw new Error("Restore-Pass requires the Draw Things gRPC backend, which is not reachable. HTTP is not a valid fallback for edit mode.");
}
const rawSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
if (rawSize.width + rawSize.height < core_bundle_mjs_1.drawthingsRestoreLimits.targetSumRestore) {
const restoreOnProgress = onProgress
? (step, total, msg) => onProgress(step, total, step === -1 ? (msg ? `Restore ${msg}` : "Restore") : "Restore")
: undefined;
// Normalize source up to targetSum (1344) using the standard pipeline.
// targetSumRestore is only the firing threshold; targetSum is the actual
// normalize goal. Supply requestedRawW/H that exceed targetSum to force
// normalizeInputBuffer to produce the maximum 64-aligned result at targetSum.
const _restoreRawSum = rawSize.width + rawSize.height;
const _restoreFactor = Math.ceil(core_bundle_mjs_1.drawthingsRestoreLimits.targetSum / _restoreRawSum) + 1;
const restoreNormalized = await normalizeInputBuffer(rawBuf, {
requestedRawW: rawSize.width * _restoreFactor,
requestedRawH: rawSize.height * _restoreFactor,
targetSumOverride: core_bundle_mjs_1.drawthingsRestoreLimits.targetSum,
logPrefix: "[restore]",
});
const restoreAdjW = restoreNormalized.preprocess.adjusted.width;
const restoreAdjH = restoreNormalized.preprocess.adjusted.height;
const restoreNormBuf = restoreNormalized.buf;
// model must NOT be sent — Draw Things must use whatever model is currently loaded.
// input.model is irrelevant here and must not be set.
// Only pass runtime overrides — the service fills baseDefaults itself.
// Pre-spreading defaultParamsRestore would flood `filtered` and override the overlay.
const restoreParams = {
width: restoreAdjW,
height: restoreAdjH,
prompt: input.prompt ?? "",
_dt_i2i_profile: "restore",
_dt_needs_upscaler: false,
};
const restoreStartMs = Date.now();
const restoreResult = await imageService.generateImageEdit(restoreParams, [restoreNormBuf], restoreOnProgress, undefined);
const restoreInferenceMs = Date.now() - restoreStartMs;
if (restoreResult.isError || restoreResult.error) {
throw new Error(`Restore-Pass failed: ${restoreResult.errorMessage ?? restoreResult.error ?? "unknown"}`);
}
let restoreBuffers = [];
if (Array.isArray(restoreResult.images) && restoreResult.images.length > 0) {
for (const img of restoreResult.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
restoreBuffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (Buffer.isBuffer(restoreResult.imageBuffer)) {
restoreBuffers.push(restoreResult.imageBuffer);
}
else if (restoreResult.imageData) {
const d = restoreResult.imageData;
if (Buffer.isBuffer(d))
restoreBuffers.push(d);
else if (typeof d === "string") {
const b64 = d.startsWith("data:") ? d.split(",")[1] : d;
restoreBuffers.push(Buffer.from(b64, "base64"));
}
}
if (restoreBuffers.length === 0)
throw new Error("Restore-Pass returned no image data");
const restoreMeta = restoreResult?.metadata ?? {};
let restoreBackendW;
let restoreBackendH;
try {
const rm = await (0, core_bundle_mjs_1.getSize)(restoreBuffers[0]);
restoreBackendW = rm.width;
restoreBackendH = rm.height;
}
catch { }
rawBuf = restoreBuffers[0];
try {
const auditR = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "drawthings", mode: "edit", requestId: undefined });
if (currentLmChatId)
auditR.setChatId(currentLmChatId);
auditR.setUserRequest({ canvas: input.canvas, prompt: input.prompt ?? "" });
auditR.setRenderTarget({
requested_raw: { width: restoreAdjW, height: restoreAdjH },
requested_effective: { width: restoreAdjW, height: restoreAdjH },
needs_upscaler: false,
});
auditR.setInputs({
canvas: {
notation: input.canvas ?? undefined,
original: {
width: restoreNormalized.preprocess.original.width,
height: restoreNormalized.preprocess.original.height,
bytes: restoreNormalized.preprocess.original.bytes,
},
adjusted: {
width: restoreNormalized.preprocess.adjusted.width,
height: restoreNormalized.preprocess.adjusted.height,
bytes: restoreNormalized.preprocess.adjusted.bytes,
},
},
});
const restoreOutput = { restore_pass: true };
if (restoreBackendW !== undefined && restoreBackendH !== undefined) {
restoreOutput.backend_returned = { width: restoreBackendW, height: restoreBackendH };
}
restoreOutput.inference_time_ms = restoreInferenceMs;
if (typeof restoreMeta.prompt_used === "string") {
restoreOutput.prompt_used = restoreMeta.prompt_used;
}
if (typeof restoreMeta.model === "string" && restoreMeta.model.trim()) {
restoreOutput.model_used = path_1.default.basename(restoreMeta.model);
}
if (typeof restoreMeta.overlay_source === "string" && restoreMeta.overlay_source.trim()) {
restoreOutput.overlay_source = restoreMeta.overlay_source;
}
if (typeof restoreMeta.overlay_preset === "string" && restoreMeta.overlay_preset.trim()) {
restoreOutput.overlay_preset = restoreMeta.overlay_preset;
}
if (typeof restoreMeta.defaults_used === "string" && restoreMeta.defaults_used.trim()) {
restoreOutput.defaults_used = restoreMeta.defaults_used;
}
if (typeof restoreMeta.i2i_profile === "string" && restoreMeta.i2i_profile.trim()) {
restoreOutput.i2i_profile = restoreMeta.i2i_profile;
}
if (typeof restoreMeta.strength_used === "number" && Number.isFinite(restoreMeta.strength_used)) {
restoreOutput.strength_used = restoreMeta.strength_used;
}
if (typeof restoreMeta.steps_used === "number" && Number.isFinite(restoreMeta.steps_used)) {
restoreOutput.steps_used = restoreMeta.steps_used;
}
if (typeof restoreMeta.sampler_used === "string" && restoreMeta.sampler_used.trim()) {
restoreOutput.sampler_used = restoreMeta.sampler_used;
}
if (typeof restoreMeta.guidance_scale_used === "number" && Number.isFinite(restoreMeta.guidance_scale_used)) {
restoreOutput.guidance_scale_used = restoreMeta.guidance_scale_used;
}
if (Array.isArray(restoreMeta.loras_used) && restoreMeta.loras_used.length > 0) {
restoreOutput.loras_used = restoreMeta.loras_used;
}
auditR.setOutput(restoreOutput);
await auditR.write();
}
catch (auditErr) {
log(`[restore] audit write failed: ${String(auditErr)}`);
}
}
}
// ── END RESTORE-PASS ───────────────────────────────────────────────────────
// For the detectLabel path the crop was just applied above: origW/H are the
// pre-crop source dims. Use those as overrideZoomDims (= source resolution
// reference) and derive zoomSrcW/H from the post-crop buffer so that the
// render target scales the detection region AR up to source resolution —
// matching the iN detect path behaviour.
let zoomSrcW = origW;
let zoomSrcH = origH;
if (detectLabelWasUsed && !skipImgCrop) {
const croppedSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
overrideZoomDims = { w: origW, h: origH };
zoomSrcW = croppedSize.width;
zoomSrcH = croppedSize.height;
}
// Derive render_target dimensions.
// When overrideZoomDims is available (iN canvas or detectLabel case), zoomSrcW/H are
// the cropped region dims. Scale the crop to the source resolution while preserving
// crop AR, using the longer source axis as the constraining dim.
// When overrideZoomDims is absent (variant/attachment canvas), zoomSrcW/H are the
// pre-crop source dims and are used directly.
let zoomTargetW;
let zoomTargetH;
if (overrideZoomDims) {
if (zoomSrcW > zoomSrcH) {
// landscape crop: constrain to source width
zoomTargetW = overrideZoomDims.w;
zoomTargetH = Math.round(overrideZoomDims.w * zoomSrcH / zoomSrcW);
}
else {
// portrait or square crop: constrain to source height
zoomTargetH = overrideZoomDims.h;
zoomTargetW = Math.round(overrideZoomDims.h * zoomSrcW / zoomSrcH);
}
}
else {
zoomTargetW = zoomSrcW;
zoomTargetH = zoomSrcH;
}
// Safety floor: if either dimension is below 256, scale both up proportionally
// so the short side reaches 512 — preserving aspect ratio.
const minZoomDim = Math.min(zoomTargetW, zoomTargetH);
if (minZoomDim < 256) {
const scale = Math.ceil(256 / minZoomDim);
zoomTargetW = Math.round(zoomTargetW * scale);
zoomTargetH = Math.round(zoomTargetH * scale);
}
// Build merged params for handleGenerateImage (image2image mode).
// Explicit user-supplied width/height/imageFormat always override the cropSource-derived dims.
// cropSource-derived zoomTargetW/H are used only when the user provided none of those.
const hasExplicitDims = input.width !== undefined || input.height !== undefined || input.imageFormat !== undefined;
const mergedParams = {
mode: "image2image",
prompt: input.prompt ?? "",
};
if (hasExplicitDims) {
if (input.width !== undefined)
mergedParams.width = input.width;
if (input.height !== undefined)
mergedParams.height = input.height;
if (input.imageFormat !== undefined)
mergedParams.imageFormat = input.imageFormat;
}
else {
mergedParams.width = zoomTargetW;
mergedParams.height = zoomTargetH;
}
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
const zoomHandleProgress = onProgress
? (step, total, msg) => onProgress(step, total, step === -1 ? (msg ? `Zoom ${msg}` : "Zoom") : "Zoom")
: undefined;
const result = await handleGenerateImage(mergedParams, zoomHandleProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:zoom-in",
cropMeta: resolvedCropMeta ? {
left: resolvedCropMeta.cropLeft ?? 0,
top: resolvedCropMeta.cropTop ?? 0,
right: resolvedCropMeta.cropRight ?? 0,
bottom: resolvedCropMeta.cropBottom ?? 0,
cropSource: resolvedCropMeta.cropSource,
} : undefined,
auditSourceOverride: _zoomAuditSourceOverride,
chatId: currentLmChatId ?? undefined,
workingDir: currentLmWorkingDir ?? undefined,
});
// Inject zoom-specific metadata into tool result summary so the agent
// (and user) can see what canvas state was used without checking audit logs.
if (result.content && Array.isArray(result.content) && result.content.length > 0) {
const lastItem = result.content[result.content.length - 1];
if (lastItem && lastItem.type === "text" && typeof lastItem.text === "string") {
try {
const summary = JSON.parse(lastItem.text);
summary.zoom_input = {
canvas_source: { width: origW, height: origH },
render_target: { width: zoomTargetW, height: zoomTargetH },
...(resolvedCropMeta ? { crop: resolvedCropMeta } : {}),
};
lastItem.text = JSON.stringify(summary);
}
catch { /* summary not JSON — leave as-is */ }
}
}
return result;
}
// ─────────────────────────────────────────────────────────────────────────────
// handleInpaint: load canvas, resize to aligned dims, build white-on-black mask
// (white = region to repaint, from cropLeft/cropTop/cropRight/cropBottom),
// call handleGenerateImage in edit mode with sourceTag canvas:inpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleInpaint(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.InpaintToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [{ type: "text", text: `Invalid inpaint parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}` }],
};
}
const input = parsed.data;
let rawBuf;
let resolvedBBoxes = [{ left: 0, top: 0, right: 0, bottom: 0 }];
let chatDir;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (!rawCanvas || !chatDir)
throw new Error("canvas is required for inpaint");
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: pref.index });
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
// Resolve crop coordinates from mask/crop record or detectLabel(s).
let labels = input.detectLabel ?? [];
let indices = input.detectIndex ?? [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0).
if (labels.length === 0) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
labels = [autoLabel];
}
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
if (labels.length > 0) {
const boxes = [];
let resolvedSrcCanvas = undefined;
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] !== undefined ? indices[li] : (indices[0] !== undefined ? indices[0] : 0);
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step after per-box overrides
);
if (!detectResult.ok)
throw new Error(detectResult.error);
// Per-box overrides: scalar = all boxes; array[li] non-null = override; null/absent = keep detect value.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
const tmpInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
const resolved = await applyCropFormat(rawBuf, tmpInput);
if (typeof resolved === "string")
throw new Error(resolved);
boxes.push({ left: resolved.left, top: resolved.top, right: resolved.right, bottom: resolved.bottom });
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
if (resolvedSrcCanvas !== undefined && resolvedSrcCanvas !== detectResult.srcCanvas) {
throw new Error(`Mixed sources — detectLabel entries resolve to different source images ` +
`('${resolvedSrcCanvas}' vs '${detectResult.srcCanvas}'). ` +
`All labels must refer to detections on the same source image.`);
}
resolvedSrcCanvas = detectResult.srcCanvas;
}
}
if (resolvedSrcCanvas) {
const srcRef = parsePrefixedNotation(resolvedSrcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
resolvedBBoxes = boxes;
}
else if (pref && pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (found) {
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
if (hasCropFields) {
// Multi-region: prefer bboxes array; fall back to scalar fields for backward compat.
if (Array.isArray(found.bboxes) && found.bboxes.length > 0) {
resolvedBBoxes = found.bboxes;
}
else {
resolvedBBoxes = [{ left: found.cropLeft ?? 0, top: found.cropTop ?? 0, right: found.cropRight ?? 0, bottom: found.cropBottom ?? 0 }];
}
if (typeof found.cropSource === "string") {
const srcRef = parsePrefixedNotation(found.cropSource);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
}
}
}
}
catch (e) {
return { content: [{ type: "text", text: String(e?.message || e) }], isError: true };
}
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const adjW = canvasSize.width;
const adjH = canvasSize.height;
// Build inpaint mask: white (0xffffffff) inside each BBox, black (0x000000ff) outside.
const Jimp = (await import("jimp")).Jimp;
const mask = new Jimp({ width: adjW, height: adjH, color: 0x000000ff });
for (const bbox of resolvedBBoxes) {
const bx1 = Math.round(bbox.left / 100 * adjW);
const bx2 = Math.round((100 - bbox.right) / 100 * adjW);
const by1 = Math.round(bbox.top / 100 * adjH);
const by2 = Math.round((100 - bbox.bottom) / 100 * adjH);
mask.scan(bx1, by1, Math.max(0, bx2 - bx1), Math.max(0, by2 - by1), (_x, _y, idx) => {
mask.bitmap.data[idx] = 255;
mask.bitmap.data[idx + 1] = 255;
mask.bitmap.data[idx + 2] = 255;
mask.bitmap.data[idx + 3] = 255;
});
}
// DEBUG mask save:
// await fs.promises.writeFile(path.join(logsDir, `debug-mask-${Date.now()}.png`), await mask.getBuffer("image/png"));
mask.blur(2);
const maskBuf = await mask.getBuffer("image/png");
const mergedParams = {
mode: "edit",
prompt: input.prompt ?? "",
width: adjW,
height: adjH,
};
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:inpaint",
maskBuf,
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}
// ─────────────────────────────────────────────────────────────────────────────
// handleOutpaint: load canvas, build black-on-white mask
// (white = region outside image = area to extend), call handleGenerateImage in
// edit mode with sourceTag canvas:outpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleOutpaint(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.OutpaintToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [{ type: "text", text: `Invalid outpaint parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}` }],
};
}
const input = parsed.data;
let rawBuf;
let resolvedBBoxes = [{ left: 0, top: 0, right: 0, bottom: 0 }];
let chatDir;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (!rawCanvas || !chatDir)
throw new Error("canvas is required for outpaint");
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: pref.index });
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
// Resolve crop coordinates from mask/crop record or detectLabel(s).
let labels = input.detectLabel ?? [];
let indices = input.detectIndex ?? [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0).
if (labels.length === 0) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
labels = [autoLabel];
}
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
if (labels.length > 0) {
const boxes = [];
let resolvedSrcCanvas = undefined;
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] !== undefined ? indices[li] : (indices[0] !== undefined ? indices[0] : 0);
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step after per-box overrides
);
if (!detectResult.ok)
throw new Error(detectResult.error);
// Per-box overrides: scalar = all boxes; array[li] non-null = override; null/absent = keep detect value.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
const tmpInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
const resolved = await applyCropFormat(rawBuf, tmpInput);
if (typeof resolved === "string")
throw new Error(resolved);
boxes.push({ left: resolved.left, top: resolved.top, right: resolved.right, bottom: resolved.bottom });
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
if (resolvedSrcCanvas !== undefined && resolvedSrcCanvas !== detectResult.srcCanvas) {
throw new Error(`Mixed sources — detectLabel entries resolve to different source images ` +
`('${resolvedSrcCanvas}' vs '${detectResult.srcCanvas}'). ` +
`All labels must refer to detections on the same source image.`);
}
resolvedSrcCanvas = detectResult.srcCanvas;
}
}
if (resolvedSrcCanvas) {
const srcRef = parsePrefixedNotation(resolvedSrcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
resolvedBBoxes = boxes;
}
else if (pref && pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (found) {
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
if (hasCropFields) {
// Multi-region: prefer bboxes array; fall back to scalar fields for backward compat.
if (Array.isArray(found.bboxes) && found.bboxes.length > 0) {
resolvedBBoxes = found.bboxes;
}
else {
resolvedBBoxes = [{ left: found.cropLeft ?? 0, top: found.cropTop ?? 0, right: found.cropRight ?? 0, bottom: found.cropBottom ?? 0 }];
}
if (typeof found.cropSource === "string") {
const srcRef = parsePrefixedNotation(found.cropSource);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
}
}
}
}
catch (e) {
return { content: [{ type: "text", text: String(e?.message || e) }], isError: true };
}
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const adjW = canvasSize.width;
const adjH = canvasSize.height;
// Build outpaint mask: black (0x000000ff) inside each BBox (preserve), white (0xffffffff) outside (repaint).
const Jimp = (await import("jimp")).Jimp;
const mask = new Jimp({ width: adjW, height: adjH, color: 0xffffffff });
for (const bbox of resolvedBBoxes) {
const bx1 = Math.round(bbox.left / 100 * adjW);
const bx2 = Math.round((100 - bbox.right) / 100 * adjW);
const by1 = Math.round(bbox.top / 100 * adjH);
const by2 = Math.round((100 - bbox.bottom) / 100 * adjH);
mask.scan(bx1, by1, Math.max(0, bx2 - bx1), Math.max(0, by2 - by1), (_x, _y, idx) => {
mask.bitmap.data[idx] = 0;
mask.bitmap.data[idx + 1] = 0;
mask.bitmap.data[idx + 2] = 0;
mask.bitmap.data[idx + 3] = 255;
});
}
// DEBUG mask save:
// await fs.promises.writeFile(path.join(logsDir, `debug-mask-${Date.now()}.png`), await mask.getBuffer("image/png"));
mask.blur(2);
const maskBuf = await mask.getBuffer("image/png");
const mergedParams = {
mode: "edit",
prompt: input.prompt ?? "",
width: adjW,
height: adjH,
};
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:outpaint",
maskBuf,
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}
// handleRefine: image2image with refine overlay, no dimension override
// ─────────────────────────────────────────────────────────────────────────────
async function handleRefine(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.RefineToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid refine parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Load canvas buffer
let rawBuf;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
let chatDir;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas && chatDir) {
const resolvedChatDir = chatDir;
const st = await (0, core_bundle_mjs_1.readState)(resolvedChatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(resolvedChatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
// picture pool (pN)
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
}
else {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: input.canvas ? undefined : undefined,
explicitAttachmentSource: true,
});
if (!lm?.ok || !lm.buffer)
throw new Error("No source image available.");
rawBuf = lm.buffer;
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
const mergedParams = {
mode: "image2image",
model: input.model,
};
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
if (input.width !== undefined)
mergedParams.width = input.width;
if (input.height !== undefined)
mergedParams.height = input.height;
if (input.imageFormat !== undefined)
mergedParams.imageFormat = input.imageFormat;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:refine",
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}
"use strict";
/*
* Core tool handlers for LM Studio Plugin (transport-agnostic)
* This file extracts the logic from LM Studio Plugin tool handlers in src/index.ts
* with minimal changes to keep behavior identical.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ToolSchemas = void 0;
exports.warmupBackendAtStartup = warmupBackendAtStartup;
exports.normalizeInputBuffer = normalizeInputBuffer;
exports.handleGenerateImage = handleGenerateImage;
exports.handleCrop = handleCrop;
exports.handleMask = handleMask;
exports.handleZoomIn = handleZoomIn;
exports.handleInpaint = handleInpaint;
exports.handleOutpaint = handleOutpaint;
exports.handleRefine = handleRefine;
const path_1 = __importDefault(require("path"));
const fs_1 = __importDefault(require("fs"));
const net_1 = __importDefault(require("net"));
const url_1 = require("url");
const core_bundle_mjs_1 = require("../core-bundle.mjs");
const pngMetadata_js_1 = require("../helpers/pngMetadata.js");
const drawThingsService_js_1 = require("../services/drawThingsService.js");
const modelOverlays_js_1 = require("../services/modelOverlays.js");
// Global debug toggle
const DEBUG_MODE = true;
(0, core_bundle_mjs_1.ensureLogsDir)();
const logsDir = (0, core_bundle_mjs_1.getLogsDir)();
try {
const line = `${localTimestamp()} - paths: logsDir=${logsDir}\n`;
fs_1.default.appendFileSync(path_1.default.join(logsDir, (0, core_bundle_mjs_1.getPluginLogFilename)()), line);
}
catch { }
const logFile = path_1.default.join(logsDir, (0, core_bundle_mjs_1.getPluginLogFilename)());
// LM Studio only – no client resolver switching
function resolvePreferredLocale() {
const envPref = process.env.LOG_LOCALE;
const lc = envPref || process.env.LC_ALL || process.env.LC_TIME || process.env.LANG;
if (!lc)
return undefined;
const cleaned = String(lc).split(".")[0].replace(/_/g, "-");
return cleaned || undefined;
}
function localTimestamp() {
const opts = {
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
hour12: false,
timeZoneName: "short",
};
const loc = resolvePreferredLocale();
try {
return new Date().toLocaleString(loc, opts);
}
catch {
const d = new Date();
const day = String(d.getDate()).padStart(2, "0");
const month = String(d.getMonth() + 1).padStart(2, "0");
const year = d.getFullYear();
const hh = String(d.getHours()).padStart(2, "0");
const mm = String(d.getMinutes()).padStart(2, "0");
const ss = String(d.getSeconds()).padStart(2, "0");
return `${day}/${month}/${year} ${hh}:${mm}:${ss}`;
}
}
function log(message) {
const timestamp = localTimestamp();
const line = `${timestamp} - ${message}\n`;
try {
const dir = path_1.default.dirname(logFile);
if (!fs_1.default.existsSync(dir))
fs_1.default.mkdirSync(dir, { recursive: true });
fs_1.default.appendFileSync(logFile, line);
}
catch { }
console.log(line.trim());
}
async function logError(error) {
try {
const errorLogFile = path_1.default.join(logsDir, "error.log");
await fs_1.default.promises.mkdir(logsDir, { recursive: true }).catch(() => { });
const timestamp = localTimestamp();
const details = error instanceof Error
? `${error.message}\n${error.stack}`
: String(error);
const block = `${timestamp} - ERROR:\n${details}\n\n`;
await fs_1.default.promises.appendFile(errorLogFile, block);
if (DEBUG_MODE)
console.error(block);
}
catch { }
}
async function appendErrorRaw(raw, status) {
try {
const errorLogFile = path_1.default.join(logsDir, "error.log");
await fs_1.default.promises.mkdir(logsDir, { recursive: true }).catch(() => { });
const timestamp = localTimestamp();
const header = typeof status === "number"
? `BACKEND RAW (status ${status})`
: "BACKEND RAW";
const block = `${timestamp} - ${header}:\n${raw}\n\n`;
await fs_1.default.promises.appendFile(errorLogFile, block);
}
catch { }
}
function isoStamp() {
return new Date().toISOString().replace(/[:.]/g, "-");
}
// Compact timestamp for filenames, e.g. 20251115T232635722Z
function isoStampCompact() {
const d = new Date();
const year = d.getUTCFullYear();
const month = String(d.getUTCMonth() + 1).padStart(2, "0");
const day = String(d.getUTCDate()).padStart(2, "0");
const hours = String(d.getUTCHours()).padStart(2, "0");
const minutes = String(d.getUTCMinutes()).padStart(2, "0");
const seconds = String(d.getUTCSeconds()).padStart(2, "0");
const millis = String(d.getUTCMilliseconds()).padStart(3, "0");
return `${year}${month}${day}T${hours}${minutes}${seconds}${millis}Z`;
}
function encodeFileUrl(abs) {
return (0, url_1.pathToFileURL)(abs).toString();
}
function stripInternalToolKeys(obj) {
if (!obj || typeof obj !== "object")
return {};
const out = {};
for (const [k, v] of Object.entries(obj)) {
// Prevent user injection of internal/legacy knobs (e.g. _dt_*, _i2i_*).
if (k.startsWith("_"))
continue;
out[k] = v;
}
return out;
}
function isSupportedImageBuffer(buf) {
if (!buf || buf.length < 12)
return false;
if (buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a)
return true; // PNG
if (buf[0] === 0xff && buf[1] === 0xd8)
return true; // JPEG
if (buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50)
return true; // WEBP
return false;
}
function isPng(buf) {
return (buf &&
buf.length >= 8 &&
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a);
}
function normalizeSourceNotation(s) {
let t = String(s || "")
.trim()
.toLowerCase();
if (t === "a")
t = "a1";
if (t === "v")
t = "v1";
if (t === "p")
t = "p1";
if (t === "i")
t = "i1";
return t;
}
function parsePrefixedNotation(s) {
const t = normalizeSourceNotation(s);
const m = t.match(/^([avpi])\s*(\d+)$/);
if (!m)
return null;
const idx = Math.max(1, parseInt(m[2], 10));
const pool = m[1] === "a" ? "attachment" : m[1] === "v" ? "variant" : m[1] === "i" ? "image" : "picture";
return { pool, index: idx };
}
function parseDigitOnlyNotation(s) {
const t = String(s || "").trim();
const m = t.match(/^(\d+)$/);
if (!m)
return null;
return Math.max(1, parseInt(m[1], 10));
}
// A canvas value that isn't aN/vN/pN/iN notation is a scratchpad basename or an absolute path
// (see schemas.ts's SourceNotationOrPath for the accepted shapes) — mirrors generate-image's
// core/tools.ts inline resolution; deliberately local/non-shared, same as there. The MCP layer
// (src/mcp/index.ts) already resolves and containment-checks these against the bound scratchpad
// before this handler ever runs (made-for-bionic-core's resolveMcpSourceToken()) — by the time a
// canvas value reaches here it is either an aN/vN/iN/pN notation or an already-validated absolute
// path, so this is a plain existence check, not a security boundary.
async function resolveCanvasPathToken(rawCanvas, baseDir) {
const trimmed = rawCanvas.trim();
const candidate = path_1.default.isAbsolute(trimmed)
? trimmed
: baseDir && trimmed === path_1.default.basename(trimmed)
? path_1.default.join(baseDir, trimmed)
: null;
if (!candidate)
return null;
const exists = await fs_1.default.promises.stat(candidate).then((s) => s.isFile()).catch(() => false);
return exists ? candidate : null;
}
// Pictures from third-party sources (e.g. find_image) may only carry a `sourceUrl` pointing at an
// external reference, with no locally materialized `filename` in this scratchpad — a Draw Things
// project blob (project://...sqlite3#id, resolvable via resolveProjectUri, same as find_image's own
// tool), a path on an unmounted/external volume, or a source this codebase cannot fetch at all
// (Apple Photos, a page inside a multi-page document). Mirrors find_image's own concept: resolve
// Draw Things project references properly, and otherwise fall back to the locally cached preview
// JPEG rather than failing — a lower-resolution result beats a crash or a bare error.
async function resolvePictureBuffer(found, dir, pIndex) {
if (typeof found?.filename === "string" && found.filename) {
const abs = path_1.default.join(dir, found.filename);
if (await fs_1.default.promises.stat(abs).then((s) => s.isFile()).catch(() => false)) {
return fs_1.default.promises.readFile(abs);
}
}
const sourceUrl = typeof found?.sourceUrl === "string" ? found.sourceUrl : undefined;
if (sourceUrl && (0, core_bundle_mjs_1.isProjectUri)(sourceUrl)) {
const buf = await (0, core_bundle_mjs_1.resolveProjectUri)(sourceUrl).catch(() => null);
if (buf)
return buf;
}
else if (sourceUrl && path_1.default.isAbsolute(sourceUrl)) {
if (await fs_1.default.promises.stat(sourceUrl).then((s) => s.isFile()).catch(() => false)) {
return fs_1.default.promises.readFile(sourceUrl);
}
}
if (typeof found?.preview === "string" && found.preview) {
const previewAbs = path_1.default.join(dir, found.preview);
if (await fs_1.default.promises.stat(previewAbs).then((s) => s.isFile()).catch(() => false)) {
log(`[picture] p${pIndex}: original unavailable${sourceUrl ? ` (sourceUrl: ${sourceUrl})` : ""} — falling back to preview ${previewAbs}`);
return fs_1.default.promises.readFile(previewAbs);
}
}
const hint = sourceUrl ? ` (sourceUrl: ${sourceUrl})` : "";
throw new Error(`Picture p${pIndex} has no local file and no preview in this scratchpad${hint}. Re-run the tool that produced it so it materializes a local copy, or use a different canvas.`);
}
async function saveOriginalPng(sourceBuffer, preferredDir, preferredFileName, xmpParams) {
if (!preferredDir) {
throw new Error("No output directory resolved (LM Studio chat working directory missing).");
}
const dir = path_1.default.resolve(preferredDir);
await fs_1.default.promises.mkdir(dir, { recursive: true });
const name = preferredFileName && preferredFileName.endsWith(".png")
? preferredFileName
: preferredFileName
? `${preferredFileName}.png`
: `generated-image-${isoStampCompact()}.png`;
const abs = path_1.default.join(dir, name);
const embedMeta = xmpParams != null &&
/^(1|true|yes)$/i.test(String(process.env.EMBED_PNG_METADATA ?? "true").trim());
try {
if (!isSupportedImageBuffer(sourceBuffer)) {
const binName = name.replace(/\.png$/i, ".bin");
const binAbs = path_1.default.join(dir, binName);
await fs_1.default.promises.writeFile(binAbs, sourceBuffer);
const stat = await fs_1.default.promises.stat(binAbs);
return {
savedPath: binAbs,
fileName: binName,
size: stat.size,
mimeType: "application/octet-stream",
fileUrl: encodeFileUrl(binAbs),
};
}
let pngBuf;
if (isPng(sourceBuffer)) {
pngBuf = sourceBuffer;
}
else {
pngBuf = await (0, core_bundle_mjs_1.toPng)(sourceBuffer);
}
if (embedMeta) {
pngBuf = (0, pngMetadata_js_1.injectXmpIntoBuffer)(pngBuf, xmpParams);
}
await fs_1.default.promises.writeFile(abs, pngBuf);
}
catch {
await fs_1.default.promises.writeFile(abs, sourceBuffer);
}
const stat = await fs_1.default.promises.stat(abs);
return {
savedPath: abs,
fileName: name,
size: stat.size,
mimeType: "image/png",
fileUrl: encodeFileUrl(abs),
};
}
// Legacy TinyPreviewOptions and buildAndSavePreview removed.
// Use generatePreviewFromBuffer() from media-promotion-core/image.js instead.
let lastPreviewRef = null;
let lastOriginalRef = null;
// Per-chat image tracking (LM Studio) - stores i-value and path for proper lookup
const LAST_IMAGES_BY_LM_CHAT = {};
// PHASE 4: Sticky mode removed - mode is now effectively required when sources exist
const LAST_CONSUMED_ATTACHMENT_ID_BY_LM_CHAT = {};
/**
* Get current connection settings from process.env (set by toolsProvider)
* Defaults to hardcoded settings if env vars are not set.
*/
function getCurrentConnectionSettings() {
return (0, core_bundle_mjs_1.getEngineConnectionDefaults)({
host: process.env.DRAW_THINGS_HOST,
httpPort: process.env.DRAW_THINGS_HTTP_PORT
? parseInt(process.env.DRAW_THINGS_HTTP_PORT, 10)
: undefined,
grpcPort: process.env.DRAW_THINGS_GRPC_PORT
? parseInt(process.env.DRAW_THINGS_GRPC_PORT, 10)
: undefined,
});
}
// Backend service: Draw Things only
const drawthingsService = new drawThingsService_js_1.DrawThingsService(core_bundle_mjs_1.engineConnectionDefaults.http?.baseUrl || "http://127.0.0.1:7860", core_bundle_mjs_1.engineConnectionDefaults.sharedSecret || undefined);
let imageService = drawthingsService;
async function ensureBackendReady(opts) {
// Draw Things backend only - always proceed with connection check
// Use config-aware connection settings (reads from process.env)
const conn = getCurrentConnectionSettings();
const httpBaseUrl = conn.http?.baseUrl ||
`http://${conn.http?.host || "127.0.0.1"}:${conn.http?.port || 7860}`;
const resolvedGrpc = (conn.grpc?.target || `127.0.0.1:7859`).replace(/^grpc:\/\//i, "");
const [host, portStr] = (() => {
const lastColon = resolvedGrpc.lastIndexOf(":");
if (lastColon > -1)
return [
resolvedGrpc.slice(0, lastColon),
resolvedGrpc.slice(lastColon + 1),
];
return [resolvedGrpc, String(7859)];
})();
const portNum = parseInt(portStr, 10);
const tcpReachable = (h, p, timeoutMs) => new Promise((resolve) => {
try {
const socket = net_1.default.connect({ host: h, port: p });
const onOk = () => {
cleanup();
resolve(true);
};
const onErr = () => {
cleanup();
resolve(false);
};
const timer = setTimeout(() => onErr(), timeoutMs);
const cleanup = () => {
try {
clearTimeout(timer);
}
catch { }
try {
socket.destroy();
}
catch { }
};
socket.once("connect", onOk);
socket.once("error", onErr);
}
catch {
resolve(false);
}
});
// requireGrpc=true (edit-mode-only callers, e.g. zoom-in's Restore-Pass):
// HTTP can never serve generateImageEdit, so don't even probe/fall back to it.
const transport = opts?.requireGrpc === true ? "grpc" : (conn.transport || "auto");
const wantGrpc = transport === "grpc" || transport === "auto";
const wantHttp = transport === "http" || transport === "auto";
const grpcOk = wantGrpc ? await tcpReachable(host, portNum, 1200) : false;
// probe HTTP only when desired
const httpProbeHost = (() => {
try {
const u = new URL(httpBaseUrl);
return u.hostname || "127.0.0.1";
}
catch {
return "127.0.0.1";
}
})();
const httpProbePort = (() => {
try {
const u = new URL(httpBaseUrl);
return Number(u.port) || 7860;
}
catch {
return 7860;
}
})();
const httpOk = wantHttp
? await tcpReachable(httpProbeHost, httpProbePort, 1200)
: false;
const httpDesc = (() => {
try {
const u = new URL(httpBaseUrl);
return `${u.protocol}//${u.hostname}:${u.port || 80}`;
}
catch {
return httpBaseUrl;
}
})();
log([
"Attempting to connect to Draw Things API at:",
` grpc://${host}:${portNum} - ${grpcOk ? "OK" : "UNAVAILABLE"}`,
` ${httpDesc} - ${httpOk ? "OK" : "UNAVAILABLE"}`,
"",
"Starting service...",
"",
].join("\n"));
let usedTransport = null;
if (grpcOk) {
// Map defaults to expected gRPC envs for downstream service compatibility
try {
if (conn.grpc?.target)
process.env.DRAWTHINGS_GRPC_TARGET = conn.grpc.target;
if (conn.grpc?.service)
process.env.DRAWTHINGS_GRPC_SERVICE = conn.grpc.service;
if (conn.grpc?.compression)
process.env.DRAWTHINGS_GRPC_COMPRESSION = conn.grpc.compression;
if (conn.grpc?.acceptEncoding)
process.env.DRAWTHINGS_GRPC_ACCEPT_ENCODING = conn.grpc
.acceptEncoding;
if (conn.grpc?.protoPath)
process.env.DRAWTHINGS_GRPC_PROTO = conn.grpc.protoPath;
if (conn.sharedSecret)
process.env.DRAWTHINGS_SHARED_SECRET = conn.sharedSecret;
}
catch { }
try {
const mod = await import("../services/drawThingsGrpcService.js");
const GrpcCtor = mod?.DrawThingsGrpcService;
if (typeof GrpcCtor !== "function")
throw new Error("DrawThingsGrpcService not exported");
const grpcSvc = new GrpcCtor(`${host}:${portNum}`);
const ok = await grpcSvc.checkApiConnection();
if (ok) {
imageService = grpcSvc;
usedTransport = "grpc";
// Startup-only: log if SOLL models/LoRAs exist on the gRPC server.
// Non-blocking by design; it helps diagnose silent fallback behavior.
try {
const client = grpcSvc?.client;
if (client) {
const bn = (s) => {
try {
return path_1.default.basename(String(s || "").trim());
}
catch {
return "";
}
};
const { MODEL_PRESET_TO_CAPABILITY_KEY, selectAutoModel, checkModeSupport, } = await import("../core-bundle.mjs");
const { getModelRequiredFiles } = await import("../services/modelOverlays.js");
const { defaultParams: defaultT2I } = await import("../core-bundle.mjs");
const { defaultParamsImg2Img: defaultI2I } = await import("../core-bundle.mjs");
const { defaultParamsEdit: defaultEdit } = await import("../core-bundle.mjs");
const requiredModels = new Set();
const requiredLoras = new Set();
const optionalLoras = new Set();
// Defaults (used when model preset is "auto" or when no overlay is applied)
if (defaultT2I?.model)
requiredModels.add(bn(defaultT2I.model));
if (defaultI2I?.model)
requiredModels.add(bn(defaultI2I.model));
if (defaultEdit?.model)
requiredModels.add(bn(defaultEdit.model));
// Default LoRAs are treated as optional to avoid hard assumptions.
for (const d of [defaultT2I, defaultI2I, defaultEdit]) {
const ls = Array.isArray(d?.loras)
? d.loras
: [];
for (const l of ls) {
const f = bn(l?.file);
if (f)
optionalLoras.add(f);
}
}
const toolModes = ["text2image", "image2image", "edit", "text2video", "image2video"];
const toOverlayMode = (m) => m === "text2image"
? "txt2img"
: m === "image2image"
? "img2img"
: m === "text2video"
? "txt2vid"
: m === "image2video"
? "img2vid"
: "edit";
const presetKeys = Object.keys(MODEL_PRESET_TO_CAPABILITY_KEY || {});
// Overlay SOLL files (models + LoRAs)
for (const preset of presetKeys) {
for (const tm of toolModes) {
const supported = checkModeSupport(preset, tm);
if (!supported?.supported)
continue;
const files = getModelRequiredFiles(preset, toOverlayMode(tm));
for (const fRaw of files) {
const f = bn(fRaw);
if (!f)
continue;
if (/lora/i.test(f))
requiredLoras.add(f);
else
requiredModels.add(f);
}
}
}
// Custom Configs: Not checked at warmup (requires config access via toolsProvider).
// Hard-fail happens per-request in the gRPC backend if model/LoRA is missing.
// Explicit log of auto resolution
const autoMap = toolModes.map((m) => `${m}→${selectAutoModel(m)}`);
log(`[startup] auto preset resolution: ${autoMap.join(", ")}`);
const allToCheck = [
...Array.from(requiredModels),
...Array.from(requiredLoras),
...Array.from(optionalLoras),
];
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const ex = await (0, core_bundle_mjs_1.checkDrawThingsGrpcFilesExist)({
client,
sharedSecret: sharedSecret || undefined,
files: allToCheck,
});
if (!ex.usedFilesExist) {
log("[startup] gRPC asset preflight skipped (FilesExist RPC unavailable or failed).");
}
else {
const missingSet = new Set(ex.missing);
const missingModels = Array.from(requiredModels).filter((f) => missingSet.has(f));
const missingReqLoras = Array.from(requiredLoras).filter((f) => missingSet.has(f));
const missingOptLoras = Array.from(optionalLoras).filter((f) => missingSet.has(f));
if (missingModels.length === 0 && missingReqLoras.length === 0) {
log(`[startup] gRPC asset preflight OK: required models=${requiredModels.size}, required LoRAs=${requiredLoras.size}`);
}
else {
if (missingModels.length) {
log(`[startup] gRPC asset preflight MISSING models: ${missingModels.join(", ")}`);
}
if (missingReqLoras.length) {
log(`[startup] gRPC asset preflight MISSING required LoRAs: ${missingReqLoras.join(", ")}`);
}
}
if (missingOptLoras.length) {
log(`[startup] gRPC asset preflight (optional) missing LoRAs: ${missingOptLoras.join(", ")}`);
}
}
}
}
catch (e) {
log(`[startup] gRPC asset preflight warning: ${e?.message || String(e)}`);
}
}
else {
// fall back to HTTP if desired and reachable
if (httpOk && (transport === "auto" || transport === "http")) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
else {
console.error("Draw Things gRPC reachable but not ready; no HTTP fallback available.");
}
}
}
catch (e) {
console.error(`Draw Things gRPC init failed: ${e?.message || String(e)}.`);
// prefer HTTP fallback on init error
if (httpOk && (transport === "auto" || transport === "http")) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
}
}
else if (httpOk) {
imageService = drawthingsService;
imageService.setBaseUrl(httpBaseUrl);
usedTransport = "http";
}
else {
imageService = drawthingsService; // not connected yet
}
globalThis.__DT_SELECTED_TRANSPORT__ = usedTransport;
try {
const isApiConnected = await imageService.checkApiConnection();
const t = globalThis.__DT_SELECTED_TRANSPORT__;
const suffix = t === "grpc" ? " - gRPC" : t === "http" ? " - HTTP" : "";
if (isApiConnected) {
log(`Connected to Draw Things API${suffix}.`);
if (t === "grpc") {
const sec = imageService?.currentSecurity ?? globalThis.__DT_GRPC_TLS_SELECTED__ ?? "unknown";
log(`[gRPC] security: ${sec}`);
}
}
else
log(`Failed to connect to Draw Things API.`);
}
catch { }
}
/**
* Startup warmup entrypoint.
* Invoked from the Tools Provider during plugin initialization so the backend probe
* (and gRPC model/LoRA preflight logging) happens before the first tool call.
*/
async function warmupBackendAtStartup() {
await ensureBackendReady();
}
// Utility: read last audit prompt and mode for context
async function getLastAuditPromptAndMode() {
try {
const p = path_1.default.join(logsDir, "generate-image-plugin.audit.jsonl");
const txt = await fs_1.default.promises.readFile(p, "utf8").catch(() => "");
if (!txt)
return null;
const chunks = txt
.split(/\n\s*\n/g)
.map((s) => s.trim())
.filter((s) => s.length > 0);
for (let i = chunks.length - 1; i >= 0; i--) {
const s = chunks[i];
try {
const obj = JSON.parse(s);
if (obj && typeof obj === "object") {
// Read prompt from output.prompt_used (what was actually used)
const prompt = typeof obj.output?.prompt_used === "string"
? obj.output.prompt_used
: undefined;
const mode = typeof obj.mode === "string" ? obj.mode : undefined;
return { prompt, mode };
}
}
catch { }
}
return null;
}
catch {
return null;
}
}
async function getLMConversationFilePath(chatId) {
try {
const home = (0, core_bundle_mjs_1.findLMStudioHome)();
const convDir = path_1.default.join(home, "conversations");
if (!fs_1.default.existsSync(convDir))
return null;
if (chatId) {
const p = path_1.default.join(convDir, `${chatId}.conversation.json`);
return (await fs_1.default.promises
.stat(p)
.then((s) => (s.isFile() ? p : null))
.catch(() => null));
}
const entries = await fs_1.default.promises
.readdir(convDir)
.catch(() => []);
const convFiles = entries
.filter((f) => f.endsWith(".conversation.json"))
.map((f) => path_1.default.join(convDir, f));
if (convFiles.length === 0)
return null;
const withTimes = (await Promise.all(convFiles.map(async (p) => {
try {
const s = await fs_1.default.promises.stat(p);
return s.isFile() ? { p, t: s.mtimeMs } : null;
}
catch {
return null;
}
}))).filter(Boolean);
if (withTimes.length === 0)
return null;
withTimes.sort((a, b) => b.t - a.t);
return withTimes[0].p;
}
catch {
return null;
}
}
async function getLastVariantGroupFromLMConversation(chatId) {
try {
const convPath = await getLMConversationFilePath(chatId || undefined);
if (!convPath)
return null;
const text = await fs_1.default.promises.readFile(convPath, "utf8");
const byBase = new Map();
const reOrig = /file:\/\/[\S)"']+\/(images|working-directories\/\d+)\/(image-[^\/]*)-i(\d+)\.png/gi;
let m;
while ((m = reOrig.exec(text)) != null) {
try {
const basePlus = m[2];
const iNum = parseInt(m[3], 10);
const urlStr = m[0].match(/file:\/\/[^^\s)"']+/i)?.[0];
if (!urlStr)
continue;
let absPath = null;
try {
absPath = (0, url_1.fileURLToPath)(urlStr);
}
catch {
absPath = null;
}
if (!absPath)
continue;
const g = byBase.get(basePlus) || {
lastIndex: m.index,
variants: new Set(),
originals: new Map(),
};
g.lastIndex = Math.max(g.lastIndex, m.index);
g.variants.add(iNum);
g.originals.set(iNum, absPath);
byBase.set(basePlus, g);
}
catch { }
}
const rePrev = /file:\/\/[\S)"']+\/(images\/previews|working-directories\/\d+)\/(preview-image-[^\/]*)-i(\d+)\.(jpg|jpeg|webp)/gi;
while ((m = rePrev.exec(text)) != null) {
try {
const nameWithPreview = m[2];
const iNum = parseInt(m[3], 10);
const urlStr = m[0]
.replace(/\/images\/previews\//i, "/images/")
.replace(/\/working-directories\/(\d+)\//i, "/working-directories/$1/")
.replace(/preview-/, "")
.replace(/\.(jpg|jpeg|webp)$/i, ".png");
let absPath = null;
try {
const urlOnly = urlStr.match(/file:\/\/[^^\s)"']+/i)?.[0];
absPath = urlOnly ? (0, url_1.fileURLToPath)(urlOnly) : null;
}
catch {
absPath = null;
}
if (!absPath)
continue;
const originalBase = nameWithPreview.replace(/^preview-/, "");
const g = byBase.get(originalBase) || {
lastIndex: m.index,
variants: new Set(),
originals: new Map(),
};
g.lastIndex = Math.max(g.lastIndex, m.index);
g.variants.add(iNum);
g.originals.set(iNum, absPath);
byBase.set(originalBase, g);
}
catch { }
}
if (byBase.size === 0)
return null;
const best = Array.from(byBase.entries())
.map(([base, g]) => ({ base, g }))
.sort((a, b) => b.g.lastIndex - a.g.lastIndex)[0];
if (!best)
return null;
const out = [];
for (let v = 1; v <= 3; v++) {
const p = best.g.originals.get(v);
if (!p)
continue;
const exists = await fs_1.default.promises
.stat(p)
.then((s) => s.isFile())
.catch(() => false);
if (exists)
out.push(p);
}
return out.length > 0 ? out : null;
}
catch {
return null;
}
}
// Decide which event is the most recent in the conversation file.
// Returns:
// - "image" when the latest reference to a generated image is i1 (original or preview)
// - "attachment" when the latest user image attachment appears after the last image
// - null when neither could be detected
async function getLastEventTypeFromLMConversation(chatId) {
try {
const convPath = await getLMConversationFilePath(chatId || undefined);
if (!convPath)
return null;
const text = await fs_1.default.promises.readFile(convPath, "utf8");
let lastVariantIdx = -1;
let m;
const reOrig = /file:\/\/[\S)"']+\/(images|working-directories\/\d+)\/(image-[^\/]*)-i(\d+)\.png/gi;
while ((m = reOrig.exec(text)) != null) {
const iNum = parseInt(m[3], 10);
if (iNum === 1)
lastVariantIdx = Math.max(lastVariantIdx, m.index);
}
const rePrev = /file:\/\/[\S)"']+\/(images\/previews|working-directories\/\d+)\/(preview-image-[^\/]*)-i(\d+)\.(jpg|jpeg|webp)/gi;
while ((m = rePrev.exec(text)) != null) {
const iNum = parseInt(m[3], 10);
if (iNum === 1)
lastVariantIdx = Math.max(lastVariantIdx, m.index);
}
let lastAttachmentIdx = -1;
const reAtt = /"(fileIdentifier|identifier)"\s*:\s*"([^"\n]+\.(png|jpg|jpeg|webp|gif|bmp|tif|tiff|heic))"/gi;
while ((m = reAtt.exec(text)) != null) {
lastAttachmentIdx = Math.max(lastAttachmentIdx, m.index);
}
if (lastVariantIdx < 0 && lastAttachmentIdx < 0)
return null;
return lastVariantIdx > lastAttachmentIdx ? "image" : "attachment";
}
catch {
return null;
}
}
/**
* Unified input normalization for all image sources (Attachments, Variants, Pictures).
* Applies in order:
* 1. Adopt target aspect ratio (if user specified output dimensions)
* 2. Sum constraint (w + h <= targetSum)
* 3. Alignment (multiples of 64)
* 4. Minimum dimension (256px)
* 5. Convert to PNG
*/
async function normalizeInputBuffer(buf, opts) {
const prefix = opts?.logPrefix || "[normalize]";
const size = await (0, core_bundle_mjs_1.getSize)(buf);
let w = size.width || 0;
let h = size.height || 0;
const align = core_bundle_mjs_1.drawthingsLimits.align;
const minDim = core_bundle_mjs_1.drawthingsLimits.minDim;
const targetSum = opts?.targetSumOverride ?? core_bundle_mjs_1.drawthingsLimits.targetSum;
const origW = w;
const origH = h;
const origFmt = isPng(buf) ? "png" : undefined;
const origBytes = buf.byteLength;
let reason = "unchanged";
// 1. Adopt target aspect ratio when both requested dims are given.
// The adjusted image must match the OUTPUT format (e.g. landscape)
// rather than preserving the source image's aspect ratio.
// resizeCoverToPng() then uniformly scales + centre-crops the original
// to fill these dimensions without distortion.
// Subsequent steps (sum clamp, 64-alignment, minDim) refine the size.
const hasReqW = typeof opts?.requestedRawW === "number" &&
Number.isFinite(opts.requestedRawW) &&
opts.requestedRawW > 0;
const hasReqH = typeof opts?.requestedRawH === "number" &&
Number.isFinite(opts.requestedRawH) &&
opts.requestedRawH > 0;
if (hasReqW && hasReqH) {
w = opts.requestedRawW;
h = opts.requestedRawH;
reason = "clamped_to_requested_raw";
log(`${prefix} adopting target dimensions: ${origW}x${origH} → ${w}x${h}`);
}
// 2. Sum constraint: w + h <= targetSum (skipped for zoom profile)
const currentSum = w + h;
if (!opts?.skipSumConstraint && currentSum > targetSum) {
const aspect = w / Math.max(1, h);
const rawH = targetSum / (aspect + 1);
const rawW = aspect * rawH;
// Build candidates from floor/round/ceil on each axis, plus aspect-derived pairs.
// Independent rounding (round×round) can distort the aspect ratio when both
// axes happen to align to different multiples. The candidate selection below
// picks the pair that best preserves the target aspect ratio while satisfying
// the sum constraint and 64-alignment.
const wFloor = Math.max(align, Math.floor(rawW / align) * align);
const wRound = Math.max(align, Math.round(rawW / align) * align);
const wCeil = Math.max(align, Math.ceil(rawW / align) * align);
const hFloor = Math.max(align, Math.floor(rawH / align) * align);
const hRound = Math.max(align, Math.round(rawH / align) * align);
const hCeil = Math.max(align, Math.ceil(rawH / align) * align);
const sumCandidates = [];
for (const cw of [wFloor, wRound, wCeil]) {
for (const ch of [hFloor, hRound, hCeil]) {
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
}
// Aspect-derived pairs: derive one axis from the other
for (const cw of [wFloor, wRound]) {
const ch = Math.max(align, Math.round(cw / aspect / align) * align);
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
for (const ch of [hFloor, hRound]) {
const cw = Math.max(align, Math.round(ch * aspect / align) * align);
if (cw + ch <= targetSum)
sumCandidates.push({ cw, ch });
}
if (sumCandidates.length > 0) {
sumCandidates.sort((a, b) => {
const ae = Math.abs(a.cw / Math.max(1, a.ch) - aspect);
const be = Math.abs(b.cw / Math.max(1, b.ch) - aspect);
if (ae !== be)
return ae - be;
return (Math.abs(a.cw - rawW) + Math.abs(a.ch - rawH)) -
(Math.abs(b.cw - rawW) + Math.abs(b.ch - rawH));
});
w = sumCandidates[0].cw;
h = sumCandidates[0].ch;
}
else {
// Fallback: reduce by align increments (should not happen within normal limits)
let fbW = wFloor;
let fbH = hFloor;
while (fbW + fbH > targetSum && (fbW > align || fbH > align)) {
if (fbW >= fbH)
fbW = Math.max(align, fbW - align);
else
fbH = Math.max(align, fbH - align);
}
w = fbW;
h = fbH;
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 3. Alignment: round to multiples of 64 while preserving aspect ratio.
// Independent floor on each axis would distort the aspect (e.g. 523×697 → 512×640).
// Instead, pick the (floor,round) / (round,floor) / (floor,floor) / (round,round)
// combination that best preserves the original aspect ratio.
if (w % align !== 0 || h % align !== 0) {
const aspect = w / Math.max(1, h);
const wFloor = Math.max(align, Math.floor(w / align) * align);
const wRound = Math.max(align, Math.round(w / align) * align);
const hFloor = Math.max(align, Math.floor(h / align) * align);
const hRound = Math.max(align, Math.round(h / align) * align);
const candidates = [
{ cw: wFloor, ch: hFloor },
{ cw: wFloor, ch: hRound },
{ cw: wRound, ch: hFloor },
{ cw: wRound, ch: hRound },
];
// Also try deriving one axis from the other to hit exact aspect multiples.
for (const wBase of [wFloor, wRound]) {
const hDerived = Math.max(align, Math.round(wBase / aspect / align) * align);
candidates.push({ cw: wBase, ch: hDerived });
}
for (const hBase of [hFloor, hRound]) {
const wDerived = Math.max(align, Math.round(hBase * aspect / align) * align);
candidates.push({ cw: wDerived, ch: hBase });
}
// Filter valid candidates (within targetSum, ≥ align), then pick by lowest aspect error.
const valid = candidates.filter((c) => c.cw >= align && c.ch >= align && c.cw + c.ch <= targetSum);
if (valid.length > 0) {
valid.sort((a, b) => {
const ae = Math.abs(a.cw / Math.max(1, a.ch) - aspect);
const be = Math.abs(b.cw / Math.max(1, b.ch) - aspect);
if (ae !== be)
return ae - be;
// Tie-break: prefer closer to original pixel count.
const ad = Math.abs(a.cw - w) + Math.abs(a.ch - h);
const bd = Math.abs(b.cw - w) + Math.abs(b.ch - h);
return ad - bd;
});
w = valid[0].cw;
h = valid[0].ch;
}
else {
// Hard fallback: independent floor (should not happen within normal limits).
w = Math.max(align, Math.floor(w / align) * align);
h = Math.max(align, Math.floor(h / align) * align);
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 4. Minimum dimension: upscale only if needed to satisfy minDim
if (w < minDim || h < minDim) {
const scale = minDim / Math.min(w, h);
w = Math.round(w * scale);
h = Math.round(h * scale);
// Re-align after upscale
w = Math.max(align, Math.floor(w / align) * align);
h = Math.max(align, Math.floor(h / align) * align);
// Ensure sum constraint still met after minDim upscale
while (w + h > targetSum && (w > minDim || h > minDim)) {
if (w > h)
w = Math.max(minDim, w - align);
else
h = Math.max(minDim, h - align);
}
if (reason === "unchanged")
reason = "normalized_to_constraints";
}
// 5. Resize and/or convert to PNG
let outBuf;
if (w !== origW || h !== origH) {
outBuf = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, w, h);
if (reason === "unchanged")
reason = "normalized_to_constraints";
log(`${prefix} dimension normalization: ${origW}x${origH} → ${w}x${h} (sum=${w + h})`);
}
else {
if (isPng(buf)) {
outBuf = buf;
}
else {
outBuf = await (0, core_bundle_mjs_1.toPng)(buf);
if (reason === "unchanged")
reason = "converted_to_png";
}
}
return {
buf: outBuf,
preprocess: {
original: {
width: origW,
height: origH,
format: origFmt,
bytes: origBytes,
},
adjusted: {
width: w,
height: h,
format: "png",
bytes: outBuf.byteLength,
},
reason,
},
normalizedLongSide: w !== origW || h !== origH ? Math.max(w, h) : undefined,
};
}
const ALLOWED_GEN_INPUT_KEYS = [
"prompt",
"width",
"height",
"imageFormat",
"quality",
"variants",
"canvas",
"moodboard",
];
async function handleGenerateImage(pluginParams, onProgress, _internal) {
await ensureBackendReady().catch((e) => {
log(`[startup] ensureBackendReady failed: ${String(e)}`);
});
try {
const rawIncoming = pluginParams || {};
// zoom/upscale sources may supply width/height up to ZOOM_TOOL_MAX_DIM (2048),
// which exceeds the standard schema limit (1536). Strip them before schema
// validation and re-inject after, so the rest of the pipeline sees them.
const isZoomLikeSource = _internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale";
const rawForValidation = isZoomLikeSource
? (({ width: _w, height: _h, ...rest }) => rest)(rawIncoming)
: rawIncoming;
const parsed = core_bundle_mjs_1.GenerateToolParamsSchemaMinimalStrict.safeParse(rawForValidation);
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid generate_image parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Re-inject zoom/upscale dims after schema validation
if (isZoomLikeSource) {
if (rawIncoming.width !== undefined)
input.width = rawIncoming.width;
if (rawIncoming.height !== undefined)
input.height = rawIncoming.height;
}
// ─────────────────────────────────────────────────────────────────────────
// HARD LIMIT CHECK: Reject requests exceeding maxWidth/maxHeight immediately.
// No silent clamping – explicit error with clear guidance.
// Skipped for zoom-like sources (canvas:zoom-in / canvas:upscale): input.width/height
// represent requestedRaw (the final post-processing target), not the backend input.
// The zoom-pass pipeline handles any requestedRaw size via SeedVR2 + post-processing.
// ─────────────────────────────────────────────────────────────────────────
if (!isZoomLikeSource) {
const reqW = input.width;
const reqH = input.height;
const maxW = core_bundle_mjs_1.drawthingsLimits.maxWidth;
const maxH = core_bundle_mjs_1.drawthingsLimits.maxHeight;
if (typeof reqW === "number" && reqW > maxW) {
log(`[validation] REJECTED: width ${reqW} exceeds maxWidth ${maxW}`);
return {
content: [
{
type: "text",
text: `Invalid width: ${reqW}px exceeds maximum allowed width of ${maxW}px. Please use width ≤ ${maxW}.`,
},
],
isError: true,
};
}
if (typeof reqH === "number" && reqH > maxH) {
log(`[validation] REJECTED: height ${reqH} exceeds maxHeight ${maxH}`);
return {
content: [
{
type: "text",
text: `Invalid height: ${reqH}px exceeds maximum allowed height of ${maxH}px. Please use height ≤ ${maxH}.`,
},
],
isError: true,
};
}
}
// Preserve the user-requested mode for logging/audit.
// "edit" is a variant of image2image with different defaults and (future) multi-source support.
const requestedMode = input.mode;
const isEditMode = requestedMode === "edit";
log(`generate_image input: ${JSON.stringify(input)}`);
try {
void (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
}
catch { }
// Validate model/mode compatibility early
const modelPreset = input.model || "auto";
const modeForValidation = requestedMode || "text2image"; // default mode if not specified
// Import capability check and custom configs (dynamic to avoid circular deps at module load)
const { checkModeSupport, checkModeSupportWithCustom, selectAutoModel, detectImageModelCapabilities, getCapabilityKeyForPreset, } = await import("../core-bundle.mjs");
const { getAvailableCustomCombinations, getCustomPreset } = await import("../services/customConfigsLoader.js");
// Use extended check that includes Custom Configs info
const modeCheck = checkModeSupportWithCustom(modelPreset, modeForValidation, getAvailableCustomCombinations);
if (!modeCheck.supported) {
log(`[validation] mode/model incompatible: ${modeCheck.reason}`);
return {
content: [
{
type: "text",
text: modeCheck.reason,
},
],
isError: true,
};
}
// NOTE: "model=auto" means: do not apply an overlay; backend uses mode-specific defaults.
// We still compute an "effectiveModelPreset" for capability logic (e.g. edit-mode limits),
// but logging should reflect the engine model that will actually be used.
const effectiveModelPreset = modelPreset === "auto" ? selectAutoModel(modeForValidation) : modelPreset;
// Resolve actual .ckpt filename for logging
const modeForFilename = modeForValidation === "edit"
? "edit"
: modeForValidation === "image2image"
? "img2img"
: modeForValidation === "text2video"
? "txt2vid"
: modeForValidation === "image2video"
? "img2vid"
: "txt2img";
// If the user did not pick a model (or explicitly picked "auto"), the backend will use
// the per-mode defaultParams*.model value (no overlay). Log that to avoid confusion.
let engineDefaultModel = null;
if (modelPreset === "auto") {
try {
if (modeForFilename === "txt2img") {
const { defaultParams } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParams?.model === "string"
? defaultParams.model
: null;
}
else if (modeForFilename === "img2img") {
const { defaultParamsImg2Img } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParamsImg2Img?.model === "string"
? defaultParamsImg2Img.model
: null;
}
else if (modeForFilename === "txt2vid") {
engineDefaultModel = null;
}
else if (modeForFilename === "img2vid") {
engineDefaultModel = null;
}
else {
const { defaultParamsEdit } = await import("../core-bundle.mjs");
engineDefaultModel =
typeof defaultParamsEdit?.model === "string"
? defaultParamsEdit.model
: null;
}
}
catch { }
}
const customPresetModel = (() => {
if (!(0, modelOverlays_js_1.getModelFilename)(effectiveModelPreset, modeForFilename)) {
const raw = getCustomPreset(`${modeForValidation}.${effectiveModelPreset}`)?.params?.model;
if (typeof raw === "string" && raw.trim())
return path_1.default.basename(raw.trim());
}
return null;
})();
const effectiveModelFilename = (engineDefaultModel ? path_1.default.basename(engineDefaultModel) : null) ||
(0, modelOverlays_js_1.getModelFilename)(effectiveModelPreset, modeForFilename) ||
customPresetModel ||
effectiveModelPreset;
log(`[validation] model=${modelPreset} → engineModel=${effectiveModelFilename} mode=${modeForValidation}`);
// Draw Things backend only
const svc = imageService;
const resolvedName = "drawthings";
const requestedVariants = input.variants;
const usedVariants = typeof requestedVariants === "number"
? Math.max(1, Math.min(4, Math.round(requestedVariants)))
: 1;
if (typeof requestedVariants === "number" &&
requestedVariants !== usedVariants) {
log(`variants: requested=${requestedVariants} used=${usedVariants}`);
}
log(`generate_image: using backend='${resolvedName}'`);
const mode = input.mode ||
"text2image";
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas !== undefined && requestedMode === undefined) {
return {
content: [
{
type: "text",
text: "A reference image (`canvas`) was provided but `mode` was not specified. Please re-run and set `mode` to one of: `image2image` (use the reference as a base), `edit` (edit / inpaint the reference), or `image2video` (animate the reference image).",
},
],
isError: true,
};
}
const rawMoodboard = Array.isArray(input.moodboard)
? input.moodboard
: undefined;
const moodboardNotations = (rawMoodboard || [])
.filter((x) => typeof x === "string")
.map((x) => String(x));
let result;
let effectiveMode = "text2image";
let sourceTag = null;
let sourceVariantUsed = undefined;
let sourceKind = undefined;
let sourceOriginAbs = undefined;
let sourceOriginalName = undefined; // Real original filename (e.g., "Katze.png")
let hasFreshAttachment = false;
let isAttachmentSource = false;
// Track reference metadata for summary (used in edit mode multi-reference)
let usedReferenceMeta = [];
// Track per-reference preprocessing (normalization) metadata for audit
let usedReferencePreprocess = [];
let currentLmChatId = null;
let currentLmWorkingDir = null;
let lmResolverConfidence = undefined;
let lmResolverReason = undefined;
let stickyScope = "none";
let sourceFileName = undefined;
let lmCrosscheckInfo = undefined;
let sourcePreprocess;
let normalizedToLongSide = undefined;
// requestedRaw: what the user asked for (or the original source size when user omitted).
// requestedEffective: the internally used aligned/clamped size (multiples of 64).
// Backend may still run at a different internal size (e.g. i2i normalization); we track that separately.
let requestedRawW = undefined;
let requestedRawH = undefined;
let requestedEffectiveW = undefined;
let requestedEffectiveH = undefined;
let _dtNeedsUpscaler = false; // propagated from i2i block; read in post-processing
let zoomPassRan = false; // set to true when zoom-pass pipeline completes
let pendingAudit2 = null;
let auditMaskBuf = undefined; // post-resize mask PNG; read by audit block
const auditRequestId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
if (resolvedName === "drawthings") {
let srcBuf;
// NOTE: chatWdForContext is computed dynamically to ensure it uses
// the latest currentLmChatId/currentLmWorkingDir after resolution.
const getChatWdForContext = () => currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : null);
// V2: Returns all images with stable i-index for proper lookup
const getAllImagesForContext = async () => {
const chatWdForContext = getChatWdForContext();
// Primary: chat working directory state file (contains ALL images, not just latest generation)
try {
if (chatWdForContext) {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForContext);
if (st && Array.isArray(st.images) && st.images.length > 0) {
const images = [...st.images]
.filter((v) => v && typeof v.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0));
const result = images.map((v) => ({
i: v.i || 1,
path: path_1.default.join(chatWdForContext, v.filename),
}));
if (result.length > 0)
return result;
}
}
}
catch { }
// Fallback: in-process memory (only contains latest generation, not all images)
// NOTE: This is less reliable than state file but useful before state is written
try {
if (currentLmChatId) {
const mem = LAST_IMAGES_BY_LM_CHAT[currentLmChatId];
if (Array.isArray(mem) && mem.length > 0)
return mem;
}
}
catch { }
// Fallback: legacy heuristic from SSOT conversation file (returns paths only, assume i=1..n)
try {
const group = await getLastVariantGroupFromLMConversation(currentLmChatId || undefined);
if (Array.isArray(group) && group.length > 0) {
return group.map((p, i) => ({ i: i + 1, path: p }));
}
}
catch { }
return null;
};
// Try to resolve current LM Studio chat for scoping
if (_internal?.chatId) {
// A delegating caller (zoom-in/inpaint/outpaint/refine) already resolved its own chat
// context before calling us — reuse it verbatim instead of re-deriving it here. Re-deriving
// would redo a TTL-gated (60s, see activeChatContext.ts) lookup that can expire mid-pipeline
// for slow, multi-stage renders (Restore-Pass + SeedVR2, model loads, ...) and silently fall
// back to the wrong working directory (~/.lmstudio/working-directories/<chatId> instead of
// the actual bound scratchpad) even when a valid chatId is still found.
currentLmChatId = _internal.chatId;
currentLmWorkingDir = _internal.workingDir ?? null;
lmResolverConfidence = "high";
lmResolverReason = "delegated_context";
log(`[chatId] resolved via delegated_context: chatId=${currentLmChatId} workingDir=${currentLmWorkingDir || "null"}`);
}
else {
try {
// Prefer the generator-provided context (deterministic within the current turn/tool-call loop)
const active = (0, core_bundle_mjs_1.getActiveChatContext)();
if (active &&
typeof active.chatId === "string" &&
/^\d+$/.test(active.chatId)) {
currentLmChatId = active.chatId;
currentLmWorkingDir = active.workingDir;
lmResolverConfidence = "high";
lmResolverReason = `active_context${active.requestId ? `:${active.requestId}` : ""}`;
}
}
catch { }
try {
if (currentLmChatId) {
// already resolved via active context
log(`[chatId] resolved via active_context: chatId=${currentLmChatId} workingDir=${currentLmWorkingDir || "null"}`);
}
else {
// Fallback: filesystem heuristic (picks newest *.conversation.json)
console.warn("[generate_image] No deterministic chat context available – falling back to filesystem heuristic (newest conversation file). " +
"This may happen if the tool is called outside a normal Generator turn or if context TTL (60s) expired.");
const lm = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)({
requireRecentMtimeSec: 600,
});
log(`[chatId] fallback heuristic result: ok=${lm?.ok} chatId=${lm?.chatId || "null"} reason=${lm?.reason || "unknown"}`);
if (lm?.ok) {
currentLmChatId = lm.chatId;
lmResolverConfidence = lm.confidence;
lmResolverReason = lm.reason;
}
}
}
catch { }
}
// ========================================================================
// PHASE 3: Global error rules (before source resolution)
// ========================================================================
// Count available sources from state
let stateAttachmentCount = 0;
let stateImageCount = 0;
let statePictureCount = 0;
const chatWdForValidation = getChatWdForContext();
if (chatWdForValidation) {
try {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForValidation);
stateAttachmentCount = Array.isArray(st.attachments)
? st.attachments.length
: 0;
stateImageCount = Array.isArray(st.images)
? st.images.length
: 0;
statePictureCount = Array.isArray(st.pictures)
? st.pictures.length
: 0;
}
catch (e) {
log(`[phase3] failed to read state for source counts: ${String(e)}`);
}
}
const totalSourcesInState = stateAttachmentCount + stateImageCount + statePictureCount;
log(`[phase3] sources in state: attachments=${stateAttachmentCount}, images=${stateImageCount}, pictures=${statePictureCount}, total=${totalSourcesInState}`);
const loadSourceState = async () => {
const chatWd = getChatWdForContext();
let attachments = [];
let pictures = [];
let variants = [];
try {
if (chatWd) {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
attachments = Array.isArray(st?.attachments) ? st.attachments : [];
pictures = Array.isArray(st?.pictures) ? st.pictures : [];
if (Array.isArray(st?.variants)) {
variants = st.variants
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({
v: typeof v.v === "number" ? v.v : 1,
path: path_1.default.join(chatWd, v.filename),
}));
}
}
}
catch (e) {
log(`[state] failed to read attachments/pictures/variants: ${String(e)}`);
}
let images = [];
try {
images = (await getAllImagesForContext()) || [];
}
catch (e) {
log(`[state] failed to enumerate images: ${String(e)}`);
}
return { chatWd, attachments, pictures, images, variants };
};
const resolveNotation = (notationRaw, ctx) => {
const pref = parsePrefixedNotation(notationRaw);
if (pref) {
const n = `${pref.pool === "attachment"
? "a"
: pref.pool === "variant"
? "v"
: pref.pool === "image"
? "i"
: "p"}${pref.index}`;
return { pool: pref.pool, index: pref.index, notation: n };
}
const digit = parseDigitOnlyNotation(notationRaw);
if (digit != null) {
const pools = [];
if (ctx.attachments.length > 0)
pools.push("attachment");
if (ctx.variants.length > 0)
pools.push("variant");
if (ctx.images.length > 0)
pools.push("image");
if (ctx.pictures.length > 0)
pools.push("picture");
if (pools.length === 0) {
throw new Error("No sources available");
}
if (pools.length > 1) {
const abbrev = pools
.map((p) => p === "attachment" ? "a" : p === "variant" ? "v" : p === "image" ? "i" : "p")
.join("/");
throw new Error(`Ambiguous: use prefix (${abbrev})`);
}
const only = pools[0];
const n = `${only === "attachment" ? "a" : only === "variant" ? "v" : only === "image" ? "i" : "p"}${digit}`;
return { pool: only, index: digit, notation: n };
}
throw new Error(`Invalid source notation: ${String(notationRaw || "").trim()}`);
};
const { chatWd, attachments, pictures, images, variants } = await loadSourceState();
const totalSources = attachments.length + variants.length + images.length + pictures.length;
const autoSelectSingleSource = () => {
if (attachments.length === 1 &&
variants.length === 0 &&
images.length === 0 &&
pictures.length === 0) {
const a = typeof attachments[0]?.a === "number" ? attachments[0].a : 1;
return { pool: "attachment", index: a, notation: `a${a}` };
}
if (variants.length === 1 &&
attachments.length === 0 &&
images.length === 0 &&
pictures.length === 0) {
const v = typeof variants[0]?.v === "number" ? variants[0].v : 1;
return { pool: "variant", index: v, notation: `v${v}` };
}
if (images.length === 1 &&
attachments.length === 0 &&
variants.length === 0 &&
pictures.length === 0) {
const i = typeof images[0]?.i === "number" ? images[0].i : 1;
return { pool: "image", index: i, notation: `i${i}` };
}
if (pictures.length === 1 &&
attachments.length === 0 &&
variants.length === 0 &&
images.length === 0) {
const p = typeof pictures[0]?.p === "number" ? pictures[0].p : 1;
return { pool: "picture", index: p, notation: `p${p}` };
}
throw new Error("Ambiguous source – specify canvas explicitly");
};
let resolvedCanvas = null;
if (mode === "text2image" || mode === "text2video") {
if (rawCanvas) {
log(`[info] canvas ignored for mode='${mode}': ${rawCanvas}`);
}
}
else {
if (rawCanvas) {
resolvedCanvas = resolveNotation(rawCanvas, {
attachments,
variants,
pictures,
images,
});
}
else {
if (totalSources === 0) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
if (totalSources === 1) {
resolvedCanvas = autoSelectSingleSource();
log(`[canvas] auto-resolved to ${resolvedCanvas.notation}`);
}
else {
return {
content: [
{
type: "text",
text: "Ambiguous source – specify canvas explicitly (e.g., canvas='a1' or canvas='v1' or canvas='p1').",
},
],
isError: true,
};
}
}
}
const loadBufferForSel = async (sel) => {
if (sel.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: sel.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${sel.index} not found.`);
}
return {
buf: lm.buffer,
originPath: typeof lm.originalPath === "string"
? lm.originalPath
: undefined,
originalName: typeof lm.originalName === "string"
? lm.originalName
: undefined,
};
}
if (sel.pool === "variant") {
const found = variants.find((v) => v.v === sel.index);
if (!found) {
const available = variants.map((v) => `v${v.v}`).join(", ") || "(none)";
throw new Error(`Variant v${sel.index} not found. Available: ${available}`);
}
return {
buf: await fs_1.default.promises.readFile(found.path),
originPath: found.path,
};
}
if (sel.pool === "image") {
const found = images.find((img) => img.i === sel.index);
if (!found) {
const available = images.map((img) => `i${img.i}`).join(", ") || "(none)";
throw new Error(`Image i${sel.index} not found. Available: ${available}`);
}
return {
buf: await fs_1.default.promises.readFile(found.path),
originPath: found.path,
};
}
// picture
const found = pictures.find((p) => typeof p?.p === "number" && p.p === sel.index);
if (!found) {
const available = pictures
.map((p) => (typeof p?.p === "number" ? `p${p.p}` : null))
.filter(Boolean)
.join(", ") || "(none)";
throw new Error(`Picture p${sel.index} not found. Available: ${available}`);
}
if (!chatWd)
throw new Error("No working directory resolved for pictures.");
const abs = path_1.default.join(chatWd, String(found.filename || ""));
const exists = await fs_1.default.promises
.stat(abs)
.then((s) => s.isFile())
.catch(() => false);
if (!exists) {
throw new Error(`Picture file missing: ${abs}`);
}
return { buf: await fs_1.default.promises.readFile(abs), originPath: abs };
};
// Resolve canvas buffer for image2image/edit/image2video
if (mode === "image2image" || mode === "edit" || mode === "image2video") {
if (!resolvedCanvas) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
// Set unconditionally, BEFORE the fallible lookup below: effectiveMode drives which
// top-level render branch runs (image2image/edit vs. plain text2image). It must never
// depend on whether the redundant loadBufferForSel() re-lookup below succeeds — a caller
// with a presuppliedSourceBuf (zoom-in/inpaint/outpaint/refine) must stay on the
// image2image/edit render path even if that lookup fails, or its already-loaded real
// source buffer gets silently ignored in favor of a pure-prompt (hallucinated) render.
effectiveMode = mode;
try {
const loaded = await loadBufferForSel(resolvedCanvas);
srcBuf = loaded.buf;
sourceTag = `canvas:${resolvedCanvas.notation}`;
sourceKind = resolvedCanvas.pool;
sourceOriginAbs = loaded.originPath;
sourceOriginalName = loaded.originalName;
sourceFileName = loaded.originPath
? path_1.default.basename(loaded.originPath)
: undefined;
if (resolvedCanvas.pool === "image") {
sourceVariantUsed = resolvedCanvas.index;
}
// Persist last canvas selection so the orchestrator can do smarter vision promotion.
try {
if (chatWd) {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
st.lastCanvasNotation = resolvedCanvas.notation;
st.lastCanvasAt = localTimestamp();
await (0, core_bundle_mjs_1.writeStateAtomic)(chatWd, st);
}
}
catch (e) {
log(`[state] failed to persist lastCanvasNotation: ${String(e)}`);
}
}
catch (e) {
// Callers with a presupplied buffer (zoom-in/inpaint/outpaint/refine) already loaded
// the source themselves — this lookup here only re-derives audit metadata (sourceKind/
// sourceFileName/...) from a SEPARATE, independently-resolved chatWd/state read, and its
// failure must not discard an already-valid buffer the caller is about to supply anyway.
if (!_internal?.presuppliedSourceBuf) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
log(`[canvas] resolution failed but a presupplied buffer is available — continuing without audit metadata: ${String(e?.message || e)}`);
}
}
// _internal override: pre-cropped buffer bypasses canvas loading
if (_internal?.presuppliedSourceBuf) {
srcBuf = _internal.presuppliedSourceBuf;
if (_internal.sourceTag)
sourceTag = _internal.sourceTag;
}
// Audit source override: lets callers (e.g. handleZoomIn) report the
// original pre-crop source instead of the presupplied post-processed buffer.
if (_internal?.auditSourceOverride) {
const aso = _internal.auditSourceOverride;
if (aso.sourceKind !== undefined)
sourceKind = aso.sourceKind;
if (aso.sourceFileName !== undefined)
sourceFileName = aso.sourceFileName;
if (aso.sourceOriginalName !== undefined)
sourceOriginalName = aso.sourceOriginalName;
if (aso.sourceOriginAbs !== undefined)
sourceOriginAbs = aso.sourceOriginAbs;
}
// Track mask separately so it can be scaled to match adjusted canvas dims
let effectiveMaskBuf = _internal?.maskBuf;
// Resolve moodboard selections (edit mode, or image2image via gRPC)
// Note: HTTP does not support moodboard for image2image; only gRPC does.
const resolvedMoodboard = [];
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
const moodboardAllowedForI2I = selectedTransport === "grpc";
if ((mode === "edit" || (mode === "image2image" && moodboardAllowedForI2I)) &&
moodboardNotations.length > 0) {
const seen = new Set();
if (resolvedCanvas) {
seen.add(`${resolvedCanvas.pool}:${resolvedCanvas.index}`);
}
for (const nRaw of moodboardNotations) {
const sel = resolveNotation(nRaw, {
attachments,
variants,
pictures,
images,
});
const key = `${sel.pool}:${sel.index}`;
if (seen.has(key))
continue;
seen.add(key);
resolvedMoodboard.push(sel);
}
log(`[${mode}] moodboard resolved: ${resolvedMoodboard.map((s) => s.notation).join(", ") || "(none)"}`);
}
else if (mode === "image2image" && moodboardNotations.length > 0 && !moodboardAllowedForI2I) {
// Warn user that moodboard is ignored for image2image via HTTP
log(`[image2image] WARNING: moodboard ignored - requires gRPC transport. Using single canvas only.`);
}
// Stash for the edit/image2image multi-ref block below
const resolvedCanvasSel = resolvedCanvas;
const resolvedMoodboardSel = resolvedMoodboard;
// Multi-reference mode: edit always, image2image only via gRPC
const isMultiReference = (mode === "edit" || (mode === "image2image" && moodboardAllowedForI2I)) &&
resolvedMoodboardSel.length > 0;
if (mode === "text2image" || mode === "text2video") {
effectiveMode = mode;
log(`effective mode: ${effectiveMode}`);
// If the user provided non-aligned sizes, round to the effective (backend-safe) multiples of 64,
// but keep the raw values for audit + final postprocess resize.
let serviceInputForT2I = stripInternalToolKeys(input);
if (mode === "text2video") {
serviceInputForT2I._dt_video_mode = "txt2vid";
}
try {
if (resolvedName === "drawthings") {
const rawW = input?.width;
const rawH = input?.height;
const hasW = typeof rawW === "number" && Number.isFinite(rawW);
const hasH = typeof rawH === "number" && Number.isFinite(rawH);
if (hasW)
requestedRawW = Math.max(1, Math.round(rawW));
if (hasH)
requestedRawH = Math.max(1, Math.round(rawH));
if (hasW || hasH) {
const align = core_bundle_mjs_1.drawthingsLimits.align;
const maxW = core_bundle_mjs_1.drawthingsLimits.maxWidth;
const maxH = core_bundle_mjs_1.drawthingsLimits.maxHeight;
const floorTo = (v, step) => Math.floor(v / step) * step;
const clamp = (v, min, max) => Math.min(max, Math.max(min, v));
// requestedEffective must be backend-safe: aligned + never exceed render limits.
// Use floor alignment (never rounds up beyond user's raw request).
if (hasW)
requestedEffectiveW = clamp(Math.max(align, floorTo(requestedRawW, align)), align, maxW);
if (hasH)
requestedEffectiveH = clamp(Math.max(align, floorTo(requestedRawH, align)), align, maxH);
if (typeof requestedEffectiveW === "number")
serviceInputForT2I.width = requestedEffectiveW;
if (typeof requestedEffectiveH === "number")
serviceInputForT2I.height = requestedEffectiveH;
// Deterministic contract: when raw dims are known, provide them + upscaler decision.
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
serviceInputForT2I._dt_requested_raw_w = requestedRawW;
serviceInputForT2I._dt_requested_raw_h = requestedRawH;
serviceInputForT2I._dt_needs_upscaler =
requestedRawW > maxW || requestedRawH > maxH;
}
log(`[t2i] requested: raw=${requestedRawW || "-"}x${requestedRawH || "-"} effective=${requestedEffectiveW || "-"}x${requestedEffectiveH || "-"}`);
}
}
}
catch (e) {
log(`[t2i] failed to compute effective size: ${String(e)}`);
}
result = await svc.generateImage(serviceInputForT2I, onProgress);
// Reconstruct render_target from service metadata if not already set
// (handles imageFormat/quality shorthand + custom config scenarios)
if ((typeof requestedRawW !== "number" ||
typeof requestedRawH !== "number") &&
result?.metadata?.requested_dimensions) {
const reqDims = result.metadata.requested_dimensions;
if (typeof reqDims.width === "number" &&
typeof reqDims.height === "number") {
requestedRawW = reqDims.width;
requestedRawH = reqDims.height;
requestedEffectiveW = reqDims.width;
requestedEffectiveH = reqDims.height;
log(`[t2i] reconstructed render_target from service: ${requestedRawW}x${requestedRawH}`);
}
}
}
else {
if (!srcBuf) {
return {
content: [{ type: "text", text: "No source image available." }],
isError: true,
};
}
// Resolve imageFormat → explicit width/height for i2i/edit/image2video
// so that normalizeInputBuffer and render_target use the correct aspect ratio.
{
const fmt = input?.imageFormat;
const hasW = typeof input?.width === "number" && Number.isFinite(input.width);
const hasH = typeof input?.height === "number" && Number.isFinite(input.height);
if (fmt && !hasW && !hasH) {
const formatDims = {
square: { w: 1024, h: 1024 },
landscape: { w: 1024, h: 768 },
portrait: { w: 768, h: 1024 },
"16:9": { w: 1024, h: 576 },
};
const dims = formatDims[fmt];
if (dims) {
input.width = dims.w;
input.height = dims.h;
log(`[i2i/edit] resolved imageFormat "${fmt}" → ${dims.w}x${dims.h}`);
}
}
}
// UNIFIED: Use normalizeInputBuffer for all i2i/edit input preprocessing
try {
if (resolvedName === "drawthings") {
const userOutW = input?.width;
const userOutH = input?.height;
const isZoomProfile = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
const isRefineProfile = !isEditMode && _internal?.sourceTag === "canvas:refine";
const normalized = await normalizeInputBuffer(srcBuf, {
requestedRawW: typeof userOutW === "number" &&
Number.isFinite(userOutW) &&
userOutW > 0
? userOutW
: undefined,
requestedRawH: typeof userOutH === "number" &&
Number.isFinite(userOutH) &&
userOutH > 0
? userOutH
: undefined,
logPrefix: "[i2i]",
targetSumOverride: isZoomProfile
? core_bundle_mjs_1.drawthingsLimits.targetSumZoom
: isRefineProfile
? core_bundle_mjs_1.drawthingsLimits.targetSumRefine
: undefined,
});
srcBuf = normalized.buf;
sourcePreprocess = normalized.preprocess;
if (typeof normalized.normalizedLongSide === "number") {
normalizedToLongSide = normalized.normalizedLongSide;
}
// Scale mask to match adjusted canvas dims
if (effectiveMaskBuf) {
const adjW = normalized.preprocess.adjusted.width;
const adjH = normalized.preprocess.adjusted.height;
const origW = normalized.preprocess.original.width;
const origH = normalized.preprocess.original.height;
if (adjW && adjH && (adjW !== origW || adjH !== origH)) {
effectiveMaskBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(effectiveMaskBuf, adjW, adjH);
}
}
auditMaskBuf = effectiveMaskBuf;
}
}
catch (e) {
const errMsg = `i2i normalization error: ${String(e)}`;
log(errMsg);
return {
content: [
{
type: "text",
text: `Image2Image setup failed. Error: ${String(e)}`,
},
],
isError: true,
};
}
log(`effective mode: ${mode === "image2video" ? "image2video" : mode} (source=${sourceTag || "unknown"}${sourceFileName ? ", file=" + sourceFileName : ""})`);
// Backend input must not accept internal knobs from user.
let serviceInputForI2I = stripInternalToolKeys(input);
// Only applies when we're actually running an i2i call.
if (mode === "image2video") {
serviceInputForI2I._dt_video_mode = "img2vid";
}
else if (isEditMode) {
if (_internal?.sourceTag === "canvas:inpaint") {
serviceInputForI2I._dt_i2i_profile = "inpaint";
}
else if (_internal?.sourceTag === "canvas:outpaint") {
serviceInputForI2I._dt_i2i_profile = "outpaint";
}
else {
serviceInputForI2I._dt_i2i_profile = "edit";
}
}
else {
serviceInputForI2I._dt_i2i_profile = (_internal?.sourceTag === "canvas:zoom-in" || _internal?.sourceTag === "canvas:upscale") ? "zoom" : _internal?.sourceTag === "canvas:refine" ? "refine" : "img2img";
}
// Derive requestedRaw + requestedEffective.
// - requestedRaw is the user request when provided; otherwise the ORIGINAL source dimensions.
// - requestedEffective is the aligned/clamped size we treat as the effective target.
// Backend internal processing size (i2i normalization) may still differ.
try {
if (resolvedName === "drawthings") {
const userW = input?.width;
const userH = input?.height;
const hasUserW = typeof userW === "number" && Number.isFinite(userW);
const hasUserH = typeof userH === "number" && Number.isFinite(userH);
{
const limits = isEditMode
? core_bundle_mjs_1.drawthingsEditLimits
: core_bundle_mjs_1.drawthingsLimits;
const align = limits.align;
const minDim = limits.minDim;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));
const roundTo = (v, step) => Math.round(v / step) * step;
const floorTo = (v, step) => Math.floor(v / step) * step;
const ceilTo = (v, step) => Math.ceil(v / step) * step;
const chooseEgalized = (origW, origH) => {
const aspect = origW / Math.max(1, origH);
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
const candidates = [];
const add = (w, h) => {
if (!Number.isFinite(w) || !Number.isFinite(h))
return;
w = Math.round(w);
h = Math.round(h);
if (w <= 0 || h <= 0)
return;
if (w % align !== 0 || h % align !== 0)
return;
if (w < minAligned || h < minAligned)
return;
if (w > maxAlignedW || h > maxAlignedH)
return;
candidates.push({ w, h });
};
// 1) Near-original multiples for W/H
const wFloor = clamp(floorTo(origW, align), align, maxAlignedW);
const wCeil = clamp(ceilTo(origW, align), align, maxAlignedW);
const hFloor = clamp(floorTo(origH, align), align, maxAlignedH);
const hCeil = clamp(ceilTo(origH, align), align, maxAlignedH);
// Try deriving H from W candidates (preserve aspect as best we can)
for (const w0 of [wFloor, wCeil, minAligned]) {
const h0 = roundTo(w0 / aspect, align);
add(w0, h0);
}
// Try deriving W from H candidates
for (const h0 of [hFloor, hCeil, minAligned]) {
const w0 = roundTo(h0 * aspect, align);
add(w0, h0);
}
// Small neighborhood search around rounded H to capture exact-aspect pairs
// (e.g., 300x200 -> 384x256 preserves 1.5 exactly).
for (const hBase of [hFloor, hCeil, minAligned]) {
for (const dh of [-2, -1, 0, 1, 2]) {
const h0 = hBase + dh * align;
const w0 = roundTo(h0 * aspect, align);
add(w0, h0);
}
}
if (candidates.length === 0) {
// Hard fallback: clamp + align independently (aspect may drift)
const w0 = clamp(roundTo(origW, align), minAligned, maxAlignedW);
const h0 = clamp(roundTo(origH, align), minAligned, maxAlignedH);
return { w: w0, h: h0 };
}
// Pick candidate minimizing aspect error, then size delta
candidates.sort((a, b) => {
const ae = Math.abs(a.w / Math.max(1, a.h) - aspect);
const be = Math.abs(b.w / Math.max(1, b.h) - aspect);
if (ae !== be)
return ae - be;
const ad = Math.abs(a.w - origW) + Math.abs(a.h - origH);
const bd = Math.abs(b.w - origW) + Math.abs(b.h - origH);
return ad - bd;
});
return candidates[0];
};
// Prefer explicit user-provided OUT size when both provided.
// Otherwise derive OUT size from ORIGINAL (pre-normalization) source.
const origW0 = sourcePreprocess?.original?.width;
const origH0 = sourcePreprocess?.original?.height;
const origAspect = typeof origW0 === "number" &&
Number.isFinite(origW0) &&
typeof origH0 === "number" &&
Number.isFinite(origH0) &&
origH0 > 0
? origW0 / origH0
: undefined;
// Fall back to current (normalized) buffer size only if original is missing.
const fallbackSz = typeof origW0 === "number" && typeof origH0 === "number"
? null
: await (0, core_bundle_mjs_1.getSize)(srcBuf);
let baseW;
let baseH;
if (hasUserW && hasUserH) {
baseW = Math.round(userW);
baseH = Math.round(userH);
}
else if (!hasUserW && !hasUserH) {
baseW =
typeof origW0 === "number" && Number.isFinite(origW0)
? origW0
: fallbackSz?.width;
baseH =
typeof origH0 === "number" && Number.isFinite(origH0)
? origH0
: fallbackSz?.height;
}
else if (hasUserW && !hasUserH) {
baseW = Math.round(userW);
if (typeof origAspect === "number") {
baseH = Math.max(1, Math.round(baseW / origAspect));
}
}
else if (!hasUserW && hasUserH) {
baseH = Math.round(userH);
if (typeof origAspect === "number") {
baseW = Math.max(1, Math.round(baseH * origAspect));
}
}
if (typeof baseW === "number" && typeof baseH === "number") {
// requestedRaw: user request when present; else original source size.
if (hasUserW)
requestedRawW = Math.max(1, Math.round(userW));
if (hasUserH)
requestedRawH = Math.max(1, Math.round(userH));
if (!hasUserW && !hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
else if (hasUserW && !hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
else if (!hasUserW && hasUserH) {
requestedRawW = Math.max(1, Math.round(baseW));
requestedRawH = Math.max(1, Math.round(baseH));
}
// requestedEffective: if user provided BOTH dims, round each independently to align.
// Otherwise, preserve aspect as closely as possible.
if (hasUserW && hasUserH) {
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
// Zoom/upscale profile: DT backend renders at input image dims, not config dims.
// Skip maxW/maxH clamp so requestedEffective = adjusted (= requestedRaw for zoom).
const isZoomProfileEffective = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
const wEff = isZoomProfileEffective
? Math.max(minAligned, roundTo(baseW, align))
: clamp(roundTo(baseW, align), minAligned, maxAlignedW);
const hEff = isZoomProfileEffective
? Math.max(minAligned, roundTo(baseH, align))
: clamp(roundTo(baseH, align), minAligned, maxAlignedH);
requestedEffectiveW = wEff;
requestedEffectiveH = hEff;
}
else {
const eg = chooseEgalized(baseW, baseH);
requestedEffectiveW = eg.w;
requestedEffectiveH = eg.h;
}
log(`[i2i/edit] requested: raw=${requestedRawW || "-"}x${requestedRawH || "-"} effective=${requestedEffectiveW || "-"}x${requestedEffectiveH || "-"} (user provided: ${hasUserW ? "w" : "-"}${hasUserH ? "h" : "-"})`);
// CRITICAL: Pass requestedEffective dimensions to the backend!
// The backend must generate at requested_effective size, not at
// the normalized source (adjusted) size.
if (typeof requestedEffectiveW === "number" &&
typeof requestedEffectiveH === "number") {
serviceInputForI2I.width = requestedEffectiveW;
serviceInputForI2I.height = requestedEffectiveH;
log(`[i2i/edit] set serviceInputForI2I dimensions to effective: ${requestedEffectiveW}x${requestedEffectiveH}`);
}
// Deterministic contract: always provide raw dims + upscaler decision for Draw Things i2i/edit.
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
serviceInputForI2I._dt_requested_raw_w =
requestedRawW;
serviceInputForI2I._dt_requested_raw_h =
requestedRawH;
// needs_upscaler: true when requestedEffective exceeds adjusted AND the
// scale-factor (rawW/adjW) meets the method-specific threshold.
// Below the threshold, Jimp alone handles the upscale; no upscaler/zoom-pass.
const _adjW = sourcePreprocess?.adjusted?.width ?? 0;
const _adjH = sourcePreprocess?.adjusted?.height ?? 0;
const _scaleFactor = _adjW > 0 && _adjH > 0
? Math.max(requestedRawW / _adjW, requestedRawH / _adjH)
: 0;
const _minThreshold = core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass"
? core_bundle_mjs_1.drawthingsLimits.minUpscaleFactorZoomPass
: core_bundle_mjs_1.drawthingsLimits.minUpscaleFactorUpscaler;
const _exceedsAdjusted = (typeof requestedEffectiveW === "number" && requestedEffectiveW > _adjW) ||
(typeof requestedEffectiveH === "number" && requestedEffectiveH > _adjH);
const _dt_needs_upscaler = _exceedsAdjusted && _scaleFactor >= _minThreshold;
_dtNeedsUpscaler = _dt_needs_upscaler;
serviceInputForI2I._dt_needs_upscaler = _dt_needs_upscaler;
// When zoom-pass will run, Pass 1 generates at adjusted size.
// Zoom-pass handles all upscaling; edit model only needs canvas resolution.
// Exception: canvas:zoom-in / canvas:upscale always run SeedVR2 at target dims — no override.
const _isZoomToolCall = _internal?.sourceTag === "canvas:zoom-in" || _internal?.sourceTag === "canvas:upscale";
if (_dt_needs_upscaler && core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass" && _adjW > 0 && _adjH > 0 && !_isZoomToolCall) {
serviceInputForI2I.width = _adjW;
serviceInputForI2I.height = _adjH;
}
}
else {
throw new Error("Invariant failed: requested_raw dims missing for drawthings i2i/edit");
}
}
}
}
}
catch (e) {
log(`[i2i] failed to derive width/height from source: ${String(e)}`);
}
// SAFETY NET: Ensure serviceInputForI2I dimensions ALWAYS respect backend limits.
// This catches edge cases where the main calculation block was skipped or failed.
// Zoom/upscale profile is excluded: DT backend ignores config dims for i2i and
// renders at input image dimensions, which may legitimately exceed 1536.
{
const isZoomProfileSafety = !isEditMode && (_internal?.sourceTag === "canvas:zoom-in" ||
_internal?.sourceTag === "canvas:upscale");
if (!isZoomProfileSafety) {
const limits = isEditMode ? core_bundle_mjs_1.drawthingsEditLimits : core_bundle_mjs_1.drawthingsLimits;
const align = limits.align;
const minDim = limits.minDim;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const minAligned = Math.ceil(minDim / align) * align;
const maxAlignedW = Math.floor(maxW / align) * align;
const maxAlignedH = Math.floor(maxH / align) * align;
const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));
const roundTo = (v, step) => Math.round(v / step) * step;
const inW = serviceInputForI2I.width;
const inH = serviceInputForI2I.height;
if (typeof inW === "number" && Number.isFinite(inW)) {
const sanitized = clamp(roundTo(inW, align), minAligned, maxAlignedW);
if (sanitized !== inW) {
log(`[i2i] SAFETY: sanitized width ${inW} → ${sanitized}`);
serviceInputForI2I.width = sanitized;
}
}
if (typeof inH === "number" && Number.isFinite(inH)) {
const sanitized = clamp(roundTo(inH, align), minAligned, maxAlignedH);
if (sanitized !== inH) {
log(`[i2i] SAFETY: sanitized height ${inH} → ${sanitized}`);
serviceInputForI2I.height = sanitized;
}
}
}
}
// Edit mode requires gRPC backend (HTTP does not support edit mode at all)
if (isEditMode && typeof svc.generateImageEdit !== "function") {
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
// If no backend is connected at all, prefer the generic backend error.
// Otherwise the message is misleading (it implies HTTP is active).
if (!selectedTransport) {
log(`[edit] ERROR: edit mode requested but no Draw Things backend is connected`);
return {
content: [
{
type: "text",
text: "Failed to generate image: backend error",
},
],
isError: true,
};
}
if (selectedTransport === "http") {
log(`[edit] ERROR: HTTP backend does not support edit mode (generateImageEdit not available)`);
return {
content: [
{
type: "text",
text: `Edit mode is not supported via HTTP. Edit mode requires the Draw Things gRPC backend. Use mode='image2image' instead, or switch to gRPC.`,
},
],
isError: true,
};
}
// Defensive fallback: transport says gRPC but method is missing.
log(`[edit] ERROR: gRPC transport selected but edit mode is unavailable (generateImageEdit missing)`);
return {
content: [
{ type: "text", text: "Failed to generate image: backend error" },
],
isError: true,
};
}
// Multi-reference edit/image2image mode: collect additional buffers and call generateImageEdit
if (isMultiReference && typeof svc.generateImageEdit === "function") {
log(`[${mode}] resolving multi-reference sources...`);
// ─────────────────────────────────────────────────────────────────
// PHASE 2: No auto-fill. Only explicitly selected sources are used.
// ─────────────────────────────────────────────────────────────────
// Get model capabilities for limit checking
// Use edit or image2image limits based on mode
const capKey = getCapabilityKeyForPreset(effectiveModelPreset);
const imageCaps = capKey
? detectImageModelCapabilities(capKey)
: null;
const maxRefs = mode === "edit"
? (imageCaps?.edit?.maxReferenceImages ?? 4)
: (imageCaps?.image2image?.maxReferenceImages ?? 1);
log(`[${mode}] model=${effectiveModelPreset}, maxReferenceImages=${maxRefs}`);
// Get available attachments/variants/pictures count (for existence validation)
const chatWd = getChatWdForContext();
let availableAttachmentCount = 0;
let availableVariantCount = 0;
let availablePictureCount = 0;
if (chatWd) {
try {
const st = await (0, core_bundle_mjs_1.readState)(chatWd);
availableAttachmentCount = Array.isArray(st.attachments)
? st.attachments.length
: 0;
availableVariantCount = Array.isArray(st.variants)
? st.variants.length
: 0;
availablePictureCount = Array.isArray(st.pictures)
? st.pictures.length
: 0;
log(`[${mode}] available: ${availableAttachmentCount} attachments, ${availableVariantCount} variants, ${availablePictureCount} pictures`);
}
catch (e) {
log(`[${mode}] failed to read state: ${String(e)}`);
}
}
// Use resolved canvas + moodboard (no legacy sourceAttachment/sourceVariant)
const canvasSel = resolvedCanvasSel;
const moodboardSel = resolvedMoodboardSel;
// ─────────────────────────────────────────────────────────────────
// LIMIT VALIDATION: Check total references against model capabilities
// ─────────────────────────────────────────────────────────────────
const totalRequested = (canvasSel ? 1 : 0) + (moodboardSel?.length || 0);
log(`[${mode}] total references requested: ${totalRequested}, limit: ${maxRefs}`);
if (totalRequested > maxRefs) {
// Build a detailed error message
const details = [
canvasSel ? `canvas=${canvasSel.notation}` : "no canvas",
moodboardSel && moodboardSel.length > 0
? `moodboard=[${moodboardSel.map((s) => s.notation).join(",")}]`
: "",
]
.filter((s) => s)
.join(", ");
return {
content: [
{
type: "text",
text: `Model '${effectiveModelPreset}' supports max ${maxRefs} reference images in ${mode} mode.\n` +
`Requested: ${totalRequested} (${details}).\n\n` +
`Please make an explicit selection:\n` +
`- Use 'canvas' to specify the priority image (e.g., canvas="a1" or canvas="v2" or canvas="p3")\n` +
`- Use 'moodboard' to add reference images (e.g., moodboard=["a2","v1","p4"])`,
},
],
isError: true,
};
}
// ─────────────────────────────────────────────────────────────────
const referenceBuffers = [];
const referenceMetadata = [];
const referencePreprocess = [];
// Get user-requested output dimensions for capping input size
const userOutW = input?.width;
const userOutH = input?.height;
const editRequestedRawW = typeof userOutW === "number" &&
Number.isFinite(userOutW) &&
userOutW > 0
? userOutW
: undefined;
const editRequestedRawH = typeof userOutH === "number" &&
Number.isFinite(userOutH) &&
userOutH > 0
? userOutH
: undefined;
const pushReference = async (sel, isCanvas) => {
const loaded = await loadBufferForSel(sel);
// UNIFIED: Use normalizeInputBuffer for all edit/image2image multi-ref inputs
// Only canvas adopts the requested output AR; moodboard preserves its native AR.
const normalized = await normalizeInputBuffer(loaded.buf, {
requestedRawW: isCanvas ? editRequestedRawW : undefined,
requestedRawH: isCanvas ? editRequestedRawH : undefined,
logPrefix: `[${mode}:${isCanvas ? "canvas" : "moodboard"}]`,
});
referenceBuffers.push(normalized.buf);
referenceMetadata.push({
type: sel.pool,
index: sel.index,
isCanvas,
originPath: loaded.originPath,
originalName: loaded.originalName,
});
referencePreprocess.push({
type: sel.pool,
index: sel.index,
role: isCanvas ? "canvas" : "moodboard",
originPath: loaded.originPath,
originalName: loaded.originalName,
preprocess: normalized.preprocess,
});
};
// 1. Resolve Canvas first
if (canvasSel) {
try {
await pushReference(canvasSel, true);
log(`[${mode}] canvas resolved: ${canvasSel.notation}`);
}
catch (e) {
return {
content: [
{
type: "text",
text: `Canvas ${canvasSel.notation} not found: ${String(e?.message || e)}`,
},
],
isError: true,
};
}
}
// 2. Resolve Moodboard selections
for (const sel of moodboardSel || []) {
try {
await pushReference(sel, false);
log(`[${mode}] moodboard resolved: ${sel.notation}`);
}
catch (e) {
return {
content: [
{
type: "text",
text: `Moodboard ${sel.notation} not found: ${String(e?.message || e)}`,
},
],
isError: true,
};
}
}
// If no canvas was explicitly or auto-selected but we have references, use first as canvas
if (!resolvedCanvasSel && referenceBuffers.length > 0) {
referenceMetadata[0].isCanvas = true;
log(`[${mode}] auto-selected first reference as canvas: ${referenceMetadata[0].type} ${referenceMetadata[0].index}`);
}
// Fallback: if still no references, use the already-resolved srcBuf
if (referenceBuffers.length === 0 && srcBuf) {
// UNIFIED: Use normalizeInputBuffer
const normalized = await normalizeInputBuffer(srcBuf, {
requestedRawW: editRequestedRawW,
requestedRawH: editRequestedRawH,
logPrefix: `[${mode}:fallback-canvas]`,
});
referenceBuffers.push(normalized.buf);
referenceMetadata.push({
type: sourceKind || "image",
index: sourceVariantUsed || 1,
isCanvas: true,
});
referencePreprocess.push({
type: sourceKind || "image",
index: sourceVariantUsed || 1,
role: "canvas",
preprocess: normalized.preprocess,
});
log(`[${mode}] fallback: using single source as canvas`);
}
// Copy metadata for summary (outside this block)
usedReferenceMeta = [...referenceMetadata];
usedReferencePreprocess = [...referencePreprocess];
log(`[${mode}] calling generateImageEdit with ${referenceBuffers.length} references`);
result = await svc.generateImageEdit(serviceInputForI2I, referenceBuffers, onProgress, effectiveMaskBuf);
// Update sourceTag for audit
sourceTag = `${mode}:refs=${referenceBuffers.length}`;
}
else {
// Single-reference path (original behavior)
// Populate usedReferenceMeta for consistency in summary
if (srcBuf && (sourceKind || effectiveMode === "image2image")) {
usedReferenceMeta = [
{
type: sourceKind || "image",
index: sourceVariantUsed || 1,
isCanvas: true, // Single reference is always canvas
},
];
}
result = await svc.generateImageImg2Img(serviceInputForI2I, srcBuf, onProgress, effectiveMaskBuf);
// Reconstruct render_target from service metadata if not already set
// (fallback for edge cases where core logic was bypassed)
if ((typeof requestedRawW !== "number" ||
typeof requestedRawH !== "number") &&
result?.metadata?.requested_dimensions) {
const reqDims = result.metadata.requested_dimensions;
if (typeof reqDims.width === "number" &&
typeof reqDims.height === "number") {
requestedRawW = reqDims.width;
requestedRawH = reqDims.height;
requestedEffectiveW = reqDims.width;
requestedEffectiveH = reqDims.height;
log(`[i2i/edit] reconstructed render_target from service: ${requestedRawW}x${requestedRawH}`);
}
}
}
}
}
if (result.isError || result.error) {
const statusRaw = result.status;
let statusNum = undefined;
if (typeof statusRaw === "number" && Number.isFinite(statusRaw))
statusNum = statusRaw;
else if (typeof statusRaw === "string") {
const p = parseInt(statusRaw, 10);
if (Number.isFinite(p))
statusNum = p;
}
const raw = result.errorMessage ||
result.error ||
"unknown error";
const codeText = typeof statusNum === "number" ? `status ${statusNum}` : "backend error";
await logError(new Error(`Failed to generate image: ${codeText}`));
await appendErrorRaw(typeof raw === "string" ? raw : String(raw), statusNum);
const snippet = (() => {
try {
const s = String(raw);
return s.length > 500 ? s.slice(0, 500) + "…" : s;
}
catch {
return "";
}
})();
return {
content: [
{ type: "text", text: `Failed to generate image: ${codeText}` },
...(snippet
? [{ type: "text", text: `Details: ${snippet}` }]
: []),
],
isError: true,
};
}
let buffers = [];
if (result.images &&
Array.isArray(result.images) &&
result.images.length > 0) {
for (const img of result.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
buffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (result.imageBuffer &&
Buffer.isBuffer(result.imageBuffer)) {
buffers.push(result.imageBuffer);
}
else if (result.imageData) {
const data = result.imageData;
if (Buffer.isBuffer(data))
buffers.push(data);
else if (typeof data === "string") {
const b64 = data.startsWith("data:") ? data.split(",")[1] : data;
buffers.push(Buffer.from(b64, "base64"));
}
}
else if (result.imagePath) {
try {
const abs = path_1.default.resolve(result.imagePath);
buffers.push(await fs_1.default.promises.readFile(abs));
}
catch (e) {
log(`Failed to read returned imagePath: ${result.imagePath}: ${String(e)}`);
}
}
if (buffers.length === 0)
throw new Error("No valid image data returned");
// Measure Pass-1 backend output before zoom-pass can replace buffers.
let backendReturnedW;
let backendReturnedH;
try {
if (buffers[0]) {
const meta0 = await (0, core_bundle_mjs_1.getSize)(buffers[0]);
backendReturnedW = meta0.width;
backendReturnedH = meta0.height;
}
}
catch { }
// ── ZOOM-PASS PIPELINE (Schritte 3-7) ────────────────────────────────────
if (imageService?.name === "drawthings" &&
core_bundle_mjs_1.drawthingsLimits.upscaleMethod === "zoom-pass" &&
_dtNeedsUpscaler &&
(effectiveMode === "image2image" || effectiveMode === "edit") &&
_internal?.sourceTag !== "canvas:zoom-in" &&
_internal?.sourceTag !== "canvas:upscale" &&
typeof requestedRawW === "number" &&
typeof requestedRawH === "number" &&
buffers.length > 0) {
try {
const rawSum = requestedRawW + requestedRawH;
const targetSumZoom = core_bundle_mjs_1.drawthingsLimits.targetSumZoom;
const align = core_bundle_mjs_1.drawthingsLimits.align;
// Step 4: Jimp Pre-Resize — build Zoom-Pass Canvas
let zoomCanvasW;
let zoomCanvasH;
let zoomCanvasBuf;
if (rawSum <= targetSumZoom) {
zoomCanvasW = requestedRawW;
zoomCanvasH = requestedRawH;
zoomCanvasBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(buffers[0], zoomCanvasW, zoomCanvasH);
}
else {
// Canvas = AR-preserving scale to targetSumZoom, 64-aligned
// Use candidate selection (floor/round/ceil on each axis) to minimise AR error.
const aspect = requestedRawW / requestedRawH;
const hRaw = targetSumZoom / (1 + aspect);
const wRaw = targetSumZoom - hRaw;
const snaps = (v) => [
Math.max(align, Math.floor(v / align) * align),
Math.max(align, Math.round(v / align) * align),
Math.max(align, Math.ceil(v / align) * align),
];
const candidates = [];
for (const wSnap of snaps(wRaw)) {
for (const hSnap of snaps(hRaw)) {
if (wSnap + hSnap <= targetSumZoom)
candidates.push({ w: wSnap, h: hSnap, arErr: Math.abs(wSnap / hSnap - aspect) });
}
}
candidates.sort((a, b) => a.arErr - b.arErr || (b.w + b.h) - (a.w + a.h));
const best = candidates[0] ?? { w: Math.max(align, Math.floor(wRaw / align) * align), h: Math.max(align, Math.floor(hRaw / align) * align) };
zoomCanvasW = best.w;
zoomCanvasH = best.h;
const arExpected = requestedRawW / requestedRawH;
const arActual = zoomCanvasW / zoomCanvasH;
if (Math.abs(arActual - arExpected) > 0.01)
log(`[AR-CHECK] zoom-canvas: ${zoomCanvasW}x${zoomCanvasH} AR=${arActual.toFixed(4)} vs raw ${requestedRawW}x${requestedRawH} AR=${arExpected.toFixed(4)} err=${Math.abs(arActual - arExpected).toFixed(4)}`);
zoomCanvasBuf = await (0, core_bundle_mjs_1.resizeFillToPng)(buffers[0], zoomCanvasW, zoomCanvasH);
}
// Step 5: Pass 2 — SeedVR2 call
const zoomOnProgress = onProgress
? (step, total, msg) => {
if (step === -1) {
onProgress(-1, total, msg ? `Zoom ${msg}` : "Zoom");
}
else {
const t = typeof total === "number" ? total : undefined;
if (t && t > 0) {
onProgress(-1, total, `Zoom Step ${step}/${t} (${Math.round((step / (t + 1)) * 100)}%)`);
}
else {
onProgress(-1, total, `Zoom Step ${step}...`);
}
}
}
: undefined;
// Only pass runtime overrides — the service fills defaultParamsZoom as baseDefaults itself.
const zoomParams = {
width: zoomCanvasW,
height: zoomCanvasH,
prompt: typeof input.prompt === "string" ? input.prompt : "",
_dt_i2i_profile: "zoom",
_dt_needs_upscaler: false,
_dt_requested_raw_w: zoomCanvasW,
_dt_requested_raw_h: zoomCanvasH,
};
const zoomStartMs = Date.now();
const zoomResult = await svc.generateImageImg2Img(zoomParams, zoomCanvasBuf, zoomOnProgress);
const zoomInferenceMs = Date.now() - zoomStartMs;
if (zoomResult.isError || zoomResult.error) {
throw new Error(`Zoom-Pass failed: ${zoomResult.errorMessage ?? zoomResult.error ?? "unknown"}`);
}
// Extract Pass-2 buffers (same pattern as main buffer extraction above)
let zoomBuffers = [];
if (Array.isArray(zoomResult.images) && zoomResult.images.length > 0) {
for (const img of zoomResult.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
zoomBuffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (Buffer.isBuffer(zoomResult.imageBuffer)) {
zoomBuffers.push(zoomResult.imageBuffer);
}
else if (zoomResult.imageData) {
const d = zoomResult.imageData;
if (Buffer.isBuffer(d))
zoomBuffers.push(d);
else if (typeof d === "string") {
const b64 = d.startsWith("data:") ? d.split(",")[1] : d;
zoomBuffers.push(Buffer.from(b64, "base64"));
}
}
if (zoomBuffers.length === 0)
throw new Error("Zoom-Pass returned no image data");
// Measure Pass-2 backend output
let zoomBackendW;
let zoomBackendH;
try {
const zm = await (0, core_bundle_mjs_1.getSize)(zoomBuffers[0]);
zoomBackendW = zm.width;
zoomBackendH = zm.height;
}
catch { }
const zoomMeta = zoomResult?.metadata ?? {};
// Step 6: Post-Processing Stage 2
if (rawSum <= targetSumZoom) {
buffers = zoomBuffers;
}
else {
const resized = [];
for (const zb of zoomBuffers) {
resized.push(await (0, core_bundle_mjs_1.resizeFillToPng)(zb, requestedRawW, requestedRawH));
}
buffers = resized;
}
// Step 7: Write Pass-2 audit entry
zoomPassRan = true;
try {
const audit2 = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: resolvedName, mode: "zoom", requestId: auditRequestId });
if (currentLmChatId)
audit2.setChatId(currentLmChatId);
const zoomUserReq = {};
if (input?.mode)
zoomUserReq.mode = input.mode;
if (input?.prompt)
zoomUserReq.prompt = input.prompt;
if (input?.canvas)
zoomUserReq.canvas = input.canvas;
audit2.setUserRequest(zoomUserReq);
audit2.setRenderTarget({
requested_raw: { width: requestedRawW, height: requestedRawH },
requested_effective: { width: zoomCanvasW, height: zoomCanvasH },
needs_upscaler: true,
});
audit2.setInputs({
canvas: {
original: backendReturnedW !== undefined && backendReturnedH !== undefined
? { width: backendReturnedW, height: backendReturnedH }
: { width: requestedRawW, height: requestedRawH },
adjusted: { width: zoomCanvasW, height: zoomCanvasH },
},
});
const zoomOutput = {};
if (zoomBackendW !== undefined && zoomBackendH !== undefined) {
zoomOutput.backend_returned = { width: zoomBackendW, height: zoomBackendH };
}
zoomOutput.post_processed = { width: requestedRawW, height: requestedRawH };
zoomOutput.inference_time_ms = zoomInferenceMs;
if (typeof zoomMeta.model === "string" && zoomMeta.model.trim()) {
zoomOutput.model_used = path_1.default.basename(zoomMeta.model);
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.model === "string") {
zoomOutput.model_used = core_bundle_mjs_1.defaultParamsZoom.model;
}
if (typeof zoomMeta.steps_used === "number") {
zoomOutput.steps_used = zoomMeta.steps_used;
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.steps === "number") {
zoomOutput.steps_used = core_bundle_mjs_1.defaultParamsZoom.steps;
}
if (typeof zoomMeta.sampler_used === "string" && zoomMeta.sampler_used.trim()) {
zoomOutput.sampler_used = zoomMeta.sampler_used;
}
else if (typeof core_bundle_mjs_1.defaultParamsZoom.sampler === "string") {
zoomOutput.sampler_used = core_bundle_mjs_1.defaultParamsZoom.sampler;
}
audit2.setOutput(zoomOutput);
pendingAudit2 = audit2;
}
catch (auditErr) {
log(`zoom-pass audit write failed: ${String(auditErr)}`);
}
log(`zoom-pass: canvas=${zoomCanvasW}x${zoomCanvasH} backend=${zoomBackendW ?? "?"}x${zoomBackendH ?? "?"} -> final=${requestedRawW}x${requestedRawH} (${zoomInferenceMs}ms)`);
}
catch (e) {
throw new Error(`Zoom-Pass pipeline failed: ${String(e?.message || e)}`);
}
}
// ── END ZOOM-PASS PIPELINE ────────────────────────────────────────────────
const userReqW = (() => {
const v = input?.width;
return typeof v === "number" && Number.isFinite(v)
? Math.max(1, Math.round(v))
: undefined;
})();
const userReqH = (() => {
const v = input?.height;
return typeof v === "number" && Number.isFinite(v)
? Math.max(1, Math.round(v))
: undefined;
})();
if (!zoomPassRan &&
imageService?.name === "drawthings" &&
(effectiveMode === "image2image" || effectiveMode === "edit") &&
sourcePreprocess &&
(sourcePreprocess.reason === "normalized_to_constraints" ||
sourcePreprocess.reason === "clamped_to_requested_raw") &&
sourcePreprocess.original?.width &&
sourcePreprocess.original?.height &&
sourcePreprocess.adjusted?.width &&
sourcePreprocess.adjusted?.height &&
(sourcePreprocess.original.width !== sourcePreprocess.adjusted.width ||
sourcePreprocess.original.height !== sourcePreprocess.adjusted.height)) {
try {
let targetW = typeof requestedEffectiveW === "number" &&
Number.isFinite(requestedEffectiveW)
? Math.max(1, Math.round(requestedEffectiveW))
: Math.max(1, Math.round(sourcePreprocess.original.width));
let targetH = typeof requestedEffectiveH === "number" &&
Number.isFinite(requestedEffectiveH)
? Math.max(1, Math.round(requestedEffectiveH))
: Math.max(1, Math.round(sourcePreprocess.original.height));
try {
const limits = effectiveMode === "edit" ? core_bundle_mjs_1.drawthingsEditLimits : core_bundle_mjs_1.drawthingsLimits;
const maxW = limits.maxWidth;
const maxH = limits.maxHeight;
const s = Math.min(maxW / targetW, maxH / targetH, 1);
if (s < 1) {
targetW = Math.max(1, Math.round(targetW * s));
targetH = Math.max(1, Math.round(targetH * s));
}
}
catch { }
const resizedBuffers = [];
for (const buf of buffers) {
const r = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, targetW, targetH);
resizedBuffers.push(r);
}
buffers = resizedBuffers;
const restoreTargetLabel = typeof requestedEffectiveW === "number" &&
typeof requestedEffectiveH === "number"
? "requested effective size"
: "original source size";
log(`postprocess: restored generated image(s) to ${restoreTargetLabel} ${targetW}x${targetH} (from normalized ${sourcePreprocess.adjusted.width}x${sourcePreprocess.adjusted.height})`);
}
catch (e) {
log(`postprocess restore-to-original-size failed: ${String(e)}`);
}
}
// Final step: if we have a raw size target (either explicit user size, or derived from the
// attached source when user omitted width/height), resize output back to that exact size.
// Only absent when user provided no size AND there is no attached source (pure defaults).
try {
const finalRawW = typeof requestedRawW === "number" && Number.isFinite(requestedRawW)
? Math.max(1, Math.round(requestedRawW))
: undefined;
const finalRawH = typeof requestedRawH === "number" && Number.isFinite(requestedRawH)
? Math.max(1, Math.round(requestedRawH))
: undefined;
if (finalRawW && finalRawH) {
const metaCur = buffers[0] ? await (0, core_bundle_mjs_1.getSize)(buffers[0]) : null;
const curW = metaCur?.width;
const curH = metaCur?.height;
if (curW !== finalRawW || curH !== finalRawH) {
const resizedFinal = [];
for (const buf of buffers) {
const r = await (0, core_bundle_mjs_1.resizeCoverToPng)(buf, finalRawW, finalRawH);
resizedFinal.push(r);
}
buffers = resizedFinal;
log(`postprocess: adjusted final generated image(s) to requested raw size ${finalRawW}x${finalRawH}`);
}
}
}
catch (e) {
log(`postprocess final-resize-to-user-request failed: ${String(e)}`);
}
let postProcessedW;
let postProcessedH;
try {
if (zoomPassRan) {
// Pass 1 post-processed = Pass 1 backend output (before zoom-pass replaced buffers).
postProcessedW = backendReturnedW;
postProcessedH = backendReturnedH;
}
else if (buffers[0]) {
const metaF = await (0, core_bundle_mjs_1.getSize)(buffers[0]);
postProcessedW = metaF.width;
postProcessedH = metaF.height;
}
}
catch { }
const saveOriginal = core_bundle_mjs_1.generateRuntimeDefaults.saveOriginal;
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw != null
? /^(1|true|yes)$/i.test(String(envPreviewRaw).trim())
: true;
log(`preview toggle: PREVIEW_IN_CHAT='${envPreviewRaw}' -> ${previewInChat}`);
const promptStr = typeof input.prompt === "string" ? input.prompt : "";
const alt = promptStr.trim()
? `Generated image: ${promptStr.slice(0, 80)}`
: "Generated image";
const savedFiles = [];
// Primary storage: write directly into the active LM Studio chat working directory.
// Fail-fast if chatId could not be resolved.
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
throw new Error("Failed to resolve LM Studio chat working directory (chatId missing).");
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Read current state to get nextImageI for FORTLAUFENDE (continuous) image numbering
// This ensures i1, i2, i3... across multiple generation runs (not resetting to i1 each time)
const currentState = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const baseImageI = Math.max(1, currentState.counters.nextImageI ?? 1);
log(`image numbering: starting at i${baseImageI} (nextImageI from state)`);
const baseStamp = isoStampCompact();
const imageRecordsForState = [];
// ── VIDEO PATH ────────────────────────────────────────────────────────
const numFramesMeta = result?.metadata?.num_frames;
// Require at least 3 buffers: [discarded-first] + [≥1 real frame] + [discarded-last].
// Without this guard, slice(1, buffers.length - 1) returns an empty array when the
// gRPC response only carried a preview/fallback buffer despite num_frames > 1.
// Mode guard: some official defaults carry wrong numFrames > 1 for non-video models.
const isVideoResult = (effectiveMode === "text2video" || effectiveMode === "image2video") &&
typeof numFramesMeta === "number" && numFramesMeta > 1 && buffers.length >= 3;
// ── BUILD XMP PARAMS (shared by video + image paths) ─────────────────
const resMeta = result?.metadata ?? {};
const xmpSourcePaths = [];
if (sourceOriginAbs)
xmpSourcePaths.push(sourceOriginAbs);
for (const r of usedReferenceMeta ?? []) {
if (r.originPath && !xmpSourcePaths.includes(r.originPath)) {
xmpSourcePaths.push(r.originPath);
}
}
const xmpLorasUsed = Array.isArray(resMeta.loras_used)
? resMeta.loras_used
: [];
const xmpParams = {
...(typeof input.prompt === "string" && input.prompt ? { prompt: input.prompt } : {}),
...(typeof resMeta.model === "string" && resMeta.model
? { model: path_1.default.basename(resMeta.model) }
: effectiveModelFilename
? { model: path_1.default.basename(effectiveModelFilename) }
: {}),
...(typeof resMeta.width === "number" ? { width: resMeta.width } : {}),
...(typeof resMeta.height === "number" ? { height: resMeta.height } : {}),
...(typeof resMeta.steps_used === "number" ? { steps: resMeta.steps_used } : {}),
...(typeof resMeta.seed === "number" ? { seed: resMeta.seed } : {}),
...(typeof resMeta.seed_mode === "string" && resMeta.seed_mode ? { seedMode: resMeta.seed_mode } : {}),
...(typeof resMeta.sampler_used === "string" && resMeta.sampler_used ? { sampler: resMeta.sampler_used } : {}),
...(typeof resMeta.guidance_scale_used === "number" ? { guidanceScale: resMeta.guidance_scale_used } : {}),
...(typeof resMeta.strength_used === "number" ? { strength: resMeta.strength_used } : {}),
...(typeof resMeta.shift_used === "number" ? { shift: resMeta.shift_used } : {}),
...(xmpLorasUsed.length > 0 ? { loras: xmpLorasUsed.map((f) => ({ file: f })) } : {}),
...(xmpSourcePaths.length > 0 ? { sources: xmpSourcePaths } : {}),
mode: effectiveMode,
...(_internal?.cropMeta ? { crop: { left: _internal.cropMeta.left, top: _internal.cropMeta.top, right: _internal.cropMeta.right, bottom: _internal.cropMeta.bottom } } : {}),
...(_internal?.cropMeta?.cropSource ? { cropSource: _internal.cropMeta.cropSource } : {}),
};
// ── END XMP PARAMS ────────────────────────────────────────────────────
let videoFrames = [];
let videoPngSaved = null;
if (isVideoResult) {
const videoFps = typeof result?.metadata?.fps === "number"
? result.metadata.fps
: 24;
// Trim: discard first frame (confirmed) and last frame (pending verification).
// Roadmap trim rule: slice(1, buffers.length - 1) == slice(1, num_frames + 1)
videoFrames = buffers.slice(1, buffers.length - 1);
const videoImageI = baseImageI;
const videoBaseName = `image-${baseStamp}-i${videoImageI}`;
const lastFrame = videoFrames[videoFrames.length - 1];
// PNG: canonical for state / VP / i2i / lastOriginalRef
videoPngSaved = await saveOriginalPng(lastFrame, primaryOutDir, `${videoBaseName}.png`, { ...xmpParams, isVideoFrame: true });
try {
log(`saved original (video last-frame): ${videoPngSaved.savedPath} (${videoPngSaved.size} bytes) [i${videoImageI}]`);
}
catch { }
imageRecordsForState.push({
filename: `${videoBaseName}.png`,
preview: `preview-${videoBaseName}.jpg`,
i: videoImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/generate_image`,
});
// MOV: goes into savedFiles so originalLinksText shows .mov. On failure, falls back to PNG.
try {
onProgress?.(-1, undefined, "Assembling video...");
const { assembleVideo } = await import("../helpers/videoAssembler.js");
const audioChunks = result?.audioBuffers;
const audioRaw = audioChunks && audioChunks.length > 0
? Buffer.concat(audioChunks)
: undefined;
const audioSampleRateRaw = (0, core_bundle_mjs_1.getAudioSampleRateForModel)(effectiveModelFilename);
if (audioSampleRateRaw === undefined) {
log(`[video] ERROR: no audioSampleRate registered for model '${effectiveModelFilename}' — falling back to 48 000 Hz`);
}
const audioSampleRate = audioSampleRateRaw ?? 48_000;
const movBuffer = await assembleVideo(videoFrames, videoFps, audioRaw, audioSampleRate);
const movFileName = `${videoBaseName}.mov`;
const movPath = path_1.default.join(primaryOutDir, movFileName);
await fs_1.default.promises.writeFile(movPath, movBuffer);
const movUrl = encodeFileUrl(movPath);
savedFiles.push({ savedPath: movPath, fileUrl: movUrl, size: movBuffer.length, fileName: movFileName });
log(`saved video: ${movPath} (${movBuffer.length} bytes) [i${videoImageI}]`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`video assembly failed (i${videoImageI}): ${msg}`);
savedFiles.push(videoPngSaved);
}
}
// ── END VIDEO PATH ────────────────────────────────────────────────────
if (!isVideoResult) {
for (let i = 0; i < buffers.length; i++) {
const buf = buffers[i];
const imageI = baseImageI + i; // Fortlaufende Nummerierung
const baseName = `image-${baseStamp}-i${imageI}`;
const s = await saveOriginalPng(buf, primaryOutDir, `${baseName}.png`, xmpParams);
try {
log(`saved original: ${s.savedPath} (${s.size} bytes) [i${imageI}]`);
}
catch { }
savedFiles.push(s);
// Track for state update later
imageRecordsForState.push({
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: imageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/generate_image`,
});
}
}
const firstSaved = savedFiles[0];
// Video: lastOriginalRef must point to PNG (not MOV) so follow-up i2i/canvas loads work.
lastOriginalRef =
isVideoResult && videoPngSaved
? { path: videoPngSaved.savedPath, url: videoPngSaved.fileUrl }
: { path: firstSaved.savedPath, url: firstSaved.fileUrl };
// Policy: generate JPEG previews only (unified for attachments + variants)
// Use PreviewSpec from VARIANT_FULL_CONFIG with central generatePreviewFromBuffer()
// VARIANT_FULL_CONFIG.preview uses maxSum: 1536 for proper sizing
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const previews = [];
if (isVideoResult) {
// Generate one JPEG preview from the last trimmed video frame
const previewFrame = videoFrames[videoFrames.length - 1];
const videoImageI = baseImageI;
const videoBaseName = `image-${baseStamp}-i${videoImageI}`;
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(previewFrame, primaryOutDir, videoPngSaved.fileName, variantPreviewSpec, { customFilename: `preview-${videoBaseName}.jpg` });
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`video preview saved (i${videoImageI}): ${previewFilePath} ${p.width}x${p.height} ${p.data.length} bytes ok=true`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`video preview build failed (i${videoImageI}): ${msg} spec=${JSON.stringify(variantPreviewSpec)}`);
}
}
if (!isVideoResult) {
for (let i = 0; i < buffers.length; i++) {
const buf = buffers[i];
const imageI = baseImageI + i; // Use fortlaufende i-Nummer for logging
try {
if (!isSupportedImageBuffer(buf)) {
try {
const magic = Buffer.from(buf.slice(0, 12) || []).toString("hex");
log(`preview skip: unsupported buffer signature (i${imageI}) magic=${magic}`);
}
catch { }
continue;
}
// Use central generatePreviewFromBuffer() with correct maxSum/maxWidth constraints
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(buf, primaryOutDir, savedFiles[i].fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`preview saved (i${imageI}): ${previewFilePath} ${p.width}x${p.height} ${p.data.length} bytes ok=true`);
}
catch (e) {
const msg = e && e.message ? String(e.message) : String(e);
log(`preview build failed (i${imageI}): ${msg} spec=${JSON.stringify(variantPreviewSpec)}`);
}
}
}
if (previews.length === 0) {
log(`previews built: count=0 (no preview created)`);
}
else {
log(`previews built: count=${previews.length}`);
}
if (previews.length > 0) {
const firstPreview = previews[0];
lastPreviewRef = {
path: firstPreview.filePath,
url: firstPreview.fileUrl,
mimeType: firstPreview.mimeType,
width: firstPreview.width,
height: firstPreview.height,
};
// Track per-chat last images and clear pending sentinel after any generation
try {
if (currentLmChatId) {
// Store with i-values for proper lookup in getAllImagesForContext
LAST_IMAGES_BY_LM_CHAT[currentLmChatId] =
imageRecordsForState.map((ir, idx) => ({
i: ir.i,
path: savedFiles[idx].savedPath,
}));
}
}
catch { }
// Update chat_media_state.json with new images (append, rolling window in orchestrator)
// This ensures the State has the correct i-numbers and nextImageI is incremented
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`state updated: appended ${imageRecordsForState.length} images, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`state update failed (non-fatal): ${String(e.message)}`);
}
}
const explicit = {};
for (const k of ALLOWED_GEN_INPUT_KEYS) {
if (Object.prototype.hasOwnProperty.call(input, k) &&
input[k] !== undefined) {
explicit[k] = input[k];
}
}
const inferenceMs = result?.metadata?.inference_time_ms;
const meta = (result?.metadata || {});
const effWidth = typeof postProcessedW === "number"
? Math.round(postProcessedW)
: typeof meta.width === "number"
? Math.round(meta.width)
: undefined;
const effHeight = typeof postProcessedH === "number"
? Math.round(postProcessedH)
: typeof meta.height === "number"
? Math.round(meta.height)
: undefined;
const imgFmtRaw = typeof meta.image_format === "string"
? String(meta.image_format)
: undefined;
const effQuality = typeof meta.quality === "string" ? String(meta.quality) : undefined;
const effSteps = typeof meta.steps === "number" ? Math.round(meta.steps) : undefined;
const summary = {
width: effWidth,
height: effHeight,
image_format: imgFmtRaw,
quality: effQuality,
...(typeof effSteps === "number" ? { steps: effSteps } : {}),
backend: resolvedName,
mode_effective: effectiveMode,
source: sourceTag || undefined,
...(typeof sourceVariantUsed === "number"
? { source_variant_used: sourceVariantUsed }
: {}),
...(typeof normalizedToLongSide === "number"
? { normalized_to_long_side: normalizedToLongSide }
: {}),
images_generated: isVideoResult ? 1 : buffers.length,
// Reference sources used (for edit mode / img2img)
...(usedReferenceMeta && usedReferenceMeta.length > 0
? {
references_used: (() => {
const moodboardCount = usedReferenceMeta.filter((r) => !r.isCanvas).length;
const moodboardWeight = moodboardCount > 0 ? 1.0 / moodboardCount : 0;
return usedReferenceMeta.map((r) => ({
source: r.type,
index: r.index,
role: r.isCanvas ? "canvas" : "moodboard",
...(r.originPath ? { source_originAbs: r.originPath } : {}),
...(r.originalName
? { source_originalName: r.originalName }
: {}),
...(!r.isCanvas && moodboardCount > 0
? { weight: moodboardWeight }
: {}),
}));
})(),
canvas_source: (() => {
const canvas = usedReferenceMeta.find((r) => r.isCanvas);
return canvas
? `${canvas.type === "attachment" ? "a" : "v"}${canvas.index}`
: null;
})(),
}
: {}),
files: {
original: firstSaved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
...(typeof inferenceMs === "number"
? { inference_time_ms: inferenceMs }
: {}),
};
// Backfill originalName for audit wherever we have originAbs.
// Rationale: originAbs is the stable attachment identity; originalName may be missing
// in some resolver paths unless explicitly persisted in chat_media_state.json.
try {
const chatWdForAudit = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : null);
if (chatWdForAudit) {
const st = await (0, core_bundle_mjs_1.readState)(chatWdForAudit);
const attachments = Array.isArray(st?.attachments)
? st.attachments
: [];
const originalNameByOriginAbs = new Map();
const originalNameByFilename = new Map();
for (const a of attachments) {
if (!a || typeof a !== "object")
continue;
const oa = typeof a.originAbs === "string" && a.originAbs.trim()
? String(a.originAbs)
: null;
const fn = typeof a.filename === "string" && a.filename.trim()
? String(a.filename)
: typeof a.origin === "string" && a.origin.trim()
? String(a.origin)
: null;
const on = typeof a.originalName === "string" && a.originalName.trim()
? String(a.originalName)
: null;
if (oa && on)
originalNameByOriginAbs.set(oa, on);
if (fn && on)
originalNameByFilename.set(fn, on);
}
if (sourceKind === "attachment" &&
sourceOriginAbs &&
!sourceOriginalName) {
sourceOriginalName =
originalNameByOriginAbs.get(sourceOriginAbs) ||
originalNameByFilename.get(path_1.default.basename(sourceOriginAbs));
}
if (Array.isArray(usedReferenceMeta) && usedReferenceMeta.length > 0) {
usedReferenceMeta = usedReferenceMeta.map((r) => {
if (r &&
r.type === "attachment" &&
r.originPath &&
typeof r.originPath === "string" &&
!r.originalName) {
const filled = originalNameByOriginAbs.get(r.originPath) ||
originalNameByFilename.get(path_1.default.basename(r.originPath));
return filled ? { ...r, originalName: filled } : r;
}
return r;
});
}
}
}
catch { }
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginals = savedFiles.map((s) => httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(s.fileName, httpBase, currentLmChatId || undefined)
: "");
// Build preview URLs pointing to preview-* files in chat working directory
const httpPreviews = savedFiles.map((_s, i) => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[i]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
});
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({
backend: resolvedName,
mode: effectiveMode,
requestId: auditRequestId,
});
// Metadata
if (currentLmChatId)
audit.setChatId(currentLmChatId);
// === USER REQUEST (what the user sent) ===
const userRequest = {};
if (input?.prompt)
userRequest.prompt = input.prompt;
if (input?.mode)
userRequest.mode = input.mode;
if (input?.canvas)
userRequest.canvas = input.canvas;
if (input?.moodboard)
userRequest.moodboard = input.moodboard;
if (input?.model)
userRequest.model = input.model;
if (input?.width)
userRequest.width = input.width;
if (input?.height)
userRequest.height = input.height;
if (typeof input?.seed === "number") {
userRequest.seed = input.seed;
}
if (Object.prototype.hasOwnProperty.call(input, "seed_mode")) {
userRequest.seed_mode = input.seed_mode;
}
else if (Object.prototype.hasOwnProperty.call(input, "seedMode")) {
userRequest.seed_mode = input.seedMode;
}
if (input?.imageFormat)
userRequest.imageFormat = input.imageFormat;
if (input?.quality)
userRequest.quality = input.quality;
if (input?.variants)
userRequest.variants = input.variants;
audit.setUserRequest(userRequest);
// === RENDER TARGET (Step 0+1) ===
const renderTarget = {};
if (typeof requestedRawW === "number" ||
typeof requestedRawH === "number") {
renderTarget.requested_raw = {
...(typeof requestedRawW === "number"
? { width: requestedRawW }
: {}),
...(typeof requestedRawH === "number"
? { height: requestedRawH }
: {}),
};
}
if (typeof requestedEffectiveW === "number" ||
typeof requestedEffectiveH === "number") {
renderTarget.requested_effective = {
...(typeof requestedEffectiveW === "number"
? { width: requestedEffectiveW }
: {}),
...(typeof requestedEffectiveH === "number"
? { height: requestedEffectiveH }
: {}),
};
}
if (typeof requestedRawW === "number" &&
typeof requestedRawH === "number") {
renderTarget.needs_upscaler = _dtNeedsUpscaler;
}
if (Object.keys(renderTarget).length > 0) {
audit.setRenderTarget(renderTarget);
}
// === INPUTS (Step 2: Canvas + Moodboard) ===
const inputs = {};
// Canvas from sourcePreprocess (single-image i2i) or usedReferencePreprocess
if (sourcePreprocess && sourcePreprocess.original) {
inputs.canvas = {
notation: sourceTag || undefined,
source_type: sourceKind || undefined,
file_name: sourceFileName || undefined,
original_name: sourceOriginalName || undefined,
origin_path: sourceOriginAbs || undefined,
original: _internal?.auditSourceOverride?.originalDims ?? {
width: sourcePreprocess.original.width,
height: sourcePreprocess.original.height,
bytes: sourcePreprocess.original.bytes,
},
adjusted: sourcePreprocess.adjusted
? {
width: sourcePreprocess.adjusted.width,
height: sourcePreprocess.adjusted.height,
bytes: sourcePreprocess.adjusted.bytes,
}
: undefined,
};
}
// Multi-reference (edit mode): usedReferencePreprocess
if (usedReferencePreprocess && usedReferencePreprocess.length > 0) {
const canvasRef = usedReferencePreprocess.find((r) => r.role === "canvas");
const moodboardRefs = usedReferencePreprocess.filter((r) => r.role === "moodboard");
const moodboardCount = moodboardRefs.length;
if (canvasRef) {
const notation = `${canvasRef.type === "attachment"
? "a"
: canvasRef.type === "image"
? "i"
: "p"}${canvasRef.index}`;
inputs.canvas = {
notation,
source_type: canvasRef.type,
original_name: canvasRef.originalName || undefined,
origin_path: canvasRef.originPath || undefined,
original: canvasRef.preprocess?.original,
adjusted: canvasRef.preprocess?.adjusted,
};
}
if (moodboardRefs.length > 0) {
inputs.moodboard = moodboardRefs.map((r) => {
const notation = `${r.type === "attachment" ? "a" : r.type === "image" ? "i" : "p"}${r.index}`;
return {
notation,
source_type: r.type,
original_name: r.originalName || undefined,
origin_path: r.originPath || undefined,
original: r.preprocess?.original,
adjusted: r.preprocess?.adjusted,
weight: moodboardCount > 0 ? 1.0 / moodboardCount : undefined,
};
});
}
}
// Mask info (inpaint / outpaint) — uses auditMaskBuf (post-resize, actual dims sent)
if (auditMaskBuf && auditMaskBuf.length >= 24) {
const mb = auditMaskBuf;
// Read PNG dimensions from IHDR (bytes 16–23, big-endian uint32)
const maskW = mb.readUInt32BE(16);
const maskH = mb.readUInt32BE(20);
inputs.mask = {
bytes: mb.length,
width: maskW,
height: maskH,
...((_internal?.cropMeta) && { crop_percent: {
left: _internal.cropMeta.left,
top: _internal.cropMeta.top,
right: _internal.cropMeta.right,
bottom: _internal.cropMeta.bottom,
} }),
};
}
if (Object.keys(inputs).length > 0) {
audit.setInputs(inputs);
}
// === OUTPUT (Step 3+4) ===
const output = {};
// Backend returned dimensions
if (typeof backendReturnedW === "number" &&
typeof backendReturnedH === "number") {
output.backend_returned = {
width: backendReturnedW,
height: backendReturnedH,
};
}
// Post-processed dimensions
if (typeof effWidth === "number" && typeof effHeight === "number") {
output.post_processed = { width: effWidth, height: effHeight };
}
// Inference time
if (typeof inferenceMs === "number") {
output.inference_time_ms = inferenceMs;
}
// Model used
if (typeof meta.model === "string" && meta.model.trim()) {
output.model_used = meta.model;
}
// Model origin + presets
// Read overlay info from service result metadata (authoritative source)
try {
const meta = result?.metadata || {};
// overlay_source and overlay_preset come from the service layer
if (meta.overlay_source) {
output.overlay_source = meta.overlay_source;
}
if (meta.overlay_preset) {
output.overlay_preset = meta.overlay_preset;
}
if (typeof meta.defaults_used === "string" && meta.defaults_used.trim()) {
output.defaults_used = meta.defaults_used;
}
if (typeof meta.overlay_lookup_mode === "string" &&
meta.overlay_lookup_mode.trim()) {
output.overlay_lookup_mode = meta.overlay_lookup_mode;
}
if (typeof meta.i2i_profile === "string" && meta.i2i_profile.trim()) {
output.i2i_profile = meta.i2i_profile;
}
if (typeof meta.strength_used === "number" && Number.isFinite(meta.strength_used)) {
output.strength_used = meta.strength_used;
}
if (typeof meta.steps_used === "number" && Number.isFinite(meta.steps_used)) {
output.steps_used = meta.steps_used;
}
if (typeof meta.sampler_used === "string" && meta.sampler_used.trim()) {
output.sampler_used = meta.sampler_used;
}
if (typeof meta.guidance_scale_used === "number" &&
Number.isFinite(meta.guidance_scale_used)) {
output.guidance_scale_used = meta.guidance_scale_used;
}
if (typeof meta.shift_used === "number" && Number.isFinite(meta.shift_used)) {
output.shift_used = meta.shift_used;
}
if (typeof meta.resolution_dependent_shift_used === "boolean") {
output.resolution_dependent_shift_used = meta.resolution_dependent_shift_used;
}
if (typeof meta.compression_artifacts_used === "string") {
output.compression_artifacts_used = meta.compression_artifacts_used;
}
if (typeof meta.compression_artifacts_quality_used === "number" && Number.isFinite(meta.compression_artifacts_quality_used)) {
output.compression_artifacts_quality_used = meta.compression_artifacts_quality_used;
}
// Model used (from service - the actual model that was sent to Draw Things)
if (typeof meta.model === "string" && meta.model.trim()) {
output.model_used = path_1.default.basename(meta.model);
}
// LoRAs used (from service - actual LoRA files validated and sent)
if (Array.isArray(meta.loras_used) && meta.loras_used.length > 0) {
output.loras_used = meta.loras_used.map((f) => path_1.default.basename(f));
}
// Seed used (from service - effective seed after defaults/overlays)
if (typeof meta.seed === "number" && Number.isFinite(meta.seed)) {
output.seed = meta.seed;
}
if (typeof meta.seed_mode === "string" && meta.seed_mode.trim()) {
output.seed_mode = meta.seed_mode;
}
if (typeof meta.seed_source === "string" && meta.seed_source.trim()) {
output.seed_source = meta.seed_source;
}
if (typeof meta.seed_mode_source === "string" &&
meta.seed_mode_source.trim()) {
output.seed_mode_source = meta.seed_mode_source;
}
}
catch { }
// Prompt used (prefer backend's prompt_used, fallback to user input)
const metaPrompt = result?.metadata?.prompt_used;
const promptUsed = typeof metaPrompt === "string" && metaPrompt.trim()
? metaPrompt
: typeof input?.prompt === "string"
? input.prompt
: undefined;
if (promptUsed) {
output.prompt_used = promptUsed;
}
// Prompt origin
const backendOriginRaw = result?.metadata?.prompt_origin;
const userPromptInput = typeof input?.prompt === "string"
? String(input.prompt).trim()
: "";
if (backendOriginRaw === "user" || userPromptInput) {
output.prompt_origin = "user";
}
else {
output.prompt_origin = "default";
}
// Images (saved files)
if (Array.isArray(savedFiles) && savedFiles.length > 0) {
output.images = savedFiles.map((s, i) => {
const pv = previews[i];
const iMatch = /[-]i(\d+)\.(png|jpe?g|webp)$/i.exec(s?.fileName || "");
const idx = iMatch ? parseInt(iMatch[1], 10) : i + 1;
return {
i: idx,
path: s.savedPath,
url: s.fileUrl,
bytes: s.size,
...(httpOriginals[i]
? { http_url: httpOriginals[i] }
: {}),
...(pv
? { preview_path: pv.filePath, preview_url: pv.fileUrl }
: {}),
...(httpPreviews[i]
? { http_preview_url: httpPreviews[i] }
: {}),
};
});
}
audit.setOutput(output);
await audit.write();
if (pendingAudit2) {
try {
await pendingAudit2.write();
}
catch (e2) {
log(`zoom-pass audit2 write failed: ${String(e2)}`);
}
pendingAudit2 = null;
}
}
catch (e) {
log(`audit logging error: ${String(e)}`);
}
try {
const p = process.env.HTTP_SERVER_PORT;
log(`[httpServer] generate_image: external server ${httpBase ? "healthy" : "unavailable"}${p ? ` (port=${p})` : ""}.`);
}
catch { }
const extractStableVariantV = (fileName) => {
try {
const m = /[-]i(\d+)\.(png|jpe?g|webp|mov)$/i.exec(String(fileName || "")) ||
/-v(\d+)\.(png|jpe?g|webp|mov)$/i.exec(String(fileName || ""));
if (!m)
return undefined;
const n = parseInt(m[1], 10);
return Number.isFinite(n) && n > 0 ? n : undefined;
}
catch {
return undefined;
}
};
const variantLinksText = savedFiles.length > 0
? savedFiles
.map((s, i) => {
const stableV = extractStableVariantV(s.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1;
const httpPreviewUrl = httpPreviews[i];
const fallback = previews[i]?.fileUrl || s.fileUrl;
const url = httpPreviewUrl ? httpPreviewUrl : fallback;
return `Preview i${stableV}: ${url}`;
})
.join(" | ")
: "";
// Originals are saved directly to chat working directory.
const originalLinksText = savedFiles.length > 0
? savedFiles
.map((s, i) => {
const stableV = extractStableVariantV(s.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1;
const httpUrl = httpOriginals[i];
const url = httpUrl ? httpUrl : s.fileUrl;
return `Original i${stableV}: ${url}`;
})
.join(" | ")
: "";
const note = typeof requestedVariants === "number" &&
requestedVariants !== usedVariants
? `Note: variants=${requestedVariants} was clamped to ${usedVariants}.`
: null;
try {
const invMs = result?.metadata?.inference_time_ms;
const wLog = typeof effWidth === "number"
? effWidth
: typeof postProcessedW === "number"
? postProcessedW
: typeof meta.width === "number"
? Math.round(meta.width)
: undefined;
const hLog = typeof effHeight === "number"
? effHeight
: typeof postProcessedH === "number"
? postProcessedH
: typeof meta.height === "number"
? Math.round(meta.height)
: undefined;
log(`generation summary: backend=${resolvedName} mode=${effectiveMode} width=${typeof wLog === "number" ? wLog : "-"} height=${typeof hLog === "number" ? hLog : "-"} previews=${previews.length} variants=${buffers.length} inferenceMs=${invMs ?? "-"}`);
}
catch { }
const reviewHint = "Carefully examine the preview and comment on how well it matches your prompt. Do not assume it does.";
const { files: _files, ...summaryNoFilesBase } = summary;
const summaryNoFiles = await (async () => {
const modelUsedBasename = typeof meta.model === "string" && meta.model.trim()
? path_1.default.basename(meta.model)
: undefined;
if (!modelUsedBasename)
return summaryNoFilesBase;
let matchingPresets;
const overlaySourceFromMeta = meta.overlay_source;
const overlayPresetFromMeta = meta.overlay_preset;
if (overlayPresetFromMeta && overlaySourceFromMeta) {
const dotIdx = overlayPresetFromMeta.indexOf(".");
const presetMode = dotIdx >= 0 ? overlayPresetFromMeta.slice(0, dotIdx) : undefined;
const presetModelId = dotIdx >= 0 ? overlayPresetFromMeta.slice(dotIdx + 1) : undefined;
matchingPresets = [{
mode: presetMode,
modelId: presetModelId,
preset: overlayPresetFromMeta,
overlaySource: overlaySourceFromMeta,
...(overlaySourceFromMeta === "custom" ? { customConfig: overlayPresetFromMeta } : {}),
}];
}
else if (overlaySourceFromMeta === "modelOverlay") {
try {
const { resolveImageModelInfoFromModelUsed } = await import("../helpers/imageModelMeta.js");
const info = resolveImageModelInfoFromModelUsed(modelUsedBasename, {
mode: effectiveMode,
});
const overlayOnly = (info?.presets || []).filter((p) => p.overlaySource === "modelOverlay");
if (overlayOnly.length > 0)
matchingPresets = overlayOnly;
}
catch { }
}
return {
...summaryNoFilesBase,
model_used: modelUsedBasename,
...(matchingPresets ? { model_presets: matchingPresets } : {}),
};
})();
if (previewInChat && previews.length > 0) {
// The preview files were already written to the primary chat working directory.
// Reuse those file names to avoid duplicates.
const imageContents = previews.map((p, i) => {
const fname = String(p.fileName || "");
const stableV = extractStableVariantV(fname) ||
extractStableVariantV(savedFiles[i]?.fileName) ||
i + 1;
return {
type: "image",
fileName: String(savedFiles[i]?.fileName || fname),
mimeType: p.mimeType,
markdown: `})`,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
};
});
return {
content: [
...(note ? [{ type: "text", text: note }] : []),
...imageContents,
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
...(note ? [{ type: "text", text: note }] : []),
...(() => {
if (httpOriginals.length > 0 && httpOriginals[0]) {
// PREVIEW_IN_CHAT = false: simplified response without inline previews.
// Markdown will be injected by orchestrator after tool call.
// Still provide a hint so the model reviews the output quality.
return httpOriginals.map((u, i) => ({
type: "text",
text: `${isVideoResult ? "Video" : "Image"} i${extractStableVariantV(savedFiles[i]?.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1} successfully generated.`,
$hint: reviewHint,
}));
}
const count = typeof usedVariants === "number" && usedVariants > 0
? usedVariants
: 1;
return Array.from({ length: count }, (_, i) => ({
type: "text",
text: `${isVideoResult ? "Video" : "Image"} i${extractStableVariantV(savedFiles[i]?.fileName) ||
extractStableVariantV(previews[i]?.fileName) ||
i + 1} successfully generated.`,
$hint: reviewHint,
}));
})(),
...(variantLinksText
? [{ type: "text", text: variantLinksText }]
: []),
...(originalLinksText
? [{ type: "text", text: originalLinksText }]
: []),
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`generate_image error: ${error instanceof Error ? error.message : String(error)}`);
await logError(error);
return {
content: [
{ type: "text", text: "Failed to generate image: backend error" },
],
isError: true,
};
}
}
exports.ToolSchemas = {
generate_image: core_bundle_mjs_1.GenerateToolParamsShapeMinimal,
crop: core_bundle_mjs_1.CropToolParamsShape,
mask: core_bundle_mjs_1.CropToolParamsShape,
zoom_in: core_bundle_mjs_1.ZoomInToolParamsShape,
inpaint: core_bundle_mjs_1.InpaintToolParamsShape,
outpaint: core_bundle_mjs_1.OutpaintToolParamsShape,
refine: core_bundle_mjs_1.RefineToolParamsShape,
};
// ─────────────────────────────────────────────────────────────────────────────
// applyCropFormat: compute final crop percentages from explicit values + imageFormat.
// resolveBoxOverride: resolve a per-box crop side value for multi-region detect.
// override — the raw field value from input: scalar (applies to all boxes) |
// array (parallel to detectLabel, null entry = use detectDefault) | undefined.
// index — the box index within the detectLabel array.
// detectDefault — the percent value returned by resolveDetectionCrop for this side.
// Returns the value to pass into applyCropFormat for this specific box (or undefined = let
// applyCropFormat treat it as "unset", which will then fall through to detectDefault via the
// caller setting the field when undefined).
function resolveBoxOverride(override, index, detectDefault) {
if (override === undefined)
return undefined; // no override → caller uses detectDefault
if (Array.isArray(override)) {
const entry = override[index];
if (entry === null || entry === undefined)
return undefined; // null = keep detection
return entry;
}
// scalar: same value for every box
return override;
}
// If imageFormat is set AND not all 4 sides are given, derive missing sides from AR.
// If imageFormat is NOT set, use explicit values only (0 for unset sides).
// If all 4 sides are explicitly given, imageFormat is ignored.
// Each crop side accepts number (→ %) or string like "10", "10%", "10 %", "120px", "120 px".
// Returns { left, right, top, bottom } as percentages, or an error string.
// ─────────────────────────────────────────────────────────────────────────────
function parseCropSide(raw) {
if (raw === undefined)
return undefined;
if (typeof raw === "number") {
if (raw < 0)
return new Error(`Invalid crop value ${raw}: must be >= 0.`);
return { value: raw, unit: "%" };
}
const m = String(raw).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(px|%)?$/i);
if (!m)
return new Error(`Invalid crop value "${raw}": expected a number optionally followed by "%" or "px".`);
const num = parseFloat(m[1]);
if (num < 0)
return new Error(`Invalid crop value "${raw}": must be >= 0.`);
const unitStr = (m[2] ?? "").toLowerCase();
return { value: num, unit: unitStr === "px" ? "px" : "%" };
}
async function applyCropFormat(srcBuf, input) {
const pLeft = parseCropSide(input.cropLeft);
const pRight = parseCropSide(input.cropRight);
const pTop = parseCropSide(input.cropTop);
const pBottom = parseCropSide(input.cropBottom);
for (const [name, p] of [["cropLeft", pLeft], ["cropRight", pRight], ["cropTop", pTop], ["cropBottom", pBottom]]) {
if (p instanceof Error)
return p.message;
}
const leftSet = pLeft !== undefined;
const rightSet = pRight !== undefined;
const topSet = pTop !== undefined;
const bottomSet = pBottom !== undefined;
const hasPx = pLeft?.unit === "px" || pRight?.unit === "px" ||
pTop?.unit === "px" || pBottom?.unit === "px";
const needsImageDims = hasPx || !!input.imageFormat || input.frameAdjust !== undefined;
let imgW = 0, imgH = 0;
if (needsImageDims) {
const size = await (0, core_bundle_mjs_1.getSize)(srcBuf);
imgW = size.width;
imgH = size.height;
}
function toPct(p, dim) {
if (!p)
return 0;
return p.unit === "px" ? (dim > 0 ? (p.value / dim) * 100 : 0) : p.value;
}
let left = toPct(pLeft, imgW);
let right = toPct(pRight, imgW);
let top = toPct(pTop, imgH);
let bottom = toPct(pBottom, imgH);
if (!(leftSet && rightSet && topSet && bottomSet) && input.imageFormat) {
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetRatio = ratioMap[input.imageFormat]; // W/H
// imgW, imgH already computed above (needsImageDims covers this branch)
const hFixed = leftSet && rightSet;
const vFixed = topSet && bottomSet;
const cropW_px = imgW * (1 - left / 100 - right / 100);
const cropH_px = imgH * (1 - top / 100 - bottom / 100);
if (hFixed) {
const cropH_needed = cropW_px / targetRatio;
if (topSet) {
bottom = Math.max(0, (imgH * (1 - top / 100) - cropH_needed) / imgH * 100);
}
else if (bottomSet) {
top = Math.max(0, (imgH * (1 - bottom / 100) - cropH_needed) / imgH * 100);
}
else {
const margin = Math.max(0, (imgH - cropH_needed) / 2 / imgH * 100);
top = margin;
bottom = margin;
}
}
else if (vFixed) {
const cropW_needed = cropH_px * targetRatio;
if (leftSet) {
right = Math.max(0, (imgW * (1 - left / 100) - cropW_needed) / imgW * 100);
}
else if (rightSet) {
left = Math.max(0, (imgW * (1 - right / 100) - cropW_needed) / imgW * 100);
}
else {
const margin = Math.max(0, (imgW - cropW_needed) / 2 / imgW * 100);
left = margin;
right = margin;
}
}
else {
const currentRatio = cropW_px / cropH_px;
if (Math.abs(currentRatio - targetRatio) > 1e-6) {
if (currentRatio > targetRatio) {
const cropW_needed = cropH_px * targetRatio;
if (leftSet) {
right = Math.max(0, (imgW * (1 - left / 100) - cropW_needed) / imgW * 100);
}
else if (rightSet) {
left = Math.max(0, (imgW * (1 - right / 100) - cropW_needed) / imgW * 100);
}
else {
const margin = Math.max(0, (imgW - cropW_needed) / 2 / imgW * 100);
left = margin;
right = margin;
}
}
else {
const cropH_needed = cropW_px / targetRatio;
if (topSet) {
bottom = Math.max(0, (imgH * (1 - top / 100) - cropH_needed) / imgH * 100);
}
else if (bottomSet) {
top = Math.max(0, (imgH * (1 - bottom / 100) - cropH_needed) / imgH * 100);
}
else {
const margin = Math.max(0, (imgH - cropH_needed) / 2 / imgH * 100);
top = margin;
bottom = margin;
}
}
}
}
if (left + right >= 100 || top + bottom >= 100) {
return `imageFormat '${input.imageFormat}' cannot be achieved with the given crop values.`;
}
}
else if (input.imageFormat) {
// All 4 sides are set (e.g. from detectLabel) — apply symmetric AR expansion,
// matching the zoom-in algorithm: always expand the shorter dimension outward.
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetRatio = ratioMap[input.imageFormat];
if (targetRatio !== undefined) {
const regionW = imgW * (1 - left / 100 - right / 100);
const regionH = imgH * (1 - top / 100 - bottom / 100);
const regionAR = regionW / regionH;
if (Math.abs(regionAR - targetRatio) > 0.005) {
if (regionAR < targetRatio) {
// Region too tall → expand width symmetrically.
const totalExtraPct = ((regionH * targetRatio - regionW) / imgW) * 100;
const halfExtra = totalExtraPct / 2;
const leftReduction = Math.min(left, halfExtra);
const rightReduction = Math.min(right, halfExtra + (halfExtra - leftReduction));
const leftReduction2 = Math.min(left, halfExtra + (halfExtra - rightReduction));
left = Math.max(0, left - leftReduction2);
right = Math.max(0, right - rightReduction);
}
else {
// Region too wide → expand height symmetrically.
const totalExtraPct = ((regionW / targetRatio - regionH) / imgH) * 100;
const halfExtra = totalExtraPct / 2;
const topReduction = Math.min(top, halfExtra);
const bottomReduction = Math.min(bottom, halfExtra + (halfExtra - topReduction));
const topReduction2 = Math.min(top, halfExtra + (halfExtra - bottomReduction));
top = Math.max(0, top - topReduction2);
bottom = Math.max(0, bottom - bottomReduction);
}
}
}
}
// Apply frameAdjust to the final crop region — always the last step.
// Each side is clamped independently at 0 when the expansion overshoots the image boundary.
if (input.frameAdjust !== undefined) {
const cropW_px = imgW * (1 - left / 100 - right / 100);
const cropH_px = imgH * (1 - top / 100 - bottom / 100);
const cropDiag = Math.hypot(cropW_px, cropH_px);
let d_px;
if (typeof input.frameAdjust === "string") {
const m = String(input.frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (!m)
return `Invalid frameAdjust value: "${input.frameAdjust}"`;
const val = parseFloat(m[1]);
d_px = m[2]?.toLowerCase() === "px" ? val : (val / 100) * cropDiag;
}
else {
d_px = (input.frameAdjust / 100) * cropDiag;
}
const dLR = (d_px / imgW) * 100;
const dTB = (d_px / imgH) * 100;
left = Math.max(0, left - dLR);
right = Math.max(0, right - dLR);
top = Math.max(0, top - dTB);
bottom = Math.max(0, bottom - dTB);
}
if (left + right >= 100) {
return `cropLeft + cropRight must not cover the full image width.`;
}
if (top + bottom >= 100) {
return `cropTop + cropBottom must not cover the full image height.`;
}
return { left, right, top, bottom };
}
// ─────────────────────────────────────────────────────────────────────────────
// findDetectRecordForCanvas: locate the detect_object record relevant to rawCanvas — either
// rawCanvas IS the detect_object's own iN result, or rawCanvas is the ORIGINAL source and some
// later detect_object record references it via detectSource (latest wins, case-insensitive).
// Shared by expandDetectIndices/resolveDetectionCrop's own lookups and autoSelectDetectLabel().
// ─────────────────────────────────────────────────────────────────────────────
function findDetectRecordForCanvas(imageRecords, rawCanvas) {
if (!rawCanvas)
return undefined;
const pref = parsePrefixedNotation(rawCanvas);
if (pref?.pool === "image") {
const candidate = imageRecords.find((r) => r?.i === pref.index);
if (candidate && Array.isArray(candidate.detections))
return candidate;
}
const lc = rawCanvas.toLowerCase();
const candidates = imageRecords
.filter((r) => typeof r?.detectSource === "string" &&
r.detectSource.toLowerCase() === lc &&
Array.isArray(r.detections))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0));
return candidates[0];
}
// ─────────────────────────────────────────────────────────────────────────────
// autoSelectDetectLabel: when detectLabel is omitted and rawCanvas resolves to a prior
// detect_object record (rawCanvas itself, or the ORIGINAL source of a later detect_object call —
// see findDetectRecordForCanvas), auto-selects its label when exactly one distinct label exists
// (mirrors detectIndex's own default of 0). Throws when 2+ distinct labels exist, listing them.
// Returns the record's single label, or undefined (no-op — canvas has no/ambiguous-free detections).
// ─────────────────────────────────────────────────────────────────────────────
function autoSelectDetectLabel(imageRecords, rawCanvas) {
const detectRec = findDetectRecordForCanvas(imageRecords, rawCanvas);
if (!detectRec || !Array.isArray(detectRec.detections) || detectRec.detections.length === 0)
return undefined;
const distinctLabels = [...new Set(detectRec.detections.map((d) => d?.label).filter(Boolean))];
if (distinctLabels.length > 1) {
throw new Error(`Ambiguous — canvas '${rawCanvas}' contains ${detectRec.detections.length} detected regions. ` +
`Specify detectLabel to select one. Available labels: ${distinctLabels.join(", ")}.`);
}
return distinctLabels.length === 1 ? distinctLabels[0] : undefined;
}
// ─────────────────────────────────────────────────────────────────────────────
// expandDetectIndices: count how many detections match a single label in the
// detect_object record for rawCanvas. Returns [0..N-1], or [0] if only 1 match.
// Used for Option-A auto-expansion: single detectLabel + no explicit detectIndex
// → automatically iterate all instances of that label.
// ─────────────────────────────────────────────────────────────────────────────
function expandDetectIndices(imageRecords, rawCanvas, label) {
const detectRec = findDetectRecordForCanvas(imageRecords, rawCanvas);
if (!detectRec)
return [0];
const count = detectRec.detections.filter((d) => typeof d?.label === "string" && d.label.toLowerCase() === label.toLowerCase()).length;
return count > 1 ? Array.from({ length: count }, (_, i) => i) : [0];
}
// ─────────────────────────────────────────────────────────────────────────────
// ─────────────────────────────────────────────────────────────────────────────
// resolveDetectionCrop: find a detection by label in a prior detect_object record
// and return crop percentages, optionally adjusted by frameAdjust.
// ─────────────────────────────────────────────────────────────────────────────
function resolveDetectionCrop(imageRecords, rawCanvas, detectLabel, detectIndex, frameAdjust) {
// --- 1. find detect_object record ---
let detectRec;
let canvasWasDetectRecord = false;
let srcCanvas = rawCanvas;
if (rawCanvas) {
// Check if rawCanvas is itself an iN detect_object record
const pref = parsePrefixedNotation(rawCanvas);
if (pref?.pool === "image") {
const candidate = imageRecords.find((r) => r?.i === pref.index);
if (candidate && Array.isArray(candidate.detections)) {
detectRec = candidate;
canvasWasDetectRecord = true;
srcCanvas = typeof detectRec.detectSource === "string" ? detectRec.detectSource : undefined;
}
}
}
if (!detectRec && rawCanvas) {
// Search for detect_object record referencing rawCanvas as source
const lc = rawCanvas.toLowerCase();
const candidates = imageRecords
.filter((r) => typeof r?.detectSource === "string" &&
r.detectSource.toLowerCase() === lc &&
Array.isArray(r.detections))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0));
if (candidates.length > 0) {
detectRec = candidates[0];
canvasWasDetectRecord = false;
srcCanvas = rawCanvas;
}
}
if (!detectRec) {
return {
ok: false,
error: `No detect_object result found for canvas '${rawCanvas ?? "(auto)"}'. Run detect_object on the source image first.`,
};
}
// --- 2. find detection by label ---
const detections = detectRec.detections;
const matching = detections.filter((d) => typeof d?.label === "string" && d.label.toLowerCase() === detectLabel.toLowerCase());
if (matching.length === 0) {
const allLabels = [...new Set(detections.map((d) => d?.label).filter(Boolean))].join(", ");
return {
ok: false,
error: `Label '${detectLabel}' not found in detections. Available: ${allLabels || "(none)"}`,
};
}
if (detectIndex >= matching.length) {
return {
ok: false,
error: `detectIndex ${detectIndex} out of range for label '${detectLabel}' (${matching.length} detection(s)).`,
};
}
const det = matching[detectIndex];
// --- 3. apply frameAdjust or use pre-computed crop percentages ---
if (frameAdjust === undefined) {
const crop = det.crop;
return {
ok: true,
cropLeft: typeof crop?.cropLeft === "number" ? crop.cropLeft : 0,
cropRight: typeof crop?.cropRight === "number" ? crop.cropRight : 0,
cropTop: typeof crop?.cropTop === "number" ? crop.cropTop : 0,
cropBottom: typeof crop?.cropBottom === "number" ? crop.cropBottom : 0,
srcCanvas,
canvasWasDetectRecord,
};
}
// frameAdjust requires image dimensions stored on the detect record
const imgW = typeof detectRec.imageWidth === "number" ? detectRec.imageWidth : 0;
const imgH = typeof detectRec.imageHeight === "number" ? detectRec.imageHeight : 0;
if (imgW <= 0 || imgH <= 0) {
return {
ok: false,
error: `frameAdjust requires imageWidth/imageHeight in the detect record. Re-run detect_object to update the record.`,
};
}
// Bbox raw coordinates
const bbox = det.bbox;
const bx1 = typeof bbox?.x1 === "number" ? bbox.x1 : 0;
const by1 = typeof bbox?.y1 === "number" ? bbox.y1 : 0;
const bx2 = typeof bbox?.x2 === "number" ? bbox.x2 : imgW;
const by2 = typeof bbox?.y2 === "number" ? bbox.y2 : imgH;
// Parse frameAdjust → per-axis pixel deltas.
// %-value is relative to the respective bbox dimension (width for X, height
// for Y) so the total box grows by pct% in each dimension, preserving AR.
// px-value is an absolute margin applied uniformly on all four sides.
const bboxW = bx2 - bx1;
const bboxH = by2 - by1;
let dX_px;
let dY_px;
if (typeof frameAdjust === "string") {
const m = String(frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (!m) {
return { ok: false, error: `Invalid frameAdjust value: "${frameAdjust}"` };
}
const val = parseFloat(m[1]);
if (m[2]?.toLowerCase() === "px") {
dX_px = Math.round(val);
dY_px = Math.round(val);
}
else {
dX_px = Math.round((val / 100) * bboxW / 2);
dY_px = Math.round((val / 100) * bboxH / 2);
}
}
else {
dX_px = Math.round((frameAdjust / 100) * bboxW / 2);
dY_px = Math.round((frameAdjust / 100) * bboxH / 2);
}
// Apply adjustment. Clamp to [0, imgW] / [0, imgH] (inclusive — bbox
// coordinates are exclusive-end so x2=imgW is valid and must not become
// imgW-1, which would leave a spurious 1-pixel crop at a border edge).
let x1 = Math.max(0, bx1 - dX_px);
let x2 = Math.min(imgW, bx2 + dX_px);
let y1 = Math.max(0, by1 - dY_px);
let y2 = Math.min(imgH, by2 + dY_px);
// Ensure at least 1×1
if (x2 <= x1)
x2 = Math.min(imgW, x1 + 1);
if (y2 <= y1)
y2 = Math.min(imgH, y1 + 1);
// Convert back to crop percentages
return {
ok: true,
cropLeft: (x1 / imgW) * 100,
cropRight: ((imgW - x2) / imgW) * 100,
cropTop: (y1 / imgH) * 100,
cropBottom: ((imgH - y2) / imgH) * 100,
srcCanvas,
canvasWasDetectRecord,
};
}
// handleCrop: crop an image by percentage from each side, save as a new variant
// ─────────────────────────────────────────────────────────────────────────────
async function handleCrop(pluginParams, requestId) {
await ensureBackendReady().catch((e) => {
log(`[crop] ensureBackendReady failed: ${String(e)}`);
});
try {
const parsed = core_bundle_mjs_1.CropToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid crop parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
let currentLmChatId = null;
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
return {
content: [{ type: "text", text: "Failed to resolve LM Studio chat working directory." }],
isError: true,
};
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Load source state
const st = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const images = imageRecords
.filter((r) => r && typeof r.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0))
.map((r) => ({ i: r.i || 1, path: path_1.default.join(primaryOutDir, r.filename) }));
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variants = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(primaryOutDir, v.filename) }));
// Resolve canvas notation
let rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
let srcBuf;
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Runs before the
// detectLabel handling below, so cropLeft/cropRight/cropTop/cropBottom overrides (null = keep
// detection value) still apply.
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = [autoLabel];
}
// detectLabel: resolve crop coordinates from a prior detect_object run
if (Array.isArray(input.detectLabel) && input.detectLabel.length > 0) {
if (input.detectLabel.length > 1) {
return {
content: [{ type: "text", text: "crop supports only a single detectLabel. Use mask + inpaint for multi-region workflows." }],
isError: true,
};
}
const label = input.detectLabel[0];
const indices = Array.isArray(input.detectIndex) ? input.detectIndex : (input.detectIndex !== undefined ? [input.detectIndex] : []);
const detectIdx = indices[0] ?? 0;
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust is applied as the final step in applyCropFormat
);
if (!detectResult.ok) {
return {
content: [{ type: "text", text: detectResult.error }],
isError: true,
};
}
// Detect provides defaults; per-box overrides (scalar or array[0]) take precedence.
const oL = resolveBoxOverride(input.cropLeft, 0, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, 0, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, 0, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, 0, detectResult.cropBottom);
input.cropLeft = oL !== undefined ? oL : detectResult.cropLeft;
input.cropRight = oR !== undefined ? oR : detectResult.cropRight;
input.cropTop = oT !== undefined ? oT : detectResult.cropTop;
input.cropBottom = oB !== undefined ? oB : detectResult.cropBottom;
// frameAdjust is kept in input — applyCropFormat applies it last, clamping each side independently.
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
rawCanvas = detectResult.srcCanvas;
input.canvas = detectResult.srcCanvas;
}
}
// Auto-redirect: if canvas points to a detect_object result and no detectLabel is given, use detectSource
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const _autoRedirectPref = parsePrefixedNotation(rawCanvas);
if (_autoRedirectPref?.pool === "image") {
const _autoRedirectRec = imageRecords.find((r) => r?.i === _autoRedirectPref.index);
if (typeof _autoRedirectRec?.sourceTool === "string" &&
_autoRedirectRec.sourceTool.includes("detect_object") &&
typeof _autoRedirectRec.detectSource === "string") {
rawCanvas = _autoRedirectRec.detectSource;
input.canvas = _autoRedirectRec.detectSource;
}
}
}
// Auto-redirect: if canvas points to a mask result, crop the original unlined image instead.
// The CYAN annotation lines would otherwise appear at the crop boundary in the result.
if (rawCanvas) {
const _maskRedirectPref = parsePrefixedNotation(rawCanvas);
if (_maskRedirectPref?.pool === "image") {
const _maskRec = imageRecords.find((r) => r?.i === _maskRedirectPref.index);
if (typeof _maskRec?.sourceTool === "string" &&
_maskRec.sourceTool.includes("/mask") &&
typeof _maskRec.cropSource === "string") {
log(`[crop] canvas ${rawCanvas} is a mask result — redirecting to cropSource ${_maskRec.cropSource}`);
rawCanvas = _maskRec.cropSource;
input.canvas = _maskRec.cropSource;
}
}
}
try {
if (rawCanvas) {
const pref = parsePrefixedNotation(rawCanvas);
if (pref) {
if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${pref.index} not found.`);
}
srcBuf = lm.buffer;
}
else if (pref.pool === "variant") {
const found = variants.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else if (pref.pool === "image") {
const found = images.find((img) => img.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
srcBuf = await resolvePictureBuffer(found, primaryOutDir, pref.index);
}
}
else {
const pathToken = await resolveCanvasPathToken(rawCanvas, primaryOutDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
srcBuf = await fs_1.default.promises.readFile(pathToken);
}
}
else {
// Auto-select single source
const total = attachments.length + variants.length + images.length + pictures.length;
if (total === 0)
throw new Error("No source image available.");
if (total > 1)
throw new Error("Ambiguous source — specify canvas explicitly.");
if (attachments.length === 1) {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: typeof attachments[0]?.a === "number" ? attachments[0].a : 1,
});
if (!lm?.ok || !lm.buffer)
throw new Error("Attachment not found.");
srcBuf = lm.buffer;
}
else if (variants.length === 1) {
srcBuf = await fs_1.default.promises.readFile(variants[0].path);
}
else if (images.length === 1) {
srcBuf = await fs_1.default.promises.readFile(images[0].path);
}
else {
srcBuf = await fs_1.default.promises.readFile(path_1.default.join(primaryOutDir, String(pictures[0].filename || "")));
}
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
const cropResult = await applyCropFormat(srcBuf, input);
if (typeof cropResult === "string") {
return { content: [{ type: "text", text: cropResult }], isError: true };
}
const { left, right, top, bottom } = cropResult;
// Original dimensions for result reporting
const originalSize = await (0, core_bundle_mjs_1.getSize)(srcBuf);
const originalWidth = originalSize.width;
const originalHeight = originalSize.height;
// Apply crop
const croppedBuf = await (0, core_bundle_mjs_1.cropToPng)(srcBuf, { left, top, right, bottom });
// Measure cropped dimensions for XMP
const croppedSize = await (0, core_bundle_mjs_1.getSize)(croppedBuf);
// Build XMP params for crop (no diffusion — processing only)
const cropXmpParams = {
mode: "crop",
...(typeof croppedSize.width === "number" ? { width: croppedSize.width } : {}),
...(typeof croppedSize.height === "number" ? { height: croppedSize.height } : {}),
crop: { left, top, right, bottom },
...(typeof input.canvas === "string" ? { cropSource: input.canvas } : {}),
};
// Save as new image
const baseImageI = Math.max(1, st.counters?.nextImageI ?? 1);
const stamp = isoStampCompact();
const baseName = `image-${stamp}-i${baseImageI}`;
const saved = await saveOriginalPng(croppedBuf, primaryOutDir, `${baseName}.png`, cropXmpParams);
log(`[crop] saved: ${saved.savedPath} (${saved.size} bytes) [i${baseImageI}]`);
// Generate preview
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const imageRecordsForState = [
{
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: baseImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/crop`,
cropLeft: left,
cropTop: top,
cropRight: right,
cropBottom: bottom,
cropSource: typeof input.canvas === "string" ? input.canvas : undefined,
},
];
const previews = [];
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(croppedBuf, primaryOutDir, saved.fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`[crop] preview saved: ${previewFilePath} ${p.width}x${p.height}`);
}
catch (e) {
log(`[crop] preview failed: ${String(e)}`);
}
// Update state
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`[crop] state updated: appended i${baseImageI}, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`[crop] state update failed: ${String(e)}`);
}
// HTTP URLs
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginal = httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(saved.fileName, httpBase, currentLmChatId || undefined)
: "";
const httpPreview = (() => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[0]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
})();
// Audit log — effectiveRequestId is reused in the summary below, so a caller (the MCP
// adapter) can always name its HTML report after this exact audit requestId.
const effectiveRequestId = requestId ?? `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "crop", mode: "crop", requestId: effectiveRequestId });
if (currentLmChatId)
audit.setChatId(currentLmChatId);
const cropUserRequest = {};
if (input.canvas)
cropUserRequest.canvas = input.canvas;
cropUserRequest.cropLeft = left;
cropUserRequest.cropRight = right;
cropUserRequest.cropTop = top;
cropUserRequest.cropBottom = bottom;
audit.setUserRequest(cropUserRequest);
const auditOutput = {
i: baseImageI,
crop: { left, top, right, bottom },
images: [{
i: baseImageI,
path: saved.savedPath,
url: saved.fileUrl,
bytes: saved.size,
...(httpOriginal ? { http_url: httpOriginal } : {}),
...(previews[0] ? { preview_path: previews[0].filePath, preview_url: previews[0].fileUrl } : {}),
...(httpPreview ? { http_preview_url: httpPreview } : {}),
}],
};
audit.setOutput(auditOutput);
await audit.write();
}
catch (e) {
log(`[crop] audit logging error: ${String(e)}`);
}
// Assemble tool result — same structure as handleGenerateImage
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw === undefined
? true
: envPreviewRaw === "1" || envPreviewRaw.toLowerCase() === "true";
const summary = {
tool: "crop",
requestId: effectiveRequestId,
i: baseImageI,
originalWidth,
originalHeight,
crop: {
left: { pct: left, px: Math.round(left / 100 * originalWidth) },
right: { pct: right, px: Math.round(right / 100 * originalWidth) },
top: { pct: top, px: Math.round(top / 100 * originalHeight) },
bottom: { pct: bottom, px: Math.round(bottom / 100 * originalHeight) },
},
images_generated: 1,
files: {
original: saved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
};
const { files: _files, ...summaryNoFiles } = summary;
const fallbackPreviewUrl = previews[0]?.fileUrl || saved.fileUrl;
const variantLinksText = `Preview i${baseImageI}: ${httpPreview ? httpPreview : fallbackPreviewUrl}`;
const originalLinksText = `Original i${baseImageI}: ${httpOriginal ? httpOriginal : saved.fileUrl}`;
const reviewHint = "Carefully examine the preview and comment on how well the crop matches your intent.";
if (previewInChat && previews.length > 0) {
const fname = String(previews[0].fileName || "");
return {
content: [
{
type: "image",
fileName: saved.fileName,
mimeType: previews[0].mimeType,
markdown: ``,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
{
type: "text",
text: `Cropped Image i${baseImageI} successfully saved.`,
$hint: reviewHint,
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`[crop] error: ${error instanceof Error ? error.message : String(error)}`);
return {
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
isError: true,
};
}
}
// handleMask: draw a CYAN bounding box on the source image from crop percentages,
// save as a new image with crop metadata for use with inpaint / outpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleMask(pluginParams, requestId) {
await ensureBackendReady().catch((e) => {
log(`[mask] ensureBackendReady failed: ${String(e)}`);
});
try {
const parsed = core_bundle_mjs_1.CropToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid mask parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
let currentLmChatId = null;
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const primaryOutDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
if (!primaryOutDir) {
return {
content: [{ type: "text", text: "Failed to resolve LM Studio chat working directory." }],
isError: true,
};
}
await fs_1.default.promises.mkdir(primaryOutDir, { recursive: true }).catch(() => { });
// Load source state
const st = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const images = imageRecords
.filter((r) => r && typeof r.filename === "string")
.sort((a, b) => (a.i || 0) - (b.i || 0))
.map((r) => ({ i: r.i || 1, path: path_1.default.join(primaryOutDir, r.filename) }));
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variants = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(primaryOutDir, v.filename) }));
// Resolve canvas notation
let rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
let srcBuf;
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Runs before the
// detectLabel handling below, so per-box crop overrides (null = keep detection value) still apply.
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = [autoLabel];
}
// detectLabel: resolve crop coordinates from a prior detect_object run
// Multi-region: draw one cyan box per label.
let resolvedMaskBoxes;
if (Array.isArray(input.detectLabel) && input.detectLabel.length > 0) {
let labels = input.detectLabel;
let indices = Array.isArray(input.detectIndex) ? input.detectIndex : (input.detectIndex !== undefined ? [input.detectIndex] : []);
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
const boxes = [];
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] ?? indices[0] ?? 0;
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step in applyCropFormat
);
if (!detectResult.ok) {
return {
content: [{ type: "text", text: detectResult.error }],
isError: true,
};
}
// Per-box overrides: scalar = same for all; array[li] non-null = override; null/missing = keep detect.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
// Build a temporary input object for applyCropFormat (single-box).
const boxInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas && li === 0) {
rawCanvas = detectResult.srcCanvas;
input.canvas = detectResult.srcCanvas;
}
boxes.push(boxInput); // resolved after srcBuf is available
}
resolvedMaskBoxes = boxes; // will be resolved via applyCropFormat per box below
// For single-label path compatibility: set input sides to first box so auto-redirect etc. still work.
input.cropLeft = undefined;
input.cropRight = undefined;
input.cropTop = undefined;
input.cropBottom = undefined;
input.frameAdjust = undefined;
}
// Auto-redirect: if canvas points to a detect_object result and no detectLabel is given, use detectSource
if ((!input.detectLabel || (Array.isArray(input.detectLabel) && input.detectLabel.length === 0)) && rawCanvas) {
const _autoRedirectPref = parsePrefixedNotation(rawCanvas);
if (_autoRedirectPref?.pool === "image") {
const _autoRedirectRec = imageRecords.find((r) => r?.i === _autoRedirectPref.index);
if (typeof _autoRedirectRec?.sourceTool === "string" &&
_autoRedirectRec.sourceTool.includes("detect_object") &&
typeof _autoRedirectRec.detectSource === "string") {
rawCanvas = _autoRedirectRec.detectSource;
input.canvas = _autoRedirectRec.detectSource;
}
}
}
try {
if (rawCanvas) {
const pref = parsePrefixedNotation(rawCanvas);
if (pref) {
if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer) {
throw new Error(`Attachment a${pref.index} not found.`);
}
srcBuf = lm.buffer;
}
else if (pref.pool === "variant") {
const found = variants.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else if (pref.pool === "image") {
const found = images.find((img) => img.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
srcBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
srcBuf = await resolvePictureBuffer(found, primaryOutDir, pref.index);
}
}
else {
const pathToken = await resolveCanvasPathToken(rawCanvas, primaryOutDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
srcBuf = await fs_1.default.promises.readFile(pathToken);
}
}
else {
// Auto-select single source
const total = attachments.length + variants.length + images.length + pictures.length;
if (total === 0)
throw new Error("No source image available.");
if (total > 1)
throw new Error("Ambiguous source — specify canvas explicitly.");
if (attachments.length === 1) {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: typeof attachments[0]?.a === "number" ? attachments[0].a : 1,
});
if (!lm?.ok || !lm.buffer)
throw new Error("Attachment not found.");
srcBuf = lm.buffer;
}
else if (variants.length === 1) {
srcBuf = await fs_1.default.promises.readFile(variants[0].path);
}
else if (images.length === 1) {
srcBuf = await fs_1.default.promises.readFile(images[0].path);
}
else {
srcBuf = await fs_1.default.promises.readFile(path_1.default.join(primaryOutDir, String(pictures[0].filename || "")));
}
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
// Resolve multi-region box coordinates now that srcBuf is available.
let finalBoxes;
if (resolvedMaskBoxes) {
const resolved = [];
for (const boxInput of resolvedMaskBoxes) {
const r = await applyCropFormat(srcBuf, boxInput);
if (typeof r === "string")
return { content: [{ type: "text", text: r }], isError: true };
resolved.push(r);
}
finalBoxes = resolved;
}
else {
const cropResult = await applyCropFormat(srcBuf, input);
if (typeof cropResult === "string") {
return { content: [{ type: "text", text: cropResult }], isError: true };
}
finalBoxes = [cropResult];
}
// Original dimensions
const originalSize = await (0, core_bundle_mjs_1.getSize)(srcBuf);
const originalWidth = originalSize.width;
const originalHeight = originalSize.height;
// For state storage: use bounding union of all boxes (or single box).
const left = Math.min(...finalBoxes.map(b => b.left));
const right = Math.min(...finalBoxes.map(b => b.right));
const top = Math.min(...finalBoxes.map(b => b.top));
const bottom = Math.min(...finalBoxes.map(b => b.bottom));
// Draw CYAN bounding boxes on source image
const Jimp = (await import("jimp")).Jimp;
const img = await Jimp.read(srcBuf);
const cyanInt = (((0x44 & 0xff) << 24) | ((0x7d & 0xff) << 16) | ((0xf7 & 0xff) << 8) | (0xff & 0xff)) >>> 0;
const thickness = 2;
for (const box of finalBoxes) {
const x1 = Math.max(0, Math.min(originalWidth - 1, Math.round(box.left / 100 * originalWidth)));
const y1 = Math.max(0, Math.min(originalHeight - 1, Math.round(box.top / 100 * originalHeight)));
const x2 = Math.max(0, Math.min(originalWidth - 1, Math.round((100 - box.right) / 100 * originalWidth)));
const y2 = Math.max(0, Math.min(originalHeight - 1, Math.round((100 - box.bottom) / 100 * originalHeight)));
for (let t = 0; t < thickness; t++) {
for (let x = x1; x <= x2; x++) {
if (y1 + t < originalHeight)
img.setPixelColor(cyanInt, x, y1 + t);
if (y2 - t >= 0)
img.setPixelColor(cyanInt, x, y2 - t);
}
for (let y = y1; y <= y2; y++) {
if (x1 + t < originalWidth)
img.setPixelColor(cyanInt, x1 + t, y);
if (x2 - t >= 0)
img.setPixelColor(cyanInt, x2 - t, y);
}
}
}
const annotatedBuf = await img.getBuffer("image/png");
// Build XMP params (processing only, same mode as crop)
const maskXmpParams = {
mode: "crop",
...(typeof originalWidth === "number" ? { width: originalWidth } : {}),
...(typeof originalHeight === "number" ? { height: originalHeight } : {}),
crop: { left, top, right, bottom },
...(typeof input.canvas === "string" ? { cropSource: input.canvas } : {}),
};
// Save as new image
const baseImageI = Math.max(1, st.counters?.nextImageI ?? 1);
const stamp = isoStampCompact();
const baseName = `image-${stamp}-i${baseImageI}`;
const saved = await saveOriginalPng(annotatedBuf, primaryOutDir, `${baseName}.png`, maskXmpParams);
log(`[mask] saved: ${saved.savedPath} (${saved.size} bytes) [i${baseImageI}]`);
// Generate preview
const variantPreviewSpec = core_bundle_mjs_1.VARIANT_FULL_CONFIG.preview;
const imageRecordsForState = [
{
filename: `${baseName}.png`,
preview: `preview-${baseName}.jpg`,
i: baseImageI,
sourceTool: `${(0, core_bundle_mjs_1.getSelfPluginIdentifier)()}/mask`,
// Single-box scalar fields for backward compat (first box or the only box).
cropLeft: finalBoxes[0].left,
cropTop: finalBoxes[0].top,
cropRight: finalBoxes[0].right,
cropBottom: finalBoxes[0].bottom,
cropSource: typeof input.canvas === "string" ? input.canvas : undefined,
// Multi-box: always store full array so inpaint/outpaint can loop over all regions.
...(finalBoxes.length > 1 ? { bboxes: finalBoxes } : {}),
},
];
const previews = [];
try {
const p = await (0, core_bundle_mjs_1.generatePreviewFromBuffer)(annotatedBuf, primaryOutDir, saved.fileName, variantPreviewSpec);
const previewFilePath = p.previewAbs;
const previewFileUrl = encodeFileUrl(previewFilePath);
previews.push({
ok: true,
filePath: previewFilePath,
fileName: p.previewFilename,
fileUrl: previewFileUrl,
size_bytes: p.data.length,
width: p.width,
height: p.height,
mimeType: "image/jpeg",
format: variantPreviewSpec.format,
dataBase64: p.data.toString("base64"),
});
log(`[mask] preview saved: ${previewFilePath} ${p.width}x${p.height}`);
}
catch (e) {
log(`[mask] preview failed: ${String(e)}`);
}
// Update state
try {
const { appendImages } = await import("../core-bundle.mjs");
const stateForUpdate = await (0, core_bundle_mjs_1.readState)(primaryOutDir);
const appendResult = appendImages(stateForUpdate, imageRecordsForState);
if (appendResult.changed) {
await (0, core_bundle_mjs_1.writeStateAtomic)(primaryOutDir, stateForUpdate);
log(`[mask] state updated: appended i${baseImageI}, nextImageI=${stateForUpdate.counters.nextImageI}`);
}
}
catch (e) {
log(`[mask] state update failed: ${String(e)}`);
}
// HTTP URLs
const httpBase = await (0, core_bundle_mjs_1.getHealthyServerBaseUrl)();
const httpOriginal = httpBase
? (0, core_bundle_mjs_1.toHttpOriginalUrl)(saved.fileName, httpBase, currentLmChatId || undefined)
: "";
const httpPreview = (() => {
if (!httpBase || !currentLmChatId)
return "";
const previewFileName = previews[0]?.fileName;
if (!previewFileName)
return "";
return (0, core_bundle_mjs_1.toHttpPreviewUrl)(previewFileName, httpBase, currentLmChatId);
})();
// Audit log — effectiveRequestId is reused in the summary below, so a caller (the MCP
// adapter) can always name its HTML report after this exact audit requestId.
const effectiveRequestId = requestId ?? `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
try {
const audit = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "mask", mode: "crop", requestId: effectiveRequestId });
if (currentLmChatId)
audit.setChatId(currentLmChatId);
const maskUserRequest = {};
if (input.canvas)
maskUserRequest.canvas = input.canvas;
maskUserRequest.cropLeft = left;
maskUserRequest.cropRight = right;
maskUserRequest.cropTop = top;
maskUserRequest.cropBottom = bottom;
audit.setUserRequest(maskUserRequest);
const auditOutput = {
i: baseImageI,
crop: { left, top, right, bottom },
images: [{
i: baseImageI,
path: saved.savedPath,
url: saved.fileUrl,
bytes: saved.size,
...(httpOriginal ? { http_url: httpOriginal } : {}),
...(previews[0] ? { preview_path: previews[0].filePath, preview_url: previews[0].fileUrl } : {}),
...(httpPreview ? { http_preview_url: httpPreview } : {}),
}],
};
audit.setOutput(auditOutput);
await audit.write();
}
catch (e) {
log(`[mask] audit logging error: ${String(e)}`);
}
// Assemble tool result
const envPreviewRaw = process.env.PREVIEW_IN_CHAT;
const previewInChat = envPreviewRaw === undefined
? true
: envPreviewRaw === "1" || envPreviewRaw.toLowerCase() === "true";
const summary = {
tool: "mask",
requestId: effectiveRequestId,
i: baseImageI,
originalWidth,
originalHeight,
crop: {
left: { pct: left, px: Math.round(left / 100 * originalWidth) },
right: { pct: right, px: Math.round(right / 100 * originalWidth) },
top: { pct: top, px: Math.round(top / 100 * originalHeight) },
bottom: { pct: bottom, px: Math.round(bottom / 100 * originalHeight) },
},
images_generated: 1,
files: {
original: saved.fileUrl,
previews: previews.map((p) => p.fileUrl),
},
};
const { files: _files, ...summaryNoFiles } = summary;
const fallbackPreviewUrl = previews[0]?.fileUrl || saved.fileUrl;
const variantLinksText = `Preview i${baseImageI}: ${httpPreview ? httpPreview : fallbackPreviewUrl}`;
const originalLinksText = `Original i${baseImageI}: ${httpOriginal ? httpOriginal : saved.fileUrl}`;
const reviewHint = "Carefully examine the preview and comment on how well the mask region matches your intent.";
if (previewInChat && previews.length > 0) {
const fname = String(previews[0].fileName || "");
return {
content: [
{
type: "image",
fileName: saved.fileName,
mimeType: previews[0].mimeType,
markdown: ``,
$hint: "This is an image file. Present the image to the user by using the markdown above.",
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
else {
return {
content: [
{
type: "text",
text: `Mask Image i${baseImageI} successfully saved.`,
$hint: reviewHint,
},
{ type: "text", text: variantLinksText },
{ type: "text", text: originalLinksText },
{ type: "text", text: JSON.stringify(summaryNoFiles) },
],
};
}
}
catch (error) {
log(`[mask] error: ${error instanceof Error ? error.message : String(error)}`);
return {
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
isError: true,
};
}
}
// ─────────────────────────────────────────────────────────────────────────────
// handleZoomIn: crop + Draw Things edit + save result
// ─────────────────────────────────────────────────────────────────────────────
async function handleZoomIn(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.ZoomInToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid zoom-in parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Load canvas buffer and resolve crop metadata from state
let rawBuf;
let resolvedCropMeta;
// When canvas is an iN image, rawBuf IS already the target region — skip imgCropToPng.
let skipImgCrop = false;
// Source image dimensions for the render target (overrides origW/H when set).
let overrideZoomDims;
// Set when detectLabel resolved the crop — post-crop dims needed for render target.
let detectLabelWasUsed = false;
let currentLmChatId = null;
// Hoisted out of the try block below (not just block-scoped) so the final handleGenerateImage()
// call can pass this already-resolved context down instead of forcing a second, TTL-gated
// (60s) re-resolution that can expire during this function's own Restore-Pass/zoom render.
let currentLmWorkingDir = null;
let _zoomAuditSourceOverride = undefined;
try {
// Resolve chat context to find source
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
const chatDir = currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas && chatDir) {
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const attachments = Array.isArray(st?.attachments) ? st.attachments : [];
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0). Must run before canvas
// pool resolution below, so the later detectLabel-driven resolveDetectionCrop path picks it up
// instead of silently falling back to the full original frame.
if ((!input.detectLabel || !String(input.detectLabel).trim()) && rawCanvas) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
input.detectLabel = autoLabel;
}
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
_zoomAuditSourceOverride = {
sourceKind: "attachment",
sourceOriginAbs: typeof lm.originalPath === "string" ? lm.originalPath : undefined,
sourceFileName: typeof lm.originalPath === "string" ? path_1.default.basename(lm.originalPath) : undefined,
sourceOriginalName: typeof lm.originalName === "string" ? lm.originalName : undefined,
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this attachment
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
const detectSrc = typeof found.detectSource === "string" ? found.detectSource : undefined;
// Mask results store crop coords on the annotated full-source image, not a pre-cropped file.
// For those, rawBuf must be reloaded from cropSource and skipImgCrop must stay false.
let isMaskCanvasResult = false;
if (hasCropFields) {
resolvedCropMeta = {
cropLeft: found.cropLeft,
cropTop: found.cropTop,
cropRight: found.cropRight,
cropBottom: found.cropBottom,
cropSource: found.cropSource,
};
if (typeof found.sourceTool === "string" && found.sourceTool.includes("/mask") &&
typeof found.cropSource === "string") {
isMaskCanvasResult = true;
const _srcRef = parsePrefixedNotation(found.cropSource);
if (_srcRef) {
if (_srcRef.pool === "attachment") {
const _lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: _srcRef.index });
if (_lm?.ok && _lm.buffer)
rawBuf = _lm.buffer;
}
else if (_srcRef.pool === "image") {
const _srcRec = imageRecords.find((r) => r?.i === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename)));
}
else if (_srcRef.pool === "variant") {
const _srcPath = variantPaths.find((v) => v.v === _srcRef.index)?.path;
if (_srcPath)
rawBuf = await fs_1.default.promises.readFile(_srcPath);
}
else {
const _srcRec = pictures.find((p) => p?.p === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename || "")));
}
}
}
}
else if (detectSrc) {
// detect_object result — reload rawBuf from the clean source (not the annotated image).
resolvedCropMeta = { cropLeft: 0, cropTop: 0, cropRight: 0, cropBottom: 0, cropSource: detectSrc };
const _srcRef = parsePrefixedNotation(detectSrc);
if (_srcRef) {
if (_srcRef.pool === "attachment") {
const _lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: _srcRef.index });
if (_lm?.ok && _lm.buffer)
rawBuf = _lm.buffer;
}
else if (_srcRef.pool === "image") {
const _srcRec = imageRecords.find((r) => r?.i === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename)));
}
else if (_srcRef.pool === "variant") {
const _srcPath = variantPaths.find((v) => v.v === _srcRef.index)?.path;
if (_srcPath)
rawBuf = await fs_1.default.promises.readFile(_srcPath);
}
else {
const _srcRec = pictures.find((p) => p?.p === _srcRef.index);
if (_srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(_srcRec.filename || "")));
}
}
}
if (resolvedCropMeta && !isMaskCanvasResult) {
// rawBuf is already the target region (the iN file). Do not re-apply imgCropToPng.
skipImgCrop = true;
// Look up source image dimensions for the render target.
const sourceRef = (hasCropFields
? (typeof found.cropSource === "string" ? found.cropSource : undefined)
: detectSrc);
if (typeof sourceRef === "string") {
const srcPref = parsePrefixedNotation(sourceRef);
if (srcPref?.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcPref.index)?.path;
if (srcPath) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(srcPath));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
else if (srcPref?.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcPref.index);
if (srcRec?.filename) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename))));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
else if (srcPref?.pool === "attachment") {
const attRec = attachments.find((a) => a?.a === srcPref.index);
if (attRec?.width && attRec?.height) {
overrideZoomDims = { w: attRec.width, h: attRec.height };
}
else {
try {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: srcPref.index,
});
if (lm?.ok && lm.buffer) {
const srcSize = await (0, core_bundle_mjs_1.getSize)(lm.buffer);
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
const _oa = typeof lm.originalPath === "string" ? lm.originalPath : undefined;
_zoomAuditSourceOverride = {
sourceKind: "attachment",
sourceOriginAbs: _oa,
sourceFileName: _oa ? path_1.default.basename(_oa) : undefined,
sourceOriginalName: typeof lm.originalName === "string" ? lm.originalName : undefined,
originalDims: { width: srcSize.width, height: srcSize.height, bytes: lm.buffer.byteLength },
};
}
}
catch { }
}
}
else if (srcPref?.pool === "picture") {
const srcRec = pictures.find((p) => p?.p === srcPref.index);
if (srcRec?.filename) {
try {
const srcSize = await (0, core_bundle_mjs_1.getSize)(await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename))));
overrideZoomDims = { w: srcSize.width, h: srcSize.height };
}
catch { }
}
}
}
}
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
_zoomAuditSourceOverride = {
sourceKind: "variant",
sourceOriginAbs: found.path,
sourceFileName: path_1.default.basename(found.path),
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this variant
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
else {
// picture pool (pN)
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
_zoomAuditSourceOverride = {
sourceKind: "picture",
sourceOriginAbs: typeof found.filename === "string" ? path_1.default.join(chatDir, found.filename) : undefined,
sourceFileName: typeof found.filename === "string" ? found.filename : undefined,
// originalDims filled after imgGetSize below
};
// Find the most recent crop record whose cropSource matches this picture
const cropCandidate = imageRecords
.filter((r) => r?.cropSource === rawCanvas &&
(typeof r.cropLeft === "number" || typeof r.cropTop === "number" ||
typeof r.cropRight === "number" || typeof r.cropBottom === "number"))
.sort((a, b) => (b.i ?? 0) - (a.i ?? 0))[0];
if (cropCandidate) {
resolvedCropMeta = {
cropLeft: cropCandidate.cropLeft,
cropTop: cropCandidate.cropTop,
cropRight: cropCandidate.cropRight,
cropBottom: cropCandidate.cropBottom,
cropSource: cropCandidate.cropSource,
};
}
}
// detectLabel: resolve crop coordinates from a prior detect_object run
if (typeof input.detectLabel === "string" && input.detectLabel.trim()) {
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, input.detectLabel.trim(), typeof input.detectIndex === "number" ? Math.max(0, input.detectIndex) : 0, input.frameAdjust);
if (!detectResult.ok) {
throw new Error(detectResult.error);
}
resolvedCropMeta = {
cropLeft: detectResult.cropLeft,
cropRight: detectResult.cropRight,
cropTop: detectResult.cropTop,
cropBottom: detectResult.cropBottom,
};
skipImgCrop = false;
overrideZoomDims = undefined;
detectLabelWasUsed = true;
// If canvas was the iN detect result, reload rawBuf from the actual source
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
const srcRef = parsePrefixedNotation(detectResult.srcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: srcRef.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Source a${srcRef.index} not found.`);
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (!srcRec)
throw new Error(`Source i${srcRef.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (!srcPath)
throw new Error(`Source v${srcRef.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
// picture
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (!srcRec)
throw new Error(`Source p${srcRef.index} not found.`);
rawBuf = await resolvePictureBuffer(srcRec, chatDir, srcRef.index);
}
}
}
}
}
else {
// Use last attachment as fallback
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: input.canvas ? undefined : undefined,
explicitAttachmentSource: true,
});
if (!lm?.ok || !lm.buffer)
throw new Error("No source image available.");
rawBuf = lm.buffer;
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
// If no crop metadata was found (plain image, no prior crop), default to zero-crop
// (= full image). imgCropToPng with all-zero values is a no-op.
if (!resolvedCropMeta) {
resolvedCropMeta = { cropLeft: 0, cropTop: 0, cropRight: 0, cropBottom: 0 };
}
// Measure rawBuf dimensions (for variant/attachment paths: pre-crop source; for image path: the region itself).
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const origW = canvasSize.width;
const origH = canvasSize.height;
// Complete audit override with pre-crop dims for paths (attachment, variant, picture) that
// could not compute dims inside the try block.
if (_zoomAuditSourceOverride && !_zoomAuditSourceOverride.originalDims) {
_zoomAuditSourceOverride = {
..._zoomAuditSourceOverride,
originalDims: { width: origW, height: origH, bytes: rawBuf.byteLength },
};
}
// When imageFormat is specified and we are about to crop the source image,
// expand the crop region symmetrically so the region already has the target AR.
// This ensures normalizeInputBuffer can cover-resize without clipping the detection region.
// Always expand (never shrink) — the full selected region must remain visible.
if (input.imageFormat && resolvedCropMeta && !skipImgCrop) {
const ratioMap = {
square: 1,
landscape: 4 / 3,
portrait: 3 / 4,
"16:9": 16 / 9,
};
const targetAR = ratioMap[input.imageFormat];
if (targetAR !== undefined) {
const cl = resolvedCropMeta.cropLeft ?? 0;
const cr = resolvedCropMeta.cropRight ?? 0;
const ct = resolvedCropMeta.cropTop ?? 0;
const cb = resolvedCropMeta.cropBottom ?? 0;
const regionW = origW * (1 - cl / 100 - cr / 100);
const regionH = origH * (1 - ct / 100 - cb / 100);
const regionAR = regionW / regionH;
if (Math.abs(regionAR - targetAR) > 0.005) {
if (regionAR < targetAR) {
// Region too tall → need to expand width.
// Try symmetric first; if one side hits the image boundary (crop=0),
// transfer the remainder to the opposite side.
const totalExtraPct = ((regionH * targetAR - regionW) / origW) * 100;
const halfExtra = totalExtraPct / 2;
const leftReduction = Math.min(cl, halfExtra);
const rightReduction = Math.min(cr, halfExtra + (halfExtra - leftReduction));
const leftReduction2 = Math.min(cl, halfExtra + (halfExtra - rightReduction));
resolvedCropMeta = {
...resolvedCropMeta,
cropLeft: Math.max(0, cl - leftReduction2),
cropRight: Math.max(0, cr - rightReduction),
};
}
else {
// Region too wide → need to expand height.
const totalExtraPct = ((regionW / targetAR - regionH) / origH) * 100;
const halfExtra = totalExtraPct / 2;
const topReduction = Math.min(ct, halfExtra);
const bottomReduction = Math.min(cb, halfExtra + (halfExtra - topReduction));
const topReduction2 = Math.min(ct, halfExtra + (halfExtra - bottomReduction));
resolvedCropMeta = {
...resolvedCropMeta,
cropTop: Math.max(0, ct - topReduction2),
cropBottom: Math.max(0, cb - bottomReduction),
};
}
}
}
}
// Apply frameAdjust to resolvedCropMeta for non-detect, non-skip paths.
// detectLabelWasUsed: frameAdjust was already applied inside resolveDetectionCrop.
// skipImgCrop: rawBuf is already the cropped region; can't re-crop from source here.
if (input.frameAdjust !== undefined && !detectLabelWasUsed && !skipImgCrop) {
const cl = resolvedCropMeta.cropLeft ?? 0;
const cr = resolvedCropMeta.cropRight ?? 0;
const ct = resolvedCropMeta.cropTop ?? 0;
const cb = resolvedCropMeta.cropBottom ?? 0;
const regionW = origW * (1 - cl / 100 - cr / 100);
const regionH = origH * (1 - ct / 100 - cb / 100);
const cropDiag = Math.hypot(regionW, regionH);
let d_px;
if (typeof input.frameAdjust === "string") {
const m = String(input.frameAdjust).trim().match(/^([+-]?\d+(?:\.\d+)?)\s*(%|px)?$/i);
if (m) {
const val = parseFloat(m[1]);
d_px = m[2]?.toLowerCase() === "px" ? val : (val / 100) * cropDiag;
}
else {
d_px = 0;
}
}
else {
d_px = (input.frameAdjust / 100) * cropDiag;
}
const dLR = (d_px / origW) * 100;
const dTB = (d_px / origH) * 100;
resolvedCropMeta = {
...resolvedCropMeta,
cropLeft: Math.max(0, cl - dLR),
cropRight: Math.max(0, cr - dLR),
cropTop: Math.max(0, ct - dTB),
cropBottom: Math.max(0, cb - dTB),
};
}
// Apply the crop from persisted metadata only when rawBuf is the full source (variant/attachment).
// For iN canvas (skipImgCrop=true) rawBuf is already the target region.
if (!skipImgCrop) {
rawBuf = await (0, core_bundle_mjs_1.cropToPng)(rawBuf, {
left: resolvedCropMeta.cropLeft ?? 0,
top: resolvedCropMeta.cropTop ?? 0,
right: resolvedCropMeta.cropRight ?? 0,
bottom: resolvedCropMeta.cropBottom ?? 0,
});
}
// ── RESTORE-PASS — edit-class quality pre-stage for small sources ─────────
// Restore is EDIT, not image2image. Must use generateImageEdit.
// If the source is below the restore threshold, run a fast edit pass to
// recover quality before the zoom-in pipeline consumes the buffer.
if (imageService?.name === "drawthings") {
if (typeof imageService.generateImageEdit !== "function") {
// handleZoomIn never warms up the backend itself before this point, so
// imageService may just be unprobed/stale — retry gRPC-only. HTTP is
// never a valid fallback here (it doesn't implement generateImageEdit).
await ensureBackendReady({ requireGrpc: true }).catch((e) => {
log(`[restore] ensureBackendReady(requireGrpc) failed: ${String(e?.message || e)}`);
});
}
if (typeof imageService.generateImageEdit !== "function") {
const selectedTransport = globalThis
?.__DT_SELECTED_TRANSPORT__;
log(`[restore] ERROR: Draw Things gRPC is required for Restore-Pass but is unavailable ` +
`(imageService.name=${imageService?.name}, selectedTransport=${selectedTransport ?? "none"})`);
throw new Error("Restore-Pass requires the Draw Things gRPC backend, which is not reachable. HTTP is not a valid fallback for edit mode.");
}
const rawSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
if (rawSize.width + rawSize.height < core_bundle_mjs_1.drawthingsRestoreLimits.targetSumRestore) {
const restoreOnProgress = onProgress
? (step, total, msg) => onProgress(step, total, step === -1 ? (msg ? `Restore ${msg}` : "Restore") : "Restore")
: undefined;
// Normalize source up to targetSum (1344) using the standard pipeline.
// targetSumRestore is only the firing threshold; targetSum is the actual
// normalize goal. Supply requestedRawW/H that exceed targetSum to force
// normalizeInputBuffer to produce the maximum 64-aligned result at targetSum.
const _restoreRawSum = rawSize.width + rawSize.height;
const _restoreFactor = Math.ceil(core_bundle_mjs_1.drawthingsRestoreLimits.targetSum / _restoreRawSum) + 1;
const restoreNormalized = await normalizeInputBuffer(rawBuf, {
requestedRawW: rawSize.width * _restoreFactor,
requestedRawH: rawSize.height * _restoreFactor,
targetSumOverride: core_bundle_mjs_1.drawthingsRestoreLimits.targetSum,
logPrefix: "[restore]",
});
const restoreAdjW = restoreNormalized.preprocess.adjusted.width;
const restoreAdjH = restoreNormalized.preprocess.adjusted.height;
const restoreNormBuf = restoreNormalized.buf;
// model must NOT be sent — Draw Things must use whatever model is currently loaded.
// input.model is irrelevant here and must not be set.
// Only pass runtime overrides — the service fills baseDefaults itself.
// Pre-spreading defaultParamsRestore would flood `filtered` and override the overlay.
const restoreParams = {
width: restoreAdjW,
height: restoreAdjH,
prompt: input.prompt ?? "",
_dt_i2i_profile: "restore",
_dt_needs_upscaler: false,
};
const restoreStartMs = Date.now();
const restoreResult = await imageService.generateImageEdit(restoreParams, [restoreNormBuf], restoreOnProgress, undefined);
const restoreInferenceMs = Date.now() - restoreStartMs;
if (restoreResult.isError || restoreResult.error) {
throw new Error(`Restore-Pass failed: ${restoreResult.errorMessage ?? restoreResult.error ?? "unknown"}`);
}
let restoreBuffers = [];
if (Array.isArray(restoreResult.images) && restoreResult.images.length > 0) {
for (const img of restoreResult.images) {
if (typeof img === "string") {
const b64 = img.startsWith("data:") ? img.split(",")[1] : img;
restoreBuffers.push(Buffer.from(b64, "base64"));
}
}
}
else if (Buffer.isBuffer(restoreResult.imageBuffer)) {
restoreBuffers.push(restoreResult.imageBuffer);
}
else if (restoreResult.imageData) {
const d = restoreResult.imageData;
if (Buffer.isBuffer(d))
restoreBuffers.push(d);
else if (typeof d === "string") {
const b64 = d.startsWith("data:") ? d.split(",")[1] : d;
restoreBuffers.push(Buffer.from(b64, "base64"));
}
}
if (restoreBuffers.length === 0)
throw new Error("Restore-Pass returned no image data");
const restoreMeta = restoreResult?.metadata ?? {};
let restoreBackendW;
let restoreBackendH;
try {
const rm = await (0, core_bundle_mjs_1.getSize)(restoreBuffers[0]);
restoreBackendW = rm.width;
restoreBackendH = rm.height;
}
catch { }
rawBuf = restoreBuffers[0];
try {
const auditR = (0, core_bundle_mjs_1.buildAuditLogger)({ backend: "drawthings", mode: "edit", requestId: undefined });
if (currentLmChatId)
auditR.setChatId(currentLmChatId);
auditR.setUserRequest({ canvas: input.canvas, prompt: input.prompt ?? "" });
auditR.setRenderTarget({
requested_raw: { width: restoreAdjW, height: restoreAdjH },
requested_effective: { width: restoreAdjW, height: restoreAdjH },
needs_upscaler: false,
});
auditR.setInputs({
canvas: {
notation: input.canvas ?? undefined,
original: {
width: restoreNormalized.preprocess.original.width,
height: restoreNormalized.preprocess.original.height,
bytes: restoreNormalized.preprocess.original.bytes,
},
adjusted: {
width: restoreNormalized.preprocess.adjusted.width,
height: restoreNormalized.preprocess.adjusted.height,
bytes: restoreNormalized.preprocess.adjusted.bytes,
},
},
});
const restoreOutput = { restore_pass: true };
if (restoreBackendW !== undefined && restoreBackendH !== undefined) {
restoreOutput.backend_returned = { width: restoreBackendW, height: restoreBackendH };
}
restoreOutput.inference_time_ms = restoreInferenceMs;
if (typeof restoreMeta.prompt_used === "string") {
restoreOutput.prompt_used = restoreMeta.prompt_used;
}
if (typeof restoreMeta.model === "string" && restoreMeta.model.trim()) {
restoreOutput.model_used = path_1.default.basename(restoreMeta.model);
}
if (typeof restoreMeta.overlay_source === "string" && restoreMeta.overlay_source.trim()) {
restoreOutput.overlay_source = restoreMeta.overlay_source;
}
if (typeof restoreMeta.overlay_preset === "string" && restoreMeta.overlay_preset.trim()) {
restoreOutput.overlay_preset = restoreMeta.overlay_preset;
}
if (typeof restoreMeta.defaults_used === "string" && restoreMeta.defaults_used.trim()) {
restoreOutput.defaults_used = restoreMeta.defaults_used;
}
if (typeof restoreMeta.i2i_profile === "string" && restoreMeta.i2i_profile.trim()) {
restoreOutput.i2i_profile = restoreMeta.i2i_profile;
}
if (typeof restoreMeta.strength_used === "number" && Number.isFinite(restoreMeta.strength_used)) {
restoreOutput.strength_used = restoreMeta.strength_used;
}
if (typeof restoreMeta.steps_used === "number" && Number.isFinite(restoreMeta.steps_used)) {
restoreOutput.steps_used = restoreMeta.steps_used;
}
if (typeof restoreMeta.sampler_used === "string" && restoreMeta.sampler_used.trim()) {
restoreOutput.sampler_used = restoreMeta.sampler_used;
}
if (typeof restoreMeta.guidance_scale_used === "number" && Number.isFinite(restoreMeta.guidance_scale_used)) {
restoreOutput.guidance_scale_used = restoreMeta.guidance_scale_used;
}
if (Array.isArray(restoreMeta.loras_used) && restoreMeta.loras_used.length > 0) {
restoreOutput.loras_used = restoreMeta.loras_used;
}
auditR.setOutput(restoreOutput);
await auditR.write();
}
catch (auditErr) {
log(`[restore] audit write failed: ${String(auditErr)}`);
}
}
}
// ── END RESTORE-PASS ───────────────────────────────────────────────────────
// For the detectLabel path the crop was just applied above: origW/H are the
// pre-crop source dims. Use those as overrideZoomDims (= source resolution
// reference) and derive zoomSrcW/H from the post-crop buffer so that the
// render target scales the detection region AR up to source resolution —
// matching the iN detect path behaviour.
let zoomSrcW = origW;
let zoomSrcH = origH;
if (detectLabelWasUsed && !skipImgCrop) {
const croppedSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
overrideZoomDims = { w: origW, h: origH };
zoomSrcW = croppedSize.width;
zoomSrcH = croppedSize.height;
}
// Derive render_target dimensions.
// When overrideZoomDims is available (iN canvas or detectLabel case), zoomSrcW/H are
// the cropped region dims. Scale the crop to the source resolution while preserving
// crop AR, using the longer source axis as the constraining dim.
// When overrideZoomDims is absent (variant/attachment canvas), zoomSrcW/H are the
// pre-crop source dims and are used directly.
let zoomTargetW;
let zoomTargetH;
if (overrideZoomDims) {
if (zoomSrcW > zoomSrcH) {
// landscape crop: constrain to source width
zoomTargetW = overrideZoomDims.w;
zoomTargetH = Math.round(overrideZoomDims.w * zoomSrcH / zoomSrcW);
}
else {
// portrait or square crop: constrain to source height
zoomTargetH = overrideZoomDims.h;
zoomTargetW = Math.round(overrideZoomDims.h * zoomSrcW / zoomSrcH);
}
}
else {
zoomTargetW = zoomSrcW;
zoomTargetH = zoomSrcH;
}
// Safety floor: if either dimension is below 256, scale both up proportionally
// so the short side reaches 512 — preserving aspect ratio.
const minZoomDim = Math.min(zoomTargetW, zoomTargetH);
if (minZoomDim < 256) {
const scale = Math.ceil(256 / minZoomDim);
zoomTargetW = Math.round(zoomTargetW * scale);
zoomTargetH = Math.round(zoomTargetH * scale);
}
// Build merged params for handleGenerateImage (image2image mode).
// Explicit user-supplied width/height/imageFormat always override the cropSource-derived dims.
// cropSource-derived zoomTargetW/H are used only when the user provided none of those.
const hasExplicitDims = input.width !== undefined || input.height !== undefined || input.imageFormat !== undefined;
const mergedParams = {
mode: "image2image",
prompt: input.prompt ?? "",
};
if (hasExplicitDims) {
if (input.width !== undefined)
mergedParams.width = input.width;
if (input.height !== undefined)
mergedParams.height = input.height;
if (input.imageFormat !== undefined)
mergedParams.imageFormat = input.imageFormat;
}
else {
mergedParams.width = zoomTargetW;
mergedParams.height = zoomTargetH;
}
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
const zoomHandleProgress = onProgress
? (step, total, msg) => onProgress(step, total, step === -1 ? (msg ? `Zoom ${msg}` : "Zoom") : "Zoom")
: undefined;
const result = await handleGenerateImage(mergedParams, zoomHandleProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:zoom-in",
cropMeta: resolvedCropMeta ? {
left: resolvedCropMeta.cropLeft ?? 0,
top: resolvedCropMeta.cropTop ?? 0,
right: resolvedCropMeta.cropRight ?? 0,
bottom: resolvedCropMeta.cropBottom ?? 0,
cropSource: resolvedCropMeta.cropSource,
} : undefined,
auditSourceOverride: _zoomAuditSourceOverride,
chatId: currentLmChatId ?? undefined,
workingDir: currentLmWorkingDir ?? undefined,
});
// Inject zoom-specific metadata into tool result summary so the agent
// (and user) can see what canvas state was used without checking audit logs.
if (result.content && Array.isArray(result.content) && result.content.length > 0) {
const lastItem = result.content[result.content.length - 1];
if (lastItem && lastItem.type === "text" && typeof lastItem.text === "string") {
try {
const summary = JSON.parse(lastItem.text);
summary.zoom_input = {
canvas_source: { width: origW, height: origH },
render_target: { width: zoomTargetW, height: zoomTargetH },
...(resolvedCropMeta ? { crop: resolvedCropMeta } : {}),
};
lastItem.text = JSON.stringify(summary);
}
catch { /* summary not JSON — leave as-is */ }
}
}
return result;
}
// ─────────────────────────────────────────────────────────────────────────────
// handleInpaint: load canvas, resize to aligned dims, build white-on-black mask
// (white = region to repaint, from cropLeft/cropTop/cropRight/cropBottom),
// call handleGenerateImage in edit mode with sourceTag canvas:inpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleInpaint(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.InpaintToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [{ type: "text", text: `Invalid inpaint parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}` }],
};
}
const input = parsed.data;
let rawBuf;
let resolvedBBoxes = [{ left: 0, top: 0, right: 0, bottom: 0 }];
let chatDir;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (!rawCanvas || !chatDir)
throw new Error("canvas is required for inpaint");
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: pref.index });
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
// Resolve crop coordinates from mask/crop record or detectLabel(s).
let labels = input.detectLabel ?? [];
let indices = input.detectIndex ?? [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0).
if (labels.length === 0) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
labels = [autoLabel];
}
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
if (labels.length > 0) {
const boxes = [];
let resolvedSrcCanvas = undefined;
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] !== undefined ? indices[li] : (indices[0] !== undefined ? indices[0] : 0);
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step after per-box overrides
);
if (!detectResult.ok)
throw new Error(detectResult.error);
// Per-box overrides: scalar = all boxes; array[li] non-null = override; null/absent = keep detect value.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
const tmpInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
const resolved = await applyCropFormat(rawBuf, tmpInput);
if (typeof resolved === "string")
throw new Error(resolved);
boxes.push({ left: resolved.left, top: resolved.top, right: resolved.right, bottom: resolved.bottom });
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
if (resolvedSrcCanvas !== undefined && resolvedSrcCanvas !== detectResult.srcCanvas) {
throw new Error(`Mixed sources — detectLabel entries resolve to different source images ` +
`('${resolvedSrcCanvas}' vs '${detectResult.srcCanvas}'). ` +
`All labels must refer to detections on the same source image.`);
}
resolvedSrcCanvas = detectResult.srcCanvas;
}
}
if (resolvedSrcCanvas) {
const srcRef = parsePrefixedNotation(resolvedSrcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
resolvedBBoxes = boxes;
}
else if (pref && pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (found) {
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
if (hasCropFields) {
// Multi-region: prefer bboxes array; fall back to scalar fields for backward compat.
if (Array.isArray(found.bboxes) && found.bboxes.length > 0) {
resolvedBBoxes = found.bboxes;
}
else {
resolvedBBoxes = [{ left: found.cropLeft ?? 0, top: found.cropTop ?? 0, right: found.cropRight ?? 0, bottom: found.cropBottom ?? 0 }];
}
if (typeof found.cropSource === "string") {
const srcRef = parsePrefixedNotation(found.cropSource);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
}
}
}
}
catch (e) {
return { content: [{ type: "text", text: String(e?.message || e) }], isError: true };
}
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const adjW = canvasSize.width;
const adjH = canvasSize.height;
// Build inpaint mask: white (0xffffffff) inside each BBox, black (0x000000ff) outside.
const Jimp = (await import("jimp")).Jimp;
const mask = new Jimp({ width: adjW, height: adjH, color: 0x000000ff });
for (const bbox of resolvedBBoxes) {
const bx1 = Math.round(bbox.left / 100 * adjW);
const bx2 = Math.round((100 - bbox.right) / 100 * adjW);
const by1 = Math.round(bbox.top / 100 * adjH);
const by2 = Math.round((100 - bbox.bottom) / 100 * adjH);
mask.scan(bx1, by1, Math.max(0, bx2 - bx1), Math.max(0, by2 - by1), (_x, _y, idx) => {
mask.bitmap.data[idx] = 255;
mask.bitmap.data[idx + 1] = 255;
mask.bitmap.data[idx + 2] = 255;
mask.bitmap.data[idx + 3] = 255;
});
}
// DEBUG mask save:
// await fs.promises.writeFile(path.join(logsDir, `debug-mask-${Date.now()}.png`), await mask.getBuffer("image/png"));
mask.blur(2);
const maskBuf = await mask.getBuffer("image/png");
const mergedParams = {
mode: "edit",
prompt: input.prompt ?? "",
width: adjW,
height: adjH,
};
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:inpaint",
maskBuf,
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}
// ─────────────────────────────────────────────────────────────────────────────
// handleOutpaint: load canvas, build black-on-white mask
// (white = region outside image = area to extend), call handleGenerateImage in
// edit mode with sourceTag canvas:outpaint.
// ─────────────────────────────────────────────────────────────────────────────
async function handleOutpaint(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.OutpaintToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [{ type: "text", text: `Invalid outpaint parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}` }],
};
}
const input = parsed.data;
let rawBuf;
let resolvedBBoxes = [{ left: 0, top: 0, right: 0, bottom: 0 }];
let chatDir;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (!rawCanvas || !chatDir)
throw new Error("canvas is required for outpaint");
const st = await (0, core_bundle_mjs_1.readState)(chatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(chatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: pref.index });
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
// Resolve crop coordinates from mask/crop record or detectLabel(s).
let labels = input.detectLabel ?? [];
let indices = input.detectIndex ?? [];
// Auto-select detectLabel when omitted: with a single distinct label across the relevant
// detect_object record's detections (canvas itself, or its original source — see
// findDetectRecordForCanvas), use it (like detectIndex's default of 0).
if (labels.length === 0) {
const autoLabel = autoSelectDetectLabel(imageRecords, rawCanvas);
if (autoLabel)
labels = [autoLabel];
}
// Option A: single label + no explicit detectIndex → auto-expand to all detections for that label.
if (labels.length === 1 && indices.length === 0) {
const allIndices = expandDetectIndices(imageRecords, rawCanvas, labels[0]);
if (allIndices.length > 1) {
labels = Array(allIndices.length).fill(labels[0]);
indices = allIndices;
}
}
// Option B: single label + multiple explicit indices → expand labels to match.
if (labels.length === 1 && indices.length > 1) {
labels = Array(indices.length).fill(labels[0]);
}
if (labels.length > 0) {
const boxes = [];
let resolvedSrcCanvas = undefined;
for (let li = 0; li < labels.length; li++) {
const label = labels[li];
const detectIdx = indices[li] !== undefined ? indices[li] : (indices[0] !== undefined ? indices[0] : 0);
const detectResult = resolveDetectionCrop(imageRecords, rawCanvas, label, detectIdx, undefined // frameAdjust applied as final step after per-box overrides
);
if (!detectResult.ok)
throw new Error(detectResult.error);
// Per-box overrides: scalar = all boxes; array[li] non-null = override; null/absent = keep detect value.
const oL = resolveBoxOverride(input.cropLeft, li, detectResult.cropLeft);
const oR = resolveBoxOverride(input.cropRight, li, detectResult.cropRight);
const oT = resolveBoxOverride(input.cropTop, li, detectResult.cropTop);
const oB = resolveBoxOverride(input.cropBottom, li, detectResult.cropBottom);
const tmpInput = {
cropLeft: oL !== undefined ? oL : detectResult.cropLeft,
cropRight: oR !== undefined ? oR : detectResult.cropRight,
cropTop: oT !== undefined ? oT : detectResult.cropTop,
cropBottom: oB !== undefined ? oB : detectResult.cropBottom,
frameAdjust: input.frameAdjust,
};
const resolved = await applyCropFormat(rawBuf, tmpInput);
if (typeof resolved === "string")
throw new Error(resolved);
boxes.push({ left: resolved.left, top: resolved.top, right: resolved.right, bottom: resolved.bottom });
if (detectResult.canvasWasDetectRecord && detectResult.srcCanvas) {
if (resolvedSrcCanvas !== undefined && resolvedSrcCanvas !== detectResult.srcCanvas) {
throw new Error(`Mixed sources — detectLabel entries resolve to different source images ` +
`('${resolvedSrcCanvas}' vs '${detectResult.srcCanvas}'). ` +
`All labels must refer to detections on the same source image.`);
}
resolvedSrcCanvas = detectResult.srcCanvas;
}
}
if (resolvedSrcCanvas) {
const srcRef = parsePrefixedNotation(resolvedSrcCanvas);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
resolvedBBoxes = boxes;
}
else if (pref && pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (found) {
const hasCropFields = typeof found.cropLeft === "number" || typeof found.cropTop === "number" ||
typeof found.cropRight === "number" || typeof found.cropBottom === "number";
if (hasCropFields) {
// Multi-region: prefer bboxes array; fall back to scalar fields for backward compat.
if (Array.isArray(found.bboxes) && found.bboxes.length > 0) {
resolvedBBoxes = found.bboxes;
}
else {
resolvedBBoxes = [{ left: found.cropLeft ?? 0, top: found.cropTop ?? 0, right: found.cropRight ?? 0, bottom: found.cropBottom ?? 0 }];
}
if (typeof found.cropSource === "string") {
const srcRef = parsePrefixedNotation(found.cropSource);
if (srcRef) {
if (srcRef.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({ chatId: currentLmChatId || undefined, explicitAttachmentSource: true, attachmentIndex: srcRef.index });
if (lm?.ok && lm.buffer)
rawBuf = lm.buffer;
}
else if (srcRef.pool === "image") {
const srcRec = imageRecords.find((r) => r?.i === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename)));
}
else if (srcRef.pool === "variant") {
const srcPath = variantPaths.find((v) => v.v === srcRef.index)?.path;
if (srcPath)
rawBuf = await fs_1.default.promises.readFile(srcPath);
}
else {
const srcRec = pictures.find((p) => p?.p === srcRef.index);
if (srcRec?.filename)
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(srcRec.filename || "")));
}
}
}
}
}
}
}
catch (e) {
return { content: [{ type: "text", text: String(e?.message || e) }], isError: true };
}
const canvasSize = await (0, core_bundle_mjs_1.getSize)(rawBuf);
const adjW = canvasSize.width;
const adjH = canvasSize.height;
// Build outpaint mask: black (0x000000ff) inside each BBox (preserve), white (0xffffffff) outside (repaint).
const Jimp = (await import("jimp")).Jimp;
const mask = new Jimp({ width: adjW, height: adjH, color: 0xffffffff });
for (const bbox of resolvedBBoxes) {
const bx1 = Math.round(bbox.left / 100 * adjW);
const bx2 = Math.round((100 - bbox.right) / 100 * adjW);
const by1 = Math.round(bbox.top / 100 * adjH);
const by2 = Math.round((100 - bbox.bottom) / 100 * adjH);
mask.scan(bx1, by1, Math.max(0, bx2 - bx1), Math.max(0, by2 - by1), (_x, _y, idx) => {
mask.bitmap.data[idx] = 0;
mask.bitmap.data[idx + 1] = 0;
mask.bitmap.data[idx + 2] = 0;
mask.bitmap.data[idx + 3] = 255;
});
}
// DEBUG mask save:
// await fs.promises.writeFile(path.join(logsDir, `debug-mask-${Date.now()}.png`), await mask.getBuffer("image/png"));
mask.blur(2);
const maskBuf = await mask.getBuffer("image/png");
const mergedParams = {
mode: "edit",
prompt: input.prompt ?? "",
width: adjW,
height: adjH,
};
if (input.quality !== undefined)
mergedParams.quality = input.quality;
if (input.model !== undefined)
mergedParams.model = input.model;
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:outpaint",
maskBuf,
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}
// handleRefine: image2image with refine overlay, no dimension override
// ─────────────────────────────────────────────────────────────────────────────
async function handleRefine(pluginParams, onProgress) {
const parsed = core_bundle_mjs_1.RefineToolSchemaStrict.safeParse(pluginParams || {});
if (!parsed.success) {
return {
content: [
{
type: "text",
text: `Invalid refine parameters: ${(0, core_bundle_mjs_1.formatZodError)(parsed.error)}`,
},
],
};
}
const input = parsed.data;
// Load canvas buffer
let rawBuf;
// Hoisted so the final handleGenerateImage() call can pass this already-resolved context down
// instead of forcing a second, TTL-gated (60s) re-resolution — see handleZoomIn for the full story.
let currentLmChatId = null;
let chatDir;
try {
let currentLmWorkingDir = null;
try {
const ctx = await (0, core_bundle_mjs_1.getActiveChatContext)();
if (ctx?.chatId)
currentLmChatId = ctx.chatId;
if (ctx?.workingDir)
currentLmWorkingDir = ctx.workingDir;
}
catch { }
if (!currentLmChatId) {
try {
const _r = await (0, core_bundle_mjs_1.resolveActiveLMStudioChatId)();
if (_r?.ok)
currentLmChatId = _r.chatId;
}
catch { }
}
chatDir =
currentLmWorkingDir ||
(currentLmChatId ? (0, core_bundle_mjs_1.getLMStudioWorkingDir)(currentLmChatId) : undefined);
const rawCanvas = typeof input.canvas === "string" ? input.canvas : undefined;
if (rawCanvas && chatDir) {
const resolvedChatDir = chatDir;
const st = await (0, core_bundle_mjs_1.readState)(resolvedChatDir);
const variantRecords = Array.isArray(st?.variants) ? st.variants : [];
const variantPaths = variantRecords
.filter((v) => v && typeof v.filename === "string")
.map((v) => ({ v: v.v || 1, path: path_1.default.join(resolvedChatDir, v.filename) }));
const pictures = Array.isArray(st?.pictures) ? st.pictures : [];
const imageRecords = Array.isArray(st?.images) ? st.images : [];
const pref = parsePrefixedNotation(rawCanvas);
if (!pref) {
const pathToken = await resolveCanvasPathToken(rawCanvas, chatDir);
if (!pathToken)
throw new Error(`Invalid canvas notation: ${rawCanvas}`);
rawBuf = await fs_1.default.promises.readFile(pathToken);
}
else if (pref.pool === "attachment") {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: currentLmChatId || undefined,
explicitAttachmentSource: true,
attachmentIndex: pref.index,
});
if (!lm?.ok || !lm.buffer)
throw new Error(`Attachment a${pref.index} not found.`);
rawBuf = lm.buffer;
}
else if (pref.pool === "image") {
const found = imageRecords.find((r) => r?.i === pref.index);
if (!found)
throw new Error(`Image i${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(path_1.default.join(chatDir, String(found.filename)));
}
else if (pref.pool === "variant") {
const found = variantPaths.find((v) => v.v === pref.index);
if (!found)
throw new Error(`Variant v${pref.index} not found.`);
rawBuf = await fs_1.default.promises.readFile(found.path);
}
else {
// picture pool (pN)
const found = pictures.find((p) => p?.p === pref.index);
if (!found)
throw new Error(`Picture p${pref.index} not found.`);
rawBuf = await resolvePictureBuffer(found, chatDir, pref.index);
}
}
else {
const lm = await (0, core_bundle_mjs_1.resolveImg2ImgSourceLMStudio)({
chatId: input.canvas ? undefined : undefined,
explicitAttachmentSource: true,
});
if (!lm?.ok || !lm.buffer)
throw new Error("No source image available.");
rawBuf = lm.buffer;
}
}
catch (e) {
return {
content: [{ type: "text", text: String(e?.message || e) }],
isError: true,
};
}
const mergedParams = {
mode: "image2image",
model: input.model,
};
if (input.canvas !== undefined)
mergedParams.canvas = input.canvas;
if (input.width !== undefined)
mergedParams.width = input.width;
if (input.height !== undefined)
mergedParams.height = input.height;
if (input.imageFormat !== undefined)
mergedParams.imageFormat = input.imageFormat;
return handleGenerateImage(mergedParams, onProgress, {
presuppliedSourceBuf: rawBuf,
sourceTag: "canvas:refine",
chatId: currentLmChatId ?? undefined,
workingDir: chatDir,
});
}