Project Files
src / services / drawThingsGrpcService.ts
import path from "path";
import fs from "fs";
import { fileURLToPath } from "url";
import * as grpc from "@grpc/grpc-js";
import * as protoLoader from "@grpc/proto-loader";
import {
getSize as imgGetSize,
toPng as imgToPng,
defaultParams,
defaultParamsImg2Img,
drawthingsLimits,
defaultParamsEdit,
drawthingsEditLimits,
ImageGenerationParams,
validateImageGenerationParams,
buildAuditLogger,
checkDrawThingsGrpcAssets,
getDefaultFpsForModel,
} from "../core-bundle.mjs";
import crypto from "crypto";
import { spawnSync } from "child_process";
import os from "os";
import { ImageBackend, BackendGenerateResult, ProgressCallback } from "./imageBackend.js";
import { ModelId } from "./modelOverlays.js";
import { getEffectiveOverlay } from "./customConfigsLoader.js";
import { defaultParamsText2Video } from "./defaultParamsDrawThingsText2Video.js";
import {
defaultParamsImage2Video,
drawthingsLimits as drawthingsVideoLimits,
} from "./defaultParamsDrawThingsImage2Video.js";
// Simple file logger (aligned with index.ts logs directory) – CJS-safe
// Local project-root resolver (no top-level await, CJS-safe)
const __svcDir = (() => {
try {
const p = process.argv && process.argv[1] ? process.argv[1] : process.cwd();
return path.dirname(p);
} catch {
return process.cwd();
}
})();
function __resolveProjectRootFrom(startDir: string): string {
// Prefer directory containing manifest.json or package.json
try {
let dir = startDir;
for (let i = 0; i < 50; i++) {
try {
const hasManifest = fs.existsSync(path.join(dir, "manifest.json"));
const hasPkg = fs.existsSync(path.join(dir, "package.json"));
if (hasManifest || hasPkg) return dir;
} catch {}
const parent = path.dirname(dir);
if (parent === dir) break;
dir = parent;
}
} catch {}
// Else, fallback to dist/src markers
{
let dir = startDir;
for (let i = 0; i < 20; i++) {
if (path.basename(dir) === "dist") return path.dirname(dir);
const parent = path.dirname(dir);
if (parent === dir) break;
dir = parent;
}
}
{
let dir = startDir;
for (let i = 0; i < 5; i++) {
if (path.basename(dir) === "src") return path.dirname(dir);
const parent = path.dirname(dir);
if (parent === dir) break;
dir = parent;
}
}
return startDir;
}
const __projectRoot = __resolveProjectRootFrom(__svcDir);
const __logsDir = path.join(__projectRoot, "logs");
try {
if (!fs.existsSync(__logsDir)) fs.mkdirSync(__logsDir, { recursive: true });
} catch {}
function localTs(): string {
try {
return new Date().toLocaleString(undefined, {
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
hour12: false,
timeZoneName: "short",
});
} catch {
return new Date().toString();
}
}
function logFile(msg: string) {
const line = `${localTs()} - ${msg}\n`;
try {
fs.appendFileSync(path.join(__logsDir, "generate-image-plugin.log"), line);
} catch {}
}
function logErr(msg: string) {
const line = `${localTs()} - ERROR: ${msg}\n`;
try {
fs.appendFileSync(path.join(__logsDir, "error.log"), line);
} catch {}
try { console.error(`[draw-things-chat] ${msg}`); } catch {}
}
type ExternalToolCmd = {
cmd: string;
argsPrefix: string[];
display: string;
impl: "ts" | "mac-bin";
origin: "env" | "auto";
};
type TensorHelperName = "GRPCBin2PNG" | "PNG2GRPCBin";
type TensorHelpersMode = "auto" | "ts" | "bin";
function getTensorHelpersMode(): TensorHelpersMode {
const raw = String(process.env.DT_TENSOR_HELPERS_MODE || "auto")
.trim()
.toLowerCase();
if (raw === "ts" || raw === "node" || raw === "js") return "ts";
if (raw === "bin" || raw === "binary" || raw === "native") return "bin";
return "auto";
}
function buildTensorHelperCandidates(
projectRoot: string,
helperName: TensorHelperName,
envCmd?: string
): string[] {
const jsPath = path.resolve(
projectRoot,
"dist",
"helpers",
`${helperName}.js`
);
const binDistPath = path.resolve(projectRoot, "dist", "helpers", helperName);
const mode = getTensorHelpersMode();
const ordered =
mode === "bin"
? [envCmd, binDistPath, jsPath]
: [envCmd, jsPath, binDistPath];
return ordered.filter(Boolean) as string[];
}
function logToolSelection(context: string, tool: ExternalToolCmd | null) {
if (!tool) {
logFile(`[tool] ${context} not found (no decoder/encoder available)`);
return;
}
logFile(
`[tool] ${context} origin=${tool.origin} impl=${tool.impl} path=${tool.display}`
);
}
function resolveExternalToolCandidates(
projectRoot: string,
helperName: TensorHelperName,
envCmd?: string
): ExternalToolCmd[] {
const candidates = buildTensorHelperCandidates(
projectRoot,
helperName,
envCmd
);
const tools: ExternalToolCmd[] = [];
for (const c of candidates) {
try {
const abs = path.isAbsolute(c) ? c : path.resolve(projectRoot, c);
if (!fs.existsSync(abs)) continue;
const origin: ExternalToolCmd["origin"] = c === envCmd ? "env" : "auto";
if (abs.endsWith(".js")) {
tools.push({
cmd: process.execPath,
argsPrefix: [abs],
display: abs,
impl: "ts",
origin,
});
continue;
}
try {
const st = fs.statSync(abs);
if ((st.mode & 0o111) === 0) {
try {
fs.chmodSync(abs, 0o755);
} catch {}
}
} catch {}
tools.push({
cmd: abs,
argsPrefix: [],
display: abs,
impl: "mac-bin",
origin,
});
} catch {}
}
return tools;
}
function logToolCandidates(context: string, tools: ExternalToolCmd[]) {
if (!tools.length) {
logFile(`[tool] ${context} not found (no decoder/encoder available)`);
return;
}
logFile(
`[tool] ${context} candidates=${tools.length} primary=${tools[0].display}`
);
}
/**
* Normalize numFrames for the Draw Things backend.
* Valid values: 1 (image), or multiples of 32 + 1 (e.g. 33, 65, 97, 129, ..., 641).
* Accepts ×32 (auto-corrects to ×32+1) and rounds invalid values to nearest valid.
*/
function normalizeNumFrames(n: number): { value: number; hint?: string } {
if (n <= 1) return { value: 1 };
if ((n - 1) % 32 === 0) return { value: n }; // ×32+1 → 1:1
if (n % 32 === 0) return { value: n + 1 }; // ×32 → +1
const rounded = Math.round(n / 32) * 32 + 1; // nearest ×32+1
const clamped = Math.max(33, Math.min(641, rounded));
return {
value: clamped,
hint: `numFrames adjusted to ${clamped} (must be multiple of 32 + 1).`,
};
}
/**
* DrawThingsGrpcService
* A transport-swappable alternative to DrawThingsService (HTTP).
*
* Configuration (ENV):
* - DRAWTHINGS_GRPC_TARGET (e.g. "127.0.0.1:50051")
* - DRAWTHINGS_GRPC_PROTO (absolute path to .proto)
* - DRAWTHINGS_GRPC_PACKAGE (e.g. "drawthings")
* - DRAWTHINGS_GRPC_SERVICE (e.g. "StableDiffusion")
* - DRAWTHINGS_GRPC_METHOD_TXT2IMG (default: "Txt2Img")
* - DRAWTHINGS_GRPC_METHOD_IMG2IMG (default: "Img2Img")
*/
export class DrawThingsGrpcService implements ImageBackend {
public readonly name = "drawthings";
public baseUrl: string; // target address host:port
private client: any | null = null;
private serviceName: string;
private methodGenerate: string;
private grpcCompression: "identity" | "gzip" | "deflate";
private tlsMode: "auto" | "on" | "off" = "auto";
private currentSecurity: "unknown" | "tls" | "insecure" = "unknown";
constructor(target?: string) {
this.baseUrl =
target ||
process.env.DRAWTHINGS_GRPC_TARGET ||
`127.0.0.1:${process.env.DRAWTHINGS_GRPC_PORT || 7859}`;
this.serviceName =
process.env.DRAWTHINGS_GRPC_SERVICE || "ImageGenerationService";
// grpc-js maps RPC names to lowerCamelCase
this.methodGenerate = "generateImage";
this.grpcCompression = (
process.env.DRAWTHINGS_GRPC_COMPRESSION || "identity"
).toLowerCase() as any;
this.loadClient();
}
setBaseUrl(url: string): void {
this.baseUrl = url;
this.loadClient();
}
private loadClient(): void {
try {
const projectRoot = __projectRoot;
const resolveProtoPath = (): string => {
const fromEnv = process.env.DRAWTHINGS_GRPC_PROTO;
if (fromEnv && fs.existsSync(fromEnv)) return fromEnv;
const candidates = [
path.resolve(projectRoot, "dist", "interfaces", "imageService.proto"),
path.resolve(projectRoot, "src", "interfaces", "imageService.proto"),
path.resolve(
projectRoot,
"dist",
"src",
"interfaces",
"imageService.proto"
),
path.resolve(__dirname, "..", "interfaces", "imageService.proto"),
];
for (const p of candidates) if (fs.existsSync(p)) return p;
return candidates[0];
};
const protoPath = resolveProtoPath();
const opts: any = {
keepCase: true,
longs: String,
enums: String,
defaults: true,
oneofs: true,
};
const packageDef = protoLoader.loadSync(protoPath, opts);
const grpcObj = (grpc.loadPackageDefinition(packageDef) as any) || {};
const SvcCtor = grpcObj[this.serviceName];
if (!SvcCtor) {
console.error(
`[DrawThingsGrpcService] Service not found: ${this.serviceName} (proto: ${protoPath})`
);
logErr(
`gRPC service not found: ${this.serviceName} (proto: ${protoPath})`
);
try {
const available = Object.keys(grpcObj || {});
console.error(
`[DrawThingsGrpcService] Available services: ${available.join(
", "
)}`
);
logFile(`[gRPC] Available services: ${available.join(", ")}`);
} catch {}
this.client = null;
return;
}
const channelOptions: any = {
"grpc.max_receive_message_length": 64 * 1024 * 1024,
"grpc.max_send_message_length": 64 * 1024 * 1024,
"grpc.ssl_target_name_override": "localhost",
"grpc.default_authority": "localhost",
};
const loadRootCAs = (): Buffer | undefined => {
const tlsDirs = [
path.resolve(projectRoot, "src", "interfaces", "tls"),
path.resolve(projectRoot, "dist", "interfaces", "tls"),
path.resolve(__dirname, "..", "interfaces", "tls"),
];
const baseDir = tlsDirs.find((d) => {
try {
return fs.existsSync(d);
} catch {
return false;
}
});
if (!baseDir) return undefined;
const serverPath = path.join(baseDir, "server_crt.crt");
const rootPath = path.join(baseDir, "root_ca.crt");
if (!fs.existsSync(serverPath) || !fs.existsSync(rootPath))
return undefined;
const serverTxt = fs
.readFileSync(serverPath, "utf8")
.replace(/\r\n/g, "\n")
.trim();
const rootTxt = fs
.readFileSync(rootPath, "utf8")
.replace(/\r\n/g, "\n")
.trim();
if (
!/-----BEGIN CERTIFICATE-----/.test(serverTxt) ||
!/-----BEGIN CERTIFICATE-----/.test(rootTxt)
)
return undefined;
const bundle = `${serverTxt}\n${rootTxt}\n`;
try {
const out = path.join(baseDir, "server.pem");
const prev = fs.existsSync(out)
? fs.readFileSync(out, "utf8").replace(/\r\n/g, "\n").trim()
: "";
if (prev !== bundle.trim()) {
fs.writeFileSync(out, bundle, "utf8");
console.log(`[TLS] Generated bundle: ${out}`);
} else {
console.log(`[TLS] Using existing bundle: ${out}`);
}
} catch {}
return Buffer.from(bundle, "utf8");
};
const creds = grpc.credentials.createSsl(loadRootCAs());
this.client = new SvcCtor(this.baseUrl, creds, channelOptions);
this.currentSecurity = "tls";
try {
(globalThis as any).__DT_GRPC_TLS_MODE__ = this.tlsMode || "auto";
(globalThis as any).__DT_GRPC_TLS_SELECTED__ = this.currentSecurity;
} catch {}
try {
const proto = Object.getPrototypeOf(this.client) || {};
const methods = Object.getOwnPropertyNames(proto).filter(
(k) =>
typeof (this.client as any)[k] === "function" && k !== "constructor"
);
console.log(
`[DrawThingsGrpcService] client methods: ${methods.join(", ")}`
);
logFile(`[gRPC] client methods: ${methods.join(", ")}`);
} catch {}
} catch (e) {
console.error(
`[DrawThingsGrpcService] loadClient error: ${
e instanceof Error ? e.message : String(e)
}`
);
logErr(`loadClient error: ${e instanceof Error ? e.message : String(e)}`);
this.client = null;
}
}
async checkApiConnection(): Promise<boolean> {
if (!this.client) return false;
// Try Echo first (surfaces sharedSecret requirement)
try {
const echo = (this.client as any)["echo"]?.bind(this.client);
if (typeof echo === "function") {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const req: any = { name: "generate-image-plugin" };
if (sharedSecret) req.sharedSecret = sharedSecret;
await new Promise<void>((resolve, reject) => {
echo(req, (err: any, _resp: any) => {
if (err) return reject(err);
resolve();
});
});
return true;
}
} catch (e) {
try {
const msg =
e && (e as any).message ? String((e as any).message) : String(e);
const code = (e as any)?.code;
console.error(
`[gRPC check] echo failed (mode=${this.currentSecurity}) code=${
code ?? "?"
} msg=${msg}`
);
logErr(
`gRPC echo failed (mode=${this.currentSecurity}) code=${
code ?? "?"
} msg=${msg}`
);
} catch {}
// If auto mode, attempt opposite security on first failure
if (this.tlsMode === "auto") {
// Toggle mode by rebuilding client quickly with TLS and reasonable authority candidates
try {
const __dirname = __svcDir;
const projectRoot = __resolveProjectRootFrom(__dirname);
const protoPath = (() => {
const fromEnv = process.env.DRAWTHINGS_GRPC_PROTO;
if (fromEnv && fs.existsSync(fromEnv)) return fromEnv;
const c1 = path.resolve(
projectRoot,
"src",
"interfaces",
"imageService.proto"
);
if (fs.existsSync(c1)) return c1;
const c3 = path.resolve(
projectRoot,
"dist",
"interfaces",
"imageService.proto"
);
if (fs.existsSync(c3)) return c3;
// no dist/src fallback in Rollup build
return c1;
})();
const packageDef = protoLoader.loadSync(protoPath, {
keepCase: true,
longs: String,
enums: String,
defaults: true,
oneofs: true,
});
const grpcObj = (grpc.loadPackageDefinition(packageDef) as any) || {};
const SvcCtor = grpcObj[this.serviceName];
if (SvcCtor) {
const root = (() => {
const pieces: Buffer[] = [];
const tlsDirCandidates = [
path.resolve(projectRoot, "src", "interfaces", "tls"),
path.resolve(projectRoot, "dist", "interfaces", "tls"),
];
const names = [
"ca.crt",
"ca.pem",
"server.crt",
"server.pem",
"rootCA.crt",
"rootCA.pem",
];
for (const d of tlsDirCandidates) {
for (const n of names) {
try {
const p = path.join(d, n);
if (fs.existsSync(p)) pieces.push(fs.readFileSync(p));
} catch {}
}
}
return pieces.length ? Buffer.concat(pieces) : undefined;
})();
const hostname =
String(this.baseUrl || "").split(":")[0] || "localhost";
const candidateSet = new Set<string>([
hostname,
"localhost",
"server",
"drawthings",
"draw-things",
]);
try {
const { X509Certificate } = await import("crypto");
const protoDirs = [
path.resolve(projectRoot, "src", "interfaces", "tls"),
path.resolve(projectRoot, "dist", "interfaces", "tls"),
];
const certFiles = [
"server_crt.crt",
"server_cert.crt",
"server.crt",
"server.pem",
"root_ca.crt",
"rootCA.crt",
"ca.crt",
"ca.pem",
];
for (const d of protoDirs) {
for (const f of certFiles) {
const p = path.join(d, f);
try {
if (!fs.existsSync(p)) continue;
const txt = fs.readFileSync(p, "utf8");
const blocks = txt
.split(/-----END CERTIFICATE-----/g)
.map((b) =>
b.includes("BEGIN CERTIFICATE")
? b + "-----END CERTIFICATE-----\n"
: ""
)
.filter((b) => b.trim().length > 0);
for (const pem of blocks) {
try {
// @ts-ignore
const x = new (X509Certificate as any)(pem);
const subj: string = (x as any).subject || "";
const san: string = (x as any).subjectAltName || "";
const m = subj.match(/CN=([^,\n]+)/);
if (m && m[1]) candidateSet.add(m[1].trim());
if (san) {
san
.split(/[,]/)
.map((s: string) => s.trim())
.filter((s: string) => /^DNS:/i.test(s))
.map((s: string) => s.replace(/^DNS:\s*/i, ""))
.forEach((dname: string) =>
candidateSet.add(dname)
);
}
} catch {}
}
} catch {}
}
}
} catch {}
const authCandidates = Array.from(candidateSet);
// Try TLS with different authorities
for (const auth of authCandidates) {
const channelOptions = {
"grpc.max_receive_message_length": 64 * 1024 * 1024,
"grpc.max_send_message_length": 64 * 1024 * 1024,
"grpc.ssl_target_name_override": auth,
"grpc.default_authority": auth,
} as any;
const creds = grpc.credentials.createSsl(root);
this.client = new SvcCtor(this.baseUrl, creds, channelOptions);
this.currentSecurity = "tls";
try {
(globalThis as any).__DT_GRPC_TLS_SELECTED__ =
this.currentSecurity;
} catch {}
try {
const echo2 = (this.client as any)["echo"]?.bind(this.client);
if (typeof echo2 === "function") {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const req: any = { name: "generate-image-plugin" };
if (sharedSecret) req.sharedSecret = sharedSecret;
await new Promise<void>((resolve2, reject2) => {
echo2(req, (err2: any) =>
err2 ? reject2(err2) : resolve2()
);
});
return true;
}
} catch (e2) {
try {
const msg =
e2 && (e2 as any).message
? String((e2 as any).message)
: String(e2);
const code = (e2 as any)?.code;
console.error(
`[gRPC check] TLS echo failed (authority=${auth}) code=${
code ?? "?"
} msg=${msg}`
);
} catch {}
}
}
// As a last check, retry insecure (in case server was plaintext)
{
const channelOptions = {
"grpc.max_receive_message_length": 64 * 1024 * 1024,
"grpc.max_send_message_length": 64 * 1024 * 1024,
} as any;
const creds = grpc.credentials.createInsecure();
this.client = new SvcCtor(this.baseUrl, creds, channelOptions);
this.currentSecurity = "insecure";
try {
(globalThis as any).__DT_GRPC_TLS_SELECTED__ =
this.currentSecurity;
} catch (e3) {
try {
const msg =
e3 && (e3 as any).message
? String((e3 as any).message)
: String(e3);
const code = (e3 as any)?.code;
console.error(
`[gRPC check] insecure echo retry failed code=${
code ?? "?"
} msg=${msg}`
);
} catch {}
}
try {
const echo3 = (this.client as any)["echo"]?.bind(this.client);
if (typeof echo3 === "function") {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const req: any = { name: "generate-image-plugin" };
if (sharedSecret) req.sharedSecret = sharedSecret;
await new Promise<void>((resolve3, reject3) => {
echo3(req, (err3: any) =>
err3 ? reject3(err3) : resolve3()
);
});
return true;
}
} catch {}
}
}
} catch {}
}
}
const deadline = new Date(Date.now() + 8000);
return new Promise((resolve) => {
try {
(this.client as any).waitForReady(deadline, (err: any) => {
resolve(!err);
});
} catch {
resolve(false);
}
});
}
private ensureClient(): void {
if (!this.client)
throw new Error(
"gRPC client not configured. Set DRAWTHINGS_GRPC_* envs."
);
}
private toBackendError(e: any): BackendGenerateResult {
let status: number | undefined = undefined;
let msg = "unknown error";
if (e && typeof e === "object") {
if (typeof e.code === "number") status = e.code as number; // grpc status code
if (typeof e.details === "string") msg = e.details;
else if (typeof e.message === "string") msg = e.message;
}
return { isError: true, status, errorMessage: msg };
}
async generateImage(
inputParams: Partial<ImageGenerationParams> = {},
onProgress?: ProgressCallback
): Promise<BackendGenerateResult> {
try {
const __tmpDirs: string[] = [];
const __registerTmp = (d: string) => {
try {
__tmpDirs.push(d);
} catch {}
};
const __cleanupTmp = () => {
for (const d of __tmpDirs) {
try {
fs.rmSync(d, { recursive: true, force: true });
} catch {}
}
};
const isVideoTxt2Vid = (inputParams as any)?._dt_video_mode === "txt2vid";
const baseDefaults = isVideoTxt2Vid ? defaultParamsText2Video as typeof defaultParams : defaultParams;
const baseOverlayMode: "txt2img" | "txt2vid" = isVideoTxt2Vid ? "txt2vid" : "txt2img";
const audit = buildAuditLogger({ backend: this.name, mode: baseOverlayMode });
// NOTE: Service-level audit is kept minimal; full audit in core (tools.ts)
this.ensureClient();
// We'll build the FlatBuffer config AFTER canonical mapping so steps/size/variants land correctly
let configBytes: Buffer | undefined = undefined;
// Capture effective fields from the final FlatBuffer view
let payloadWidth: number | undefined;
let payloadHeight: number | undefined;
let payloadSteps: number | undefined;
let payloadSeed: number | undefined;
let payloadSeedMode: string | undefined;
let payloadModel: string | undefined;
let payloadNumFrames: number = 1;
let payloadFps: number = 24;
let requiredModelFile: string | undefined;
let requiredLoraFiles: string[] | undefined;
// Capture overlay source for audit logging
let overlaySource: "custom" | "modelOverlay" | "default" = "default";
let overlayPreset: string | undefined;
let seedSource: "custom" | "default" = "default";
let seedModeSource: "custom" | "default" = "default";
let defaultsUsed: string | undefined;
let overlayLookupMode: "txt2img" | "img2img" | "edit" | "txt2vid" | "img2vid" | undefined;
let strengthUsed: number | undefined;
let stepsUsed: number | undefined;
let samplerUsed: string | undefined;
let guidanceScaleUsed: number | undefined;
let shiftUsed: number | undefined;
let resolutionDependentShiftUsed: boolean | undefined;
let compressionArtifactsUsed: string | null | undefined;
let compressionArtifactsQualityUsed: number | null | undefined;
// validate + canonical mapping
let params: Partial<ImageGenerationParams> = {};
try {
const r = validateImageGenerationParams(inputParams);
if (r.valid) params = inputParams;
} catch {}
if (
params.random_string &&
(!params.prompt || Object.keys(params).length === 1)
) {
params.prompt = params.random_string;
delete (params as any).random_string;
}
if (!params.prompt)
params.prompt = inputParams.prompt || baseDefaults.prompt;
const promptOriginTxt =
typeof (inputParams as any)?.prompt === "string" &&
String((inputParams as any).prompt).trim()
? "user"
: `default:drawthings:${baseOverlayMode}`;
let usedImageFormat: string | undefined = undefined;
let usedQuality: string | undefined = undefined;
try {
const fmt = (inputParams as any).imageFormat as string | undefined;
// Shorthand should not override explicit dimensions (core passes requested_effective via width/height).
if ((params as any).width == null && (params as any).height == null) {
if (fmt === "square") {
params.width = 1024;
params.height = 1024;
usedImageFormat = "square";
} else if (fmt === "landscape") {
params.width = 1024;
params.height = 768;
usedImageFormat = "landscape";
} else if (fmt === "portrait") {
params.width = 768;
params.height = 1024;
usedImageFormat = "portrait";
} else if (fmt === "16:9") {
params.width = 1024;
params.height = 576;
usedImageFormat = "16:9";
}
}
const qual = (inputParams as any).quality as string | undefined;
if (qual === "low") {
params.steps = 4;
usedQuality = "low";
} else if (qual === "medium") {
params.steps = 8;
usedQuality = "medium";
} else if (qual === "high") {
params.steps = 12;
usedQuality = "high";
} else {
usedQuality = "auto";
}
const variants = (inputParams as any).variants as number | undefined;
if (typeof variants === "number") {
const v = Math.max(1, Math.min(4, Math.round(variants)));
(params as any).batch_size = v;
(params as any).batch_count = 1;
}
// Video: normalize numFrames to valid ×32+1 value
if (isVideoTxt2Vid) {
const rawNumFrames = (inputParams as any).numFrames as number | undefined;
if (typeof rawNumFrames === "number" && rawNumFrames > 1) {
const normalized = normalizeNumFrames(rawNumFrames);
(params as any).num_frames = normalized.value;
if (normalized.hint) console.info(`[txt2vid] ${normalized.hint}`);
}
}
} catch {}
// Build FlatBuffer configuration from effective params (defaults overlaid with mapped values)
try {
const allowedKeys = new Set(Object.keys(baseDefaults));
const filtered: Record<string, any> = {};
for (const [k, v] of Object.entries(params)) {
if (allowedKeys.has(k)) filtered[k] = v as any;
}
// Apply effective overlay (Custom Configs → Model Overlay → Defaults)
const modelId = (params as any).model as ModelId | undefined;
const {
source,
presetName,
params: overlayParams,
} = getEffectiveOverlay(modelId, baseOverlayMode);
// Size must be controlled by tool/core (or defaults), not by overlays.
// Batch must be controlled by tool/core via variants parameter.
// Upscaler must be controlled by tool/core via _dt_needs_upscaler decision.
const overlayParamsNoSize = (() => {
if (!overlayParams) return overlayParams;
const o: any = { ...(overlayParams as any) };
delete o.width;
delete o.height;
delete o.batch_count;
delete o.batch_size;
delete o.batchCount;
delete o.batchSize;
delete o.upscaler;
delete o.upscaler_scale;
delete o.upscalerScale;
delete o.fps;
return o;
})();
// Capture overlay info for result metadata
overlaySource = source;
overlayPreset = presetName;
// Tool-level preset 'auto' means: do not override engine model.
// LM Studio UI may send "auto" explicitly; never forward that literal to Draw Things.
if (modelId === "auto") {
delete filtered.model;
}
if (overlayParams) {
const msg = `Applying ${source} overlay for '${modelId ?? 'auto'}' (${baseOverlayMode}, gRPC)${
presetName ? ` preset=${presetName}` : ""
}`;
logFile(msg);
console.info(msg);
// Remove 'model' from filtered so overlay model filename wins
delete filtered.model;
}
// Log overlay source to audit
audit.setOutput({
overlay_source: source,
...(presetName && { overlay_preset: presetName }),
});
// Seed policy:
// - Tool interface does not accept seed/seed_mode (core uses strict minimal schema).
// - Custom Configs MAY set seed (including -1) and seed_mode and should be effective.
const effectiveSeed =
source === "custom" &&
typeof (overlayParamsNoSize as any)?.seed === "number" &&
Number.isFinite((overlayParamsNoSize as any)?.seed)
? (overlayParamsNoSize as any).seed
: baseDefaults.seed;
const effectiveSeedMode =
source === "custom" &&
typeof (overlayParamsNoSize as any)?.seed_mode === "string" &&
String((overlayParamsNoSize as any).seed_mode).trim()
? String((overlayParamsNoSize as any).seed_mode)
: (baseDefaults as any).seed_mode;
seedSource =
source === "custom" &&
typeof (overlayParamsNoSize as any)?.seed === "number" &&
Number.isFinite((overlayParamsNoSize as any)?.seed)
? "custom"
: "default";
seedModeSource =
source === "custom" &&
typeof (overlayParamsNoSize as any)?.seed_mode === "string" &&
String((overlayParamsNoSize as any).seed_mode).trim()
? "custom"
: "default";
const effective = {
...baseDefaults,
...(overlayParamsNoSize || {}),
...filtered,
seed: effectiveSeed,
seed_mode: effectiveSeedMode,
} as Record<string, any>;
defaultsUsed = isVideoTxt2Vid ? "defaultParamsDrawThingsText2Video" : "defaultParamsDrawThingsTxt2Img";
overlayLookupMode = baseOverlayMode;
strengthUsed =
typeof effective.strength === "number" && Number.isFinite(effective.strength)
? effective.strength
: undefined;
stepsUsed =
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined;
samplerUsed =
typeof effective.sampler === "string" && String(effective.sampler).trim()
? String(effective.sampler)
: undefined;
guidanceScaleUsed =
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined;
shiftUsed =
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined;
resolutionDependentShiftUsed =
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined;
compressionArtifactsUsed =
typeof (effective as any).compressionArtifacts === "string" ? (effective as any).compressionArtifacts
: typeof effective.compression_artifacts === "string" ? effective.compression_artifacts
: null;
compressionArtifactsQualityUsed =
typeof (effective as any).compressionArtifactsQuality === "number"
? (effective as any).compressionArtifactsQuality
: typeof effective.compression_artifacts_quality === "number"
? effective.compression_artifacts_quality
: null;
// Always log which defaults were used as the base, even when overlay_source=default.
// Also log key params like strength to disambiguate edit vs img2img behavior.
audit.setOutput({
overlay_lookup_mode: baseOverlayMode,
defaults_used: isVideoTxt2Vid ? "defaultParamsDrawThingsText2Video" : "defaultParamsDrawThingsTxt2Img",
strength_used:
typeof effective.strength === "number" && Number.isFinite(effective.strength)
? effective.strength
: undefined,
steps_used:
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined,
sampler_used:
typeof effective.sampler === "string" && String(effective.sampler).trim()
? String(effective.sampler)
: undefined,
guidance_scale_used:
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined,
shift_used:
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined,
resolution_dependent_shift_used:
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined,
num_frames_used:
typeof effective.num_frames === "number" ? effective.num_frames : null,
compression_artifacts_used:
typeof (effective as any).compressionArtifacts === "string" ? (effective as any).compressionArtifacts
: typeof effective.compression_artifacts === "string" ? effective.compression_artifacts
: null,
compression_artifacts_quality_used:
typeof (effective as any).compressionArtifactsQuality === "number"
? (effective as any).compressionArtifactsQuality
: typeof effective.compression_artifacts_quality === "number"
? effective.compression_artifacts_quality
: null,
});
// Asset preflight inputs
try {
if (typeof effective.model === "string" && effective.model.trim()) {
requiredModelFile = String(effective.model).trim();
}
// Extract LoRAs for audit: user > overlay > defaults
const userSpecifiedLoras = Object.prototype.hasOwnProperty.call(
filtered,
"loras"
);
const overlaySelected = !!overlayParams;
if (userSpecifiedLoras) {
const ls = Array.isArray(filtered.loras) ? filtered.loras : [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else if (overlaySelected) {
const ls = Array.isArray((overlayParams as any)?.loras)
? (overlayParams as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else {
// Fallback: extract LoRAs from effective params (includes defaults)
const ls = Array.isArray((effective as any)?.loras)
? (effective as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
}
} catch {}
const { buildDtGenerationConfiguration } = await import(
"./drawThingsConfigMapper.js"
);
const cfg = buildDtGenerationConfiguration(effective);
configBytes = cfg.bytes && cfg.bytes.length ? cfg.bytes : undefined;
try {
payloadWidth = (cfg as any)?.view?.width;
payloadHeight = (cfg as any)?.view?.height;
payloadSteps = (cfg as any)?.view?.steps;
payloadSeed = (cfg as any)?.view?.seed;
payloadSeedMode = (cfg as any)?.view?.seed_mode;
payloadModel = String(effective.model || "");
payloadNumFrames = typeof effective.num_frames === "number" ? effective.num_frames : 1;
payloadFps = (typeof filtered.fps === "number" ? filtered.fps : undefined)
?? getDefaultFpsForModel(payloadModel)
?? (typeof effective.fps === "number" ? effective.fps : 24);
} catch {}
try {
const view = cfg.view || {};
audit.setOutput({
flatbuf_num_frames: typeof (view as any).num_frames === "number" ? (view as any).num_frames : null,
});
console.info(
`[grpc-config] fields=${Object.keys(view).join(",")}`
);
} catch {}
} catch {}
// Preflight Echo to detect sharedSecret policy
try {
const echo = (this.client as any)["echo"]?.bind(this.client);
if (typeof echo === "function") {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const req: any = { name: "generate-image-plugin" };
if (sharedSecret) req.sharedSecret = sharedSecret;
await new Promise<void>((resolve, reject) => {
echo(req, (err: any, resp: any) => {
if (err) return reject(err);
if (resp?.sharedSecretMissing && !sharedSecret)
return reject(
new Error(
"gRPC requires sharedSecret. Set DRAWTHINGS_SHARED_SECRET."
)
);
resolve();
});
});
}
} catch (e) {
throw e;
}
// Validate requested model/LoRA presence on server to avoid silent fallback rendering.
try {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const chk = await checkDrawThingsGrpcAssets({
client: this.client,
sharedSecret: sharedSecret || undefined,
modelFile: requiredModelFile,
loraFiles: requiredLoraFiles,
});
if (!chk.ok) {
const overlayCtx =
overlaySource && overlaySource !== "default"
? `overlay=${overlaySource}${
overlayPreset ? ` preset=${overlayPreset}` : ""
}`
: "";
logErr(
`[asset-check] ${chk.details.split("\n")[0]}${
overlayCtx ? ` (${overlayCtx})` : ""
}`
);
return {
isError: true,
status: 400,
errorMessage: overlayCtx
? `${overlayCtx}\n\n${chk.details}`
: chk.details,
};
}
} catch (e) {
// If we cannot validate (RPC failure), do not block generation.
try {
logErr(
`[asset-check] warning: ${
e instanceof Error ? e.message : String(e)
}`
);
} catch {}
}
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const request: any = {
prompt: String(params.prompt || baseDefaults.prompt),
negativePrompt: String((params as any).negative_prompt || ""),
scaleFactor: 1,
keywords: [],
user: "generate-image-plugin",
device: "LAPTOP",
chunked: true,
...(sharedSecret ? { sharedSecret } : {}),
};
if (configBytes) request.configuration = configBytes;
// Optional: include FlatBuffer configuration from env (Base64)
const cfgB64 = process.env.DRAWTHINGS_CONFIG_B64;
if (cfgB64 && typeof cfgB64 === "string" && cfgB64.trim()) {
try {
request.configuration = Buffer.from(cfgB64.trim(), "base64");
} catch {}
}
const call = this.client[this.methodGenerate]?.bind(this.client);
if (typeof call !== "function")
throw new Error(`gRPC method not found: ${this.methodGenerate}`);
const startTime = Date.now();
const imagesBuffers: Buffer[] = [];
const audioBuffers: Buffer[] = [];
let chunkAccum: Buffer | null = null;
let previewBuf: Buffer | null = null;
let lastSeenStep = 0;
const handleData = (resp: any) => {
try {
// progress logging + callback
if (resp?.currentSignpost?.sampling?.step) {
const currentStep = resp.currentSignpost.sampling.step;
lastSeenStep = currentStep;
console.info(
`gRPC progress: sampling step=${currentStep}`
);
logFile(
`t2i progress: sampling step=${currentStep}`
);
// Invoke progress callback if provided
if (onProgress) {
try {
onProgress(
currentStep,
payloadSteps,
`Sampling step ${currentStep}${payloadSteps ? `/${payloadSteps}` : ""}`
);
} catch {}
}
} else if (resp?.currentSignpost != null && onProgress) {
const sp = resp.currentSignpost;
let label: string | undefined;
if (lastSeenStep === 0) {
if (sp.textEncoded != null) label = "Loading...";
else if (sp.imageEncoded != null) label = "Processing...";
} else {
label = "Finishing...";
}
if (label) {
try { onProgress(-1, payloadSteps, label); } catch {}
}
}
if (
resp?.generatedImages &&
Array.isArray(resp.generatedImages)
) {
const list = resp.generatedImages.map((b: any) =>
Buffer.isBuffer(b) ? b : Buffer.from(b)
);
const cs = resp?.chunkState;
const isMore = cs === 1 || cs === "MORE_CHUNKS";
const isLast = cs === 0 || cs === "LAST_CHUNK";
if (isMore) {
if (list.length > 0) {
chunkAccum = chunkAccum
? Buffer.concat([chunkAccum, list[0]])
: Buffer.from(list[0]);
}
} else if (isLast && list.length > 0) {
if (chunkAccum) {
const full = Buffer.concat([chunkAccum, list[0]]);
imagesBuffers.push(full);
chunkAccum = null;
} else {
for (const buf of list) imagesBuffers.push(buf);
}
} else {
for (const buf of list) imagesBuffers.push(buf);
}
}
if (resp?.previewImage) {
const b = resp.previewImage;
previewBuf = Buffer.isBuffer(b) ? b : Buffer.from(b);
}
if (resp?.generatedAudio && Array.isArray(resp.generatedAudio)) {
for (const a of resp.generatedAudio) {
const ab = Buffer.isBuffer(a) ? a : Buffer.from(a);
if (ab.length > 0) audioBuffers.push(ab);
}
}
} catch {}
};
await new Promise<void>((resolve, reject) => {
try {
// Optional response compression negotiation (off by default; enable via env)
const accept = process.env.DRAWTHINGS_GRPC_ACCEPT_ENCODING;
const md = new grpc.Metadata();
if (accept && accept.trim()) {
md.add("grpc-accept-encoding", accept.trim());
}
const stream =
md.get("grpc-accept-encoding").length > 0
? call(request, md)
: call(request);
stream.on("data", handleData);
stream.on("error", (err: any) => reject(err));
stream.on("end", () => resolve());
} catch (e) {
reject(e);
}
});
const endTime = Date.now();
let primaryBuf: Buffer | undefined;
if (imagesBuffers.length > 0) primaryBuf = imagesBuffers[0];
else if (previewBuf) primaryBuf = previewBuf;
if (!primaryBuf) throw new Error("gRPC stream yielded no image data");
// Decode tensor bytes to PNG via external CLI (always attempt)
const decoderTools = resolveExternalToolCandidates(
__projectRoot,
"GRPCBin2PNG",
process.env.DT_DECODER_CMD
);
logToolCandidates("t2i decode", decoderTools);
const decodedBuffers: Buffer[] = [];
if (decoderTools.length) {
// derive width/height hints for decoder from params/defaults
const finalW = Number(params.width || baseDefaults.width || 1024);
const finalH = Number(params.height || baseDefaults.height || 1024);
const decoderCtx = JSON.stringify({
width: finalW,
height: finalH,
channels: 3,
dtype: "f32",
});
for (let i = 0; i < imagesBuffers.length; i++) {
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), "generate-image-plugin-dt-")
);
__registerTmp(tmpDir);
const inPath = path.join(tmpDir, `img-${i + 1}.bin`);
const outPath = path.join(tmpDir, `img-${i + 1}.png`);
fs.writeFileSync(inPath, imagesBuffers[i]);
const args = ["--in", inPath, "--out", outPath];
let decoded: Buffer | null = null;
for (const tool of decoderTools) {
const proc = spawnSync(tool.cmd, [...tool.argsPrefix, ...args], {
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env, DT_DECODER_CTX: decoderCtx },
});
if (proc.error) {
logErr(
`[decoder] exception via ${tool.display}: ${proc.error.message}`
);
continue;
}
if (proc.status !== 0) {
logErr(
`[decoder] exit=${proc.status} via ${tool.display} stderr=${
proc.stderr?.toString?.() || ""
}`
);
continue;
}
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
decoded = fs.readFileSync(outPath);
break;
}
logErr(
`[decoder] no output via ${tool.display}; outPath missing/empty`
);
}
if (decoded) decodedBuffers.push(decoded);
else decodedBuffers.push(imagesBuffers[i]);
} catch {
// fall back: keep raw tensor bytes
decodedBuffers.push(imagesBuffers[i]);
logErr(
`[decoder] exception decoding t2i img-${i + 1}.bin; kept raw`
);
}
}
if (decodedBuffers.length > 0) {
logFile(
`[decoder] t2i: decoded ${decodedBuffers.length} frame(s) → PNG via ${decoderTools[0].display}`
);
}
}
let finalBuffers = decoderTools.length ? decodedBuffers : imagesBuffers;
// Fallback to preview when no generated images were provided
if ((!finalBuffers || finalBuffers.length === 0) && previewBuf) {
if (decoderTools.length) {
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), "generate-image-plugin-dt-")
);
__registerTmp(tmpDir);
const inPath = path.join(tmpDir, `preview.bin`);
const outPath = path.join(tmpDir, `preview.png`);
fs.writeFileSync(inPath, previewBuf);
const finalW = Number(params.width || baseDefaults.width || 1024);
const finalH = Number(
params.height || baseDefaults.height || 1024
);
const decoderCtx = JSON.stringify({
width: finalW,
height: finalH,
channels: 3,
dtype: "f32",
});
const args = ["--in", inPath, "--out", outPath];
let decodedPrev: Buffer | null = null;
for (const tool of decoderTools) {
const proc = spawnSync(tool.cmd, [...tool.argsPrefix, ...args], {
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env, DT_DECODER_CTX: decoderCtx },
});
if (proc.error) continue;
if (proc.status !== 0) continue;
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
decodedPrev = fs.readFileSync(outPath);
logFile(
`[decoder] decoded preview.bin → PNG (${decodedPrev.length} bytes) via ${tool.display}`
);
break;
}
}
finalBuffers = decodedPrev ? [decodedPrev] : [previewBuf];
} catch {
finalBuffers = [previewBuf];
}
} else {
finalBuffers = [previewBuf];
}
}
const pngCandidate = finalBuffers[0];
// Service-level audit disabled (consolidated audit is logged in index.ts)
// Effective dims from payload (FlatBuffer view) when available
const effW = Number(
(payloadWidth as number | undefined) ??
(params as any).width ??
(baseDefaults as any).width ??
1024
);
const effH = Number(
(payloadHeight as number | undefined) ??
(params as any).height ??
(baseDefaults as any).height ??
1024
);
// Service-level audit disabled (consolidated audit is logged in index.ts)
const dataUrl = `data:image/png;base64,${pngCandidate.toString(
"base64"
)}`;
logFile(
`t2i completed: variants=${finalBuffers.length}, chunks=${imagesBuffers.length}, bytes=${pngCandidate.length}`
);
const result: BackendGenerateResult = {
isError: false,
imageBuffer: pngCandidate,
imageData: dataUrl,
images: finalBuffers.length
? finalBuffers.map(
(b) => `data:image/png;base64,${b.toString("base64")}`
)
: undefined,
metadata: {
alt: `Image generated from prompt: ${request.prompt}`,
inference_time_ms: endTime - startTime,
model: payloadModel,
width: effW,
height: effH,
requested_dimensions: {
width: effW,
height: effH,
},
steps: Number.isFinite(Number(payloadSteps))
? Number(payloadSteps)
: undefined,
seed:
typeof payloadSeed === "number" && Number.isFinite(payloadSeed)
? payloadSeed
: undefined,
seed_mode:
typeof payloadSeedMode === "string" && payloadSeedMode.trim()
? payloadSeedMode
: undefined,
seed_source: seedSource,
seed_mode_source: seedModeSource,
prompt_used: String(request.prompt || ""),
prompt_origin: promptOriginTxt,
transport: "grpc",
stream_chunks: imagesBuffers.length,
num_frames: payloadNumFrames,
fps: payloadFps,
// Overlay source tracking for audit
overlay_source: overlaySource,
...(overlayPreset && { overlay_preset: overlayPreset }),
...(defaultsUsed && { defaults_used: defaultsUsed }),
...(overlayLookupMode && { overlay_lookup_mode: overlayLookupMode }),
...(typeof strengthUsed === "number" && { strength_used: strengthUsed }),
...(typeof stepsUsed === "number" && { steps_used: stepsUsed }),
...(samplerUsed && { sampler_used: samplerUsed }),
...(typeof guidanceScaleUsed === "number" && {
guidance_scale_used: guidanceScaleUsed,
}),
...(typeof shiftUsed === "number" && { shift_used: shiftUsed }),
...(typeof resolutionDependentShiftUsed === "boolean" && {
resolution_dependent_shift_used: resolutionDependentShiftUsed,
}),
...(compressionArtifactsUsed != null && {
compression_artifacts_used: compressionArtifactsUsed,
}),
...(compressionArtifactsQualityUsed != null && {
compression_artifacts_quality_used: compressionArtifactsQualityUsed,
}),
...(requiredLoraFiles &&
requiredLoraFiles.length > 0 && { loras_used: requiredLoraFiles }),
},
audioBuffers: audioBuffers.length > 0 ? audioBuffers : undefined,
};
__cleanupTmp();
return result;
} catch (e) {
try {
// attempt temp cleanup on error as well
// (safe even if empty)
} catch {}
return this.toBackendError(e);
}
}
async generateImageImg2Img(
inputParams: Partial<ImageGenerationParams> = {},
sourceBuffer: Buffer,
onProgress?: ProgressCallback
): Promise<BackendGenerateResult> {
try {
const __tmpDirs: string[] = [];
const __registerTmp = (d: string) => {
try {
__tmpDirs.push(d);
} catch {}
};
const __cleanupTmp = () => {
for (const d of __tmpDirs) {
try {
fs.rmSync(d, { recursive: true, force: true });
} catch {}
}
};
const isVideoImg2Vid = (inputParams as any)?._dt_video_mode === "img2vid";
const audit = buildAuditLogger({ backend: this.name, mode: isVideoImg2Vid ? "img2vid" : "img2img" });
// NOTE: Service-level audit is kept minimal; full audit in core (tools.ts)
this.ensureClient();
const i2iProfile = isVideoImg2Vid ? undefined : inputParams?._dt_i2i_profile;
const baseDefaults = isVideoImg2Vid
? defaultParamsImage2Video as typeof defaultParamsImg2Img
: i2iProfile === "edit" ? defaultParamsEdit : defaultParamsImg2Img;
const baseLimits = isVideoImg2Vid
? drawthingsVideoLimits
: i2iProfile === "edit" ? drawthingsEditLimits : drawthingsLimits;
let params: Partial<ImageGenerationParams> = {};
try {
const r = validateImageGenerationParams(inputParams);
if (r.valid) params = inputParams;
} catch {}
if (
params.random_string &&
(!params.prompt || Object.keys(params).length === 1)
) {
params.prompt = params.random_string;
delete (params as any).random_string;
}
if (!params.prompt)
params.prompt = inputParams.prompt || baseDefaults.prompt;
const promptOriginI2I =
typeof (inputParams as any)?.prompt === "string" &&
String((inputParams as any).prompt).trim()
? "user"
: `default:drawthings:${isVideoImg2Vid ? "img2vid" : "img2img"}`;
let usedImageFormat: string | undefined = undefined;
let usedQuality: string | undefined = undefined;
try {
const fmt = (inputParams as any).imageFormat as string | undefined;
if (fmt === "square") {
params.width = 1024;
params.height = 1024;
usedImageFormat = "square";
} else if (fmt === "landscape") {
params.width = 1024;
params.height = 768;
usedImageFormat = "landscape";
} else if (fmt === "portrait") {
params.width = 768;
params.height = 1024;
usedImageFormat = "portrait";
} else if (fmt === "16:9") {
params.width = 1024;
params.height = 576;
usedImageFormat = "16:9";
}
const qual = (inputParams as any).quality as string | undefined;
if (qual === "low") {
params.steps = 4;
usedQuality = "low";
} else if (qual === "medium") {
params.steps = 8;
usedQuality = "medium";
} else if (qual === "high") {
params.steps = 12;
usedQuality = "high";
} else {
usedQuality = "auto";
}
const variants = (inputParams as any).variants as number | undefined;
if (typeof variants === "number") {
const v = Math.max(1, Math.min(4, Math.round(variants)));
(params as any).batch_size = v;
(params as any).batch_count = 1;
}
// Video: normalize numFrames to valid ×32+1 value
if (isVideoImg2Vid) {
const rawNumFrames = (inputParams as any).numFrames as number | undefined;
if (typeof rawNumFrames === "number" && rawNumFrames > 1) {
const normalized = normalizeNumFrames(rawNumFrames);
(params as any).num_frames = normalized.value;
if (normalized.hint) console.info(`[img2vid] ${normalized.hint}`);
}
}
} catch {}
// Ensure PNG, capture source size
let initPng = sourceBuffer;
let srcW: number | undefined;
let srcH: number | undefined;
try {
const meta = await imgGetSize(sourceBuffer);
srcW = meta.width;
srcH = meta.height;
initPng = await imgToPng(sourceBuffer);
} catch {}
// User-requested OUT size (if explicitly provided)
const userReqW = (() => {
const v = (inputParams as any)?.width;
return typeof v === "number" && Number.isFinite(v)
? Math.round(v)
: undefined;
})();
const userReqH = (() => {
const v = (inputParams as any)?.height;
return typeof v === "number" && Number.isFinite(v)
? Math.round(v)
: undefined;
})();
// If source dimensions are known, record them as ORIGINAL size.
// Do NOT override generation size (width/height/target_*) to source size.
// Generation size should follow the requested/effective size.
if (srcW && srcH) {
(params as any).original_width = srcW;
(params as any).original_height = srcH;
const hasW =
typeof (params as any).width === "number" &&
Number.isFinite((params as any).width);
const hasH =
typeof (params as any).height === "number" &&
Number.isFinite((params as any).height);
if (!hasW) (params as any).width = userReqW ?? srcW;
if (!hasH) (params as any).height = userReqH ?? srcH;
const hasTW =
typeof (params as any).target_width === "number" &&
Number.isFinite((params as any).target_width);
const hasTH =
typeof (params as any).target_height === "number" &&
Number.isFinite((params as any).target_height);
if (!hasTW) (params as any).target_width = (params as any).width;
if (!hasTH) (params as any).target_height = (params as any).height;
}
// Build full configuration from effective params (HTTP-equivalent)
let configBytes: Buffer | undefined = undefined;
let payloadWidth: number | undefined;
let payloadHeight: number | undefined;
let payloadSteps: number | undefined;
let payloadSeed: number | undefined;
let payloadSeedMode: string | undefined;
let payloadModel: string | undefined;
let payloadNumFrames: number = 1;
let payloadFps: number = 24;
let requiredModelFile: string | undefined;
let requiredLoraFiles: string[] | undefined;
// Capture overlay source for audit logging
let overlaySource: "custom" | "modelOverlay" | "default" = "default";
let overlayPreset: string | undefined;
// Debug metadata (propagated to core audit via result.metadata)
let defaultsUsed: string | undefined;
let overlayLookupMode: "txt2img" | "img2img" | "edit" | "txt2vid" | "img2vid" | undefined;
let i2iProfileUsed: "img2img" | "edit" | undefined;
let strengthUsed: number | undefined;
let stepsUsed: number | undefined;
let samplerUsed: string | undefined;
let guidanceScaleUsed: number | undefined;
let shiftUsed: number | undefined;
let resolutionDependentShiftUsed: boolean | undefined;
let compressionArtifactsUsed: string | null | undefined;
let compressionArtifactsQualityUsed: number | null | undefined;
try {
const allowedKeys = new Set(Object.keys(baseDefaults));
const filtered: Record<string, any> = {};
for (const [k, v] of Object.entries(params)) {
if (allowedKeys.has(k)) filtered[k] = v as any;
}
// Apply effective overlay (Custom Configs → Model Overlay → Defaults)
// Use i2iProfile to determine correct mode for overlay lookup
const modelId = (params as any).model as ModelId | undefined;
const overlayMode = isVideoImg2Vid ? "img2vid" : i2iProfile === "edit" ? "edit" : "img2img";
const {
source,
presetName,
params: overlayParams,
} = getEffectiveOverlay(modelId, overlayMode);
// Size must be controlled by tool/core (or defaults), not by overlays.
// Batch must be controlled by tool/core via variants parameter.
// Upscaler must be controlled by tool/core via _dt_needs_upscaler decision.
const overlayParamsNoSize = (() => {
if (!overlayParams) return overlayParams;
const o: any = { ...(overlayParams as any) };
delete o.width;
delete o.height;
delete o.batch_count;
delete o.batch_size;
delete o.batchCount;
delete o.batchSize;
delete o.upscaler;
delete o.upscaler_scale;
delete o.upscalerScale;
delete o.fps;
return o;
})();
// Capture overlay info for result metadata
overlaySource = source;
overlayPreset = presetName;
// Tool-level preset 'auto' means: do not override engine model.
if (modelId === "auto") {
delete filtered.model;
}
if (overlayParams) {
const msg = `Applying ${source} overlay for '${modelId ?? 'auto'}' (${overlayMode}, gRPC)${
presetName ? ` preset=${presetName}` : ""
}`;
logFile(msg);
console.info(msg);
// Remove 'model' from filtered so overlay model filename wins
delete filtered.model;
}
// Log overlay source to audit
audit.setOutput({
overlay_source: source,
...(presetName && { overlay_preset: presetName }),
});
const effectiveSeed =
source === "custom" &&
typeof (overlayParamsNoSize as any)?.seed === "number" &&
Number.isFinite((overlayParamsNoSize as any)?.seed)
? (overlayParamsNoSize as any).seed
: baseDefaults.seed;
const effective = {
...baseDefaults,
...(overlayParamsNoSize || {}),
...filtered,
seed: effectiveSeed,
} as Record<string, any>;
defaultsUsed = isVideoImg2Vid
? "defaultParamsDrawThingsImage2Video"
: i2iProfile === "edit"
? "defaultParamsDrawThingsEdit"
: "defaultParamsDrawThingsImg2Img";
overlayLookupMode = overlayMode;
i2iProfileUsed = isVideoImg2Vid ? undefined : i2iProfile;
strengthUsed =
typeof effective.strength === "number" && Number.isFinite(effective.strength)
? effective.strength
: undefined;
stepsUsed =
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined;
samplerUsed =
typeof effective.sampler === "string" && String(effective.sampler).trim()
? String(effective.sampler)
: undefined;
guidanceScaleUsed =
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined;
shiftUsed =
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined;
resolutionDependentShiftUsed =
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined;
compressionArtifactsUsed =
typeof (effective as any).compressionArtifacts === "string" ? (effective as any).compressionArtifacts
: typeof effective.compression_artifacts === "string" ? effective.compression_artifacts
: null;
compressionArtifactsQualityUsed =
typeof (effective as any).compressionArtifactsQuality === "number"
? (effective as any).compressionArtifactsQuality
: typeof effective.compression_artifacts_quality === "number"
? effective.compression_artifacts_quality
: null;
{
const defaultsUsed = isVideoImg2Vid
? "defaultParamsDrawThingsImage2Video"
: i2iProfile === "edit"
? "defaultParamsDrawThingsEdit"
: "defaultParamsDrawThingsImg2Img";
audit.setOutput({
overlay_lookup_mode: overlayMode,
defaults_used: defaultsUsed,
i2i_profile: i2iProfile,
strength_used:
typeof effective.strength === "number" &&
Number.isFinite(effective.strength)
? effective.strength
: undefined,
steps_used:
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined,
sampler_used:
typeof effective.sampler === "string" &&
String(effective.sampler).trim()
? String(effective.sampler)
: undefined,
guidance_scale_used:
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined,
shift_used:
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined,
resolution_dependent_shift_used:
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined,
num_frames_used:
typeof effective.num_frames === "number" ? effective.num_frames : null,
compression_artifacts_used:
typeof (effective as any).compressionArtifacts === "string" ? (effective as any).compressionArtifacts
: typeof effective.compression_artifacts === "string" ? effective.compression_artifacts
: null,
compression_artifacts_quality_used:
typeof (effective as any).compressionArtifactsQuality === "number"
? (effective as any).compressionArtifactsQuality
: typeof effective.compression_artifacts_quality === "number"
? effective.compression_artifacts_quality
: null,
});
}
// Asset preflight inputs
try {
if (typeof effective.model === "string" && effective.model.trim()) {
requiredModelFile = String(effective.model).trim();
}
const userSpecifiedLoras = Object.prototype.hasOwnProperty.call(
filtered,
"loras"
);
const overlaySelected = !!overlayParams;
if (userSpecifiedLoras) {
const ls = Array.isArray(filtered.loras) ? filtered.loras : [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else if (overlaySelected) {
const ls = Array.isArray((overlayParams as any)?.loras)
? (overlayParams as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else {
// Fallback: extract LoRAs from effective params (includes defaults)
const ls = Array.isArray((effective as any)?.loras)
? (effective as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
}
} catch {}
// Upscaler is controlled deterministically by core.
// No heuristics/fallbacks here to avoid legacy drift.
{
const needs = (inputParams as any)?._dt_needs_upscaler;
if (typeof needs !== "boolean") {
throw new Error(
`Invariant failed: _dt_needs_upscaler must be provided by core for ${i2iProfile}`
);
}
if (needs) {
effective.upscaler = baseLimits.upscaler;
effective.upscaler_scale = baseLimits.upscalerScaleFactor;
console.info(`DT gRPC ${i2iProfile} upscaler enabled (core)`);
} else {
effective.upscaler = null;
effective.upscaler_scale = 0;
console.info(`DT gRPC ${i2iProfile} upscaler disabled (core)`);
}
}
const { buildDtGenerationConfiguration } = await import(
"./drawThingsConfigMapper.js"
);
const cfg = buildDtGenerationConfiguration(effective);
configBytes = cfg.bytes && cfg.bytes.length ? cfg.bytes : undefined;
try {
const view = (cfg as any)?.view || {};
payloadWidth = view.width;
payloadHeight = view.height;
payloadSteps = view.steps;
payloadSeed = view.seed;
payloadSeedMode = view.seed_mode;
payloadModel = String(effective.model || "");
payloadNumFrames = typeof effective.num_frames === "number" ? effective.num_frames : 1;
payloadFps = (typeof filtered.fps === "number" ? filtered.fps : undefined)
?? getDefaultFpsForModel(payloadModel)
?? (typeof effective.fps === "number" ? effective.fps : 24);
} catch {}
try {
const view = cfg.view || {};
audit.setOutput({
flatbuf_num_frames: typeof (view as any).num_frames === "number" ? (view as any).num_frames : null,
});
console.info(
`[grpc-config] fields=${Object.keys(view).join(",")}`
);
} catch {}
} catch {}
// Validate requested model/LoRA presence on server to avoid silent fallback rendering.
try {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const chk = await checkDrawThingsGrpcAssets({
client: this.client,
sharedSecret: sharedSecret || undefined,
modelFile: requiredModelFile,
loraFiles: requiredLoraFiles,
});
if (!chk.ok) {
logErr(`[asset-check] ${chk.details.split("\n")[0]}`);
__cleanupTmp();
return {
isError: true,
status: 400,
errorMessage:
overlaySource && overlaySource !== "default"
? `overlay=${overlaySource}${
overlayPreset ? ` preset=${overlayPreset}` : ""
}\n\n${chk.details}`
: chk.details,
};
}
} catch (e) {
try {
logErr(
`[asset-check] warning: ${
e instanceof Error ? e.message : String(e)
}`
);
} catch {}
}
// Validate requested model/LoRA presence on server to avoid silent fallback rendering.
try {
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
const chk = await checkDrawThingsGrpcAssets({
client: this.client,
sharedSecret: sharedSecret || undefined,
modelFile: requiredModelFile,
loraFiles: requiredLoraFiles,
});
if (!chk.ok) {
logErr(`[asset-check] ${chk.details.split("\n")[0]}`);
return {
isError: true,
status: 400,
errorMessage:
overlaySource && overlaySource !== "default"
? `overlay=${overlaySource}${
overlayPreset ? ` preset=${overlayPreset}` : ""
}\n\n${chk.details}`
: chk.details,
};
}
} catch (e) {
try {
logErr(
`[asset-check] warning: ${
e instanceof Error ? e.message : String(e)
}`
);
} catch {}
}
const sharedSecret2 = process.env.DRAWTHINGS_SHARED_SECRET;
const request: any = {
prompt: String(params.prompt || defaultParamsImg2Img.prompt),
negativePrompt: String((params as any).negative_prompt || ""),
scaleFactor: 1,
keywords: [],
user: "generate-image-plugin",
device: "LAPTOP",
chunked: true,
...(sharedSecret2 ? { sharedSecret: sharedSecret2 } : {}),
};
if (configBytes) request.configuration = configBytes;
// Convert PNG → NNC tensor blob via PNG2GRPCBin (always attempt)
const projectRoot2 = __resolveProjectRootFrom(__svcDir);
const encoderTools = resolveExternalToolCandidates(
projectRoot2,
"PNG2GRPCBin",
process.env.DT_ENCODER_CMD
);
logToolCandidates("i2i encode", encoderTools);
if (encoderTools.length) {
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), "generate-image-plugin-enc-")
);
__registerTmp(tmpDir);
const pngPath = path.join(tmpDir, "input.png");
const imgBinPath = path.join(tmpDir, "image.bin");
fs.writeFileSync(pngPath, initPng);
const args = ["--in", pngPath, "--out", imgBinPath];
let lastStdout = "";
let lastStderr = "";
let blob: Buffer | null = null;
let toolUsed: ExternalToolCmd | null = null;
for (const tool of encoderTools) {
try {
try {
if (fs.existsSync(imgBinPath)) fs.unlinkSync(imgBinPath);
} catch {}
const proc = spawnSync(tool.cmd, [...tool.argsPrefix, ...args], {
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env },
});
if (proc.error) {
lastStderr = proc.error.message;
logErr(
`[i2i-encode] exception via ${tool.display}: ${lastStderr}`
);
continue;
}
lastStdout = (proc.stdout || Buffer.alloc(0))
.toString("utf8")
.trim();
lastStderr = (proc.stderr || Buffer.alloc(0))
.toString("utf8")
.trim();
if (typeof proc.status === "number" && proc.status !== 0) {
logErr(
`[i2i-encode] exit=${proc.status} via ${tool.display} stderr=${lastStderr}`
);
continue;
}
if (
fs.existsSync(imgBinPath) &&
fs.statSync(imgBinPath).size > 0
) {
blob = fs.readFileSync(imgBinPath);
toolUsed = tool;
break;
}
logErr(
`[i2i-encode] no output via ${tool.display}; image.bin missing/empty stdout=${lastStdout}`
);
} catch (e) {
logErr(
`[i2i-encode] exception via ${tool.display}: ${
e instanceof Error ? e.message : String(e)
}`
);
continue;
}
}
if (blob && toolUsed) {
// Prefer stdout hex; fallback to local SHA-256
let imgHashBuf: Buffer | null = null;
if (/^[0-9a-fA-F]{64}$/.test(lastStdout)) {
imgHashBuf = Buffer.from(lastStdout, "hex");
} else {
const h = crypto.createHash("sha256").update(blob).digest();
imgHashBuf = h;
}
const hashHex = (imgHashBuf as Buffer).toString("hex");
logFile(
`[encoder] encoded input.png → image.bin (${blob.length} bytes) via ${toolUsed.display}`
);
// CAS preflight: FilesExist + optional UploadFile (single chunk)
try {
const filesExist = (this.client as any)["filesExist"]?.bind(
this.client
);
if (typeof filesExist === "function") {
await new Promise<void>((resolve) => {
filesExist(
{
files: [],
filesWithHash: [hashHex],
...(sharedSecret2 ? { sharedSecret: sharedSecret2 } : {}),
},
(err: any, resp: any) => {
if (err) {
logErr(
`FilesExist error: ${err?.message || String(err)}`
);
return resolve();
}
try {
const existArr: boolean[] = Array.isArray(
resp?.existences
)
? resp.existences
: [];
const exists = existArr[0] === true;
logFile(
`[CAS] FilesExist hash=${hashHex} exists=${exists}`
);
if (!exists) {
try {
const upload = (this.client as any)[
"uploadFile"
]?.call(this.client);
if (upload && typeof upload.write === "function") {
const fname = `image-${hashHex}.bin`;
upload.write({
initRequest: {
filename: fname,
sha256: imgHashBuf,
totalSize: blob.length,
},
...(sharedSecret2
? { sharedSecret: sharedSecret2 }
: {}),
});
upload.write({
chunk: {
content: blob,
filename: fname,
offset: 0,
},
...(sharedSecret2
? { sharedSecret: sharedSecret2 }
: {}),
});
upload.end();
upload.on("end", () =>
logFile(
`[CAS] UploadFile completed hash=${hashHex} size=${blob.length}`
)
);
upload.on("error", (ue: any) =>
logErr(
`[CAS] UploadFile error: ${
ue?.message || String(ue)
}`
)
);
}
} catch (ue) {
logErr(
`[CAS] UploadFile exception: ${
(ue as any)?.message || String(ue)
}`
);
}
}
} catch {}
resolve();
}
);
});
}
} catch {}
request.image = imgHashBuf;
request.contents = [blob];
logFile(
`[i2i-encode] built image.bin (${blob.length} bytes) hash=${(
imgHashBuf as Buffer
).toString("hex")}`
);
// Mask omitted intentionally
// Cleanup
try {
fs.unlinkSync(pngPath);
} catch {}
try {
fs.unlinkSync(imgBinPath);
} catch {}
// no mask cleanup
} else {
console.error(
`[i2i-encode] encoder did not produce image.bin; stdout=${lastStdout}`
);
logErr(
`[i2i-encode] encoder did not produce image.bin; stdout=${lastStdout} stderr=${lastStderr}`
);
}
} catch (e) {
console.error(
`[i2i-encode] encoder failed: ${
e instanceof Error ? e.message : String(e)
}`
);
logErr(
`[i2i-encode] encoder failed: ${
e instanceof Error ? e.message : String(e)
}`
);
}
}
// Fallback when encoder not available or failed: inline PNG (legacy behavior)
if (
!request.image ||
!request.contents ||
!Array.isArray(request.contents) ||
request.contents.length === 0
) {
request.contents = [Buffer.from(initPng)];
}
const call = this.client[this.methodGenerate]?.bind(this.client);
if (typeof call !== "function")
throw new Error(`gRPC method not found: ${this.methodGenerate}`);
const startTime = Date.now();
const imagesBuffers: Buffer[] = [];
const audioBuffers: Buffer[] = [];
let chunkAccum: Buffer | null = null;
let previewBuf: Buffer | null = null;
let lastSeenStep = 0;
const handleData = (resp: any) => {
try {
if (resp?.currentSignpost?.sampling?.step) {
const currentStep = resp.currentSignpost.sampling.step;
lastSeenStep = currentStep;
console.info(
`gRPC progress: sampling step=${currentStep}`
);
logFile(
`i2i progress: sampling step=${currentStep}`
);
// Invoke progress callback if provided
if (onProgress) {
try {
onProgress(
currentStep,
payloadSteps,
`Sampling step ${currentStep}${payloadSteps ? `/${payloadSteps}` : ""}`
);
} catch {}
}
} else if (resp?.currentSignpost != null && onProgress) {
const sp = resp.currentSignpost;
let label: string | undefined;
if (lastSeenStep === 0) {
if (sp.textEncoded != null) label = "Loading...";
else if (sp.imageEncoded != null) label = "Processing...";
} else {
label = "Finishing...";
}
if (label) {
try { onProgress(-1, payloadSteps, label); } catch {}
}
}
if (
resp?.generatedImages &&
Array.isArray(resp.generatedImages)
) {
const list = resp.generatedImages.map((b: any) =>
Buffer.isBuffer(b) ? b : Buffer.from(b)
);
const cs = resp?.chunkState;
const isMore = cs === 1 || cs === "MORE_CHUNKS";
const isLast = cs === 0 || cs === "LAST_CHUNK";
if (isMore) {
if (list.length > 0) {
chunkAccum = chunkAccum
? Buffer.concat([chunkAccum, list[0]])
: Buffer.from(list[0]);
}
} else if (isLast && list.length > 0) {
if (chunkAccum) {
const full = Buffer.concat([chunkAccum, list[0]]);
imagesBuffers.push(full);
chunkAccum = null;
} else {
for (const buf of list) imagesBuffers.push(buf);
}
} else {
for (const buf of list) imagesBuffers.push(buf);
}
}
if (resp?.previewImage) {
const b = resp.previewImage;
previewBuf = Buffer.isBuffer(b) ? b : Buffer.from(b);
}
if (resp?.generatedAudio && Array.isArray(resp.generatedAudio)) {
for (const a of resp.generatedAudio) {
const ab = Buffer.isBuffer(a) ? a : Buffer.from(a);
if (ab.length > 0) audioBuffers.push(ab);
}
}
} catch {}
};
await new Promise<void>((resolve, reject) => {
try {
const stream = call(request);
stream.on("data", handleData);
stream.on("error", (err: any) => reject(err));
stream.on("end", () => resolve());
} catch (e) {
reject(e);
}
});
const endTime = Date.now();
// Always attempt to decode tensor bytes to PNG via external CLI (same as txt2img)
const decoderTools2 = resolveExternalToolCandidates(
__projectRoot,
"GRPCBin2PNG",
process.env.DT_DECODER_CMD
);
logToolCandidates("i2i decode", decoderTools2);
if (decoderTools2.length && imagesBuffers.length > 0) {
const finalW = Number(
params.width || defaultParamsImg2Img.width || 1024
);
const finalH = Number(
params.height || defaultParamsImg2Img.height || 1024
);
const decoderCtx = JSON.stringify({
width: finalW,
height: finalH,
channels: 3,
dtype: "f32",
});
const decoded: Buffer[] = [];
for (let i = 0; i < imagesBuffers.length; i++) {
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), "generate-image-plugin-dt-")
);
__registerTmp(tmpDir);
const inPath = path.join(tmpDir, `img2img-${i + 1}.bin`);
const outPath = path.join(tmpDir, `img2img-${i + 1}.png`);
fs.writeFileSync(inPath, imagesBuffers[i]);
const args = ["--in", inPath, "--out", outPath];
let decodedOne: Buffer | null = null;
for (const tool of decoderTools2) {
const proc = spawnSync(tool.cmd, [...tool.argsPrefix, ...args], {
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env, DT_DECODER_CTX: decoderCtx },
});
if (proc.error) {
logErr(
`[decoder] exception via ${tool.display}: ${proc.error.message}`
);
continue;
}
if (proc.status !== 0) {
logErr(
`[decoder] exit=${proc.status} via ${tool.display} stderr=${
proc.stderr?.toString?.() || ""
}`
);
continue;
}
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
decodedOne = fs.readFileSync(outPath);
break;
}
}
if (decodedOne) {
decoded.push(decodedOne);
try {
fs.unlinkSync(inPath);
} catch {}
} else {
decoded.push(imagesBuffers[i]);
}
} catch {
decoded.push(imagesBuffers[i]);
}
}
if (decoded.length > 0) {
imagesBuffers.length = 0;
imagesBuffers.push(...decoded);
logFile(
`[decoder] i2i: decoded ${decoded.length} frame(s) → PNG via ${decoderTools2[0].display}`
);
}
}
let primaryBuf: Buffer | undefined;
if (imagesBuffers.length > 0) primaryBuf = imagesBuffers[0];
else if (previewBuf) {
// Try decoding preview as well when needed
if (decoderTools2.length && previewBuf) {
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), "generate-image-plugin-dt-")
);
__registerTmp(tmpDir);
const inPath = path.join(tmpDir, `preview-i2i.bin`);
const outPath = path.join(tmpDir, `preview-i2i.png`);
fs.writeFileSync(inPath, previewBuf);
const finalW = Number(
params.width || defaultParamsImg2Img.width || 1024
);
const finalH = Number(
params.height || defaultParamsImg2Img.height || 1024
);
const decoderCtx = JSON.stringify({
width: finalW,
height: finalH,
channels: 3,
dtype: "f32",
});
const args = ["--in", inPath, "--out", outPath];
let decodedPrev: Buffer | null = null;
for (const tool of decoderTools2) {
const proc = spawnSync(tool.cmd, [...tool.argsPrefix, ...args], {
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env, DT_DECODER_CTX: decoderCtx },
});
if (proc.error) continue;
if (proc.status !== 0) continue;
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
decodedPrev = fs.readFileSync(outPath);
logFile(
`[decoder] decoded preview-i2i.bin → PNG (${decodedPrev.length} bytes) via ${tool.display}`
);
break;
}
}
primaryBuf = decodedPrev ? decodedPrev : previewBuf;
try {
fs.unlinkSync(inPath);
} catch {}
} catch {
primaryBuf = previewBuf;
}
} else {
primaryBuf = previewBuf;
}
}
if (!primaryBuf) throw new Error("gRPC stream yielded no image data");
// Service-level audit disabled (consolidated audit is logged in index.ts)
// Service-level audit disabled (consolidated audit is logged in index.ts)
const dataUrl = `data:image/png;base64,${primaryBuf.toString("base64")}`;
logFile(
`i2i completed: images=${imagesBuffers.length}, bytes=${primaryBuf.length}`
);
// Effective dims from payload (FlatBuffer view) when available; else fall back
const effW = Number(
(payloadWidth as number | undefined) ??
(params as any).width ??
(defaultParamsImg2Img as any).width ??
1024
);
const effH = Number(
(payloadHeight as number | undefined) ??
(params as any).height ??
(defaultParamsImg2Img as any).height ??
1024
);
const result: BackendGenerateResult = {
isError: false,
imageBuffer: primaryBuf,
imageData: dataUrl,
images: imagesBuffers.length
? imagesBuffers.map(
(b) => `data:image/png;base64,${b.toString("base64")}`
)
: undefined,
metadata: {
alt: `Image-to-image from prompt: ${request.prompt}`,
inference_time_ms: endTime - startTime,
model: payloadModel,
width: effW,
height: effH,
requested_dimensions: {
width: effW,
height: effH,
},
steps: Number.isFinite(Number(payloadSteps))
? Number(payloadSteps)
: undefined,
seed:
typeof payloadSeed === "number" && Number.isFinite(payloadSeed)
? payloadSeed
: undefined,
seed_mode:
typeof payloadSeedMode === "string" && payloadSeedMode.trim()
? payloadSeedMode
: undefined,
prompt_used: String(request.prompt || ""),
prompt_origin: promptOriginI2I,
transport: "grpc",
stream_chunks: imagesBuffers.length,
num_frames: payloadNumFrames,
fps: payloadFps,
// Overlay source tracking for audit
overlay_source: overlaySource,
...(overlayPreset && { overlay_preset: overlayPreset }),
...(defaultsUsed && { defaults_used: defaultsUsed }),
...(overlayLookupMode && { overlay_lookup_mode: overlayLookupMode }),
...(i2iProfileUsed && { i2i_profile: i2iProfileUsed }),
...(typeof strengthUsed === "number" && { strength_used: strengthUsed }),
...(typeof stepsUsed === "number" && { steps_used: stepsUsed }),
...(samplerUsed && { sampler_used: samplerUsed }),
...(typeof guidanceScaleUsed === "number" && {
guidance_scale_used: guidanceScaleUsed,
}),
...(typeof shiftUsed === "number" && { shift_used: shiftUsed }),
...(typeof resolutionDependentShiftUsed === "boolean" && {
resolution_dependent_shift_used: resolutionDependentShiftUsed,
}),
...(compressionArtifactsUsed != null && {
compression_artifacts_used: compressionArtifactsUsed,
}),
...(compressionArtifactsQualityUsed != null && {
compression_artifacts_quality_used: compressionArtifactsQualityUsed,
}),
...(requiredLoraFiles &&
requiredLoraFiles.length > 0 && { loras_used: requiredLoraFiles }),
},
audioBuffers: audioBuffers.length > 0 ? audioBuffers : undefined,
};
__cleanupTmp();
return result;
} catch (e) {
try {
// attempt temp cleanup on error as well
} catch {}
return this.toBackendError(e);
}
}
/**
* Edit mode generation with multiple reference images.
* Canvas (first buffer) gets priority; remaining are moodboard.
*/
async generateImageEdit(
inputParams: Partial<ImageGenerationParams> = {},
referenceBuffers: Buffer[],
onProgress?: ProgressCallback
): Promise<BackendGenerateResult> {
const rawProfile = inputParams?._dt_i2i_profile;
const i2iProfileUsed = rawProfile === "img2img" ? "img2img" : "edit";
// Multi-reference calls can be used for BOTH user modes:
// - mode='edit' => use edit defaults/overlays
// - mode='image2image' (with moodboard via gRPC) => use img2img defaults/overlays
const overlayMode = i2iProfileUsed;
// For MVP: delegate to generateImageImg2Img with edit profile
// Multi-reference will use contents[] array
if (!referenceBuffers || referenceBuffers.length === 0) {
return {
isError: true,
errorMessage: "Edit mode requires at least one reference image",
};
}
// Refactor-proofing: multi-reference is a shared entry point (edit + image2image).
// If core ever forgets to pass the profile, we default to edit but emit a loud warning.
if (referenceBuffers.length > 1 && rawProfile !== "img2img" && rawProfile !== "edit") {
try {
logFile(
"[multi-ref] WARNING: _dt_i2i_profile missing; defaulting to edit profile. This indicates a core/service interface regression."
);
} catch {}
}
// For single-reference edit, use existing img2img path with edit profile
if (referenceBuffers.length === 1) {
const editParams = { ...inputParams, _dt_i2i_profile: "edit" as const };
return this.generateImageImg2Img(editParams, referenceBuffers[0], onProgress);
}
// Multi-reference edit: encode all buffers and send via contents[]
try {
const __tmpDirs: string[] = [];
const __registerTmp = (d: string) => {
try {
__tmpDirs.push(d);
} catch {}
};
const __cleanupTmp = () => {
for (const d of __tmpDirs) {
try {
fs.rmSync(d, { recursive: true, force: true });
} catch {}
}
};
const audit = buildAuditLogger({
backend: this.name,
mode: overlayMode === "img2img" ? "img2img" : "edit",
});
// NOTE: Service-level audit is kept minimal; full audit in core (tools.ts)
this.ensureClient();
// Overlay tracking for result metadata
let overlaySource: "custom" | "modelOverlay" | "default" = "default";
let overlayPreset: string | undefined;
// Debug metadata (propagated to core audit via result.metadata)
let defaultsUsed: string | undefined;
let overlayLookupMode: "txt2img" | "img2img" | "edit" | undefined;
let strengthUsed: number | undefined;
let stepsUsed: number | undefined;
let samplerUsed: string | undefined;
let guidanceScaleUsed: number | undefined;
let shiftUsed: number | undefined;
let resolutionDependentShiftUsed: boolean | undefined;
// LoRA tracking
let requiredLoraFiles: string[] | undefined;
const baseDefaults =
overlayMode === "edit" ? defaultParamsEdit : defaultParamsImg2Img;
const baseLimits =
overlayMode === "edit" ? drawthingsEditLimits : drawthingsLimits;
let params: Partial<ImageGenerationParams> = {};
try {
const r = validateImageGenerationParams(inputParams);
if (r.valid) params = inputParams;
} catch {}
if (!params.prompt)
params.prompt = inputParams.prompt || baseDefaults.prompt;
// Map variants parameter to batch_size for multi-variant generation
try {
const variants = (inputParams as any).variants as number | undefined;
if (typeof variants === "number") {
const v = Math.max(1, Math.min(4, Math.round(variants)));
(params as any).batch_size = v;
(params as any).batch_count = 1;
}
} catch {}
// Encode all reference images to NNC tensors
const resolveEncoderCmd = (): ExternalToolCmd | null => {
const envCmd = process.env.DT_ENCODER_CMD;
const projectRoot2 = __resolveProjectRootFrom(__svcDir);
const candidates = buildTensorHelperCandidates(
projectRoot2,
"PNG2GRPCBin",
envCmd
);
for (const c of candidates) {
try {
const abs = path.isAbsolute(c) ? c : path.resolve(projectRoot2, c);
if (!fs.existsSync(abs)) continue;
const origin: ExternalToolCmd["origin"] =
c === envCmd ? "env" : "auto";
if (abs.endsWith(".js")) {
return {
cmd: process.execPath,
argsPrefix: [abs],
display: abs,
impl: "ts",
origin,
};
}
try {
const st = fs.statSync(abs);
if ((st.mode & 0o111) === 0) {
try {
fs.chmodSync(abs, 0o755);
} catch {}
}
} catch {}
return {
cmd: abs,
argsPrefix: [],
display: abs,
impl: "mac-bin",
origin,
};
} catch {}
}
return null;
};
const encoderCmd = resolveEncoderCmd();
logToolSelection("edit encode", encoderCmd);
if (!encoderCmd) {
return {
isError: true,
errorMessage:
"PNG2GRPCBin encoder not found for multi-reference edit",
};
}
const contentsTensors: Buffer[] = [];
let primaryHash: Buffer | null = null;
const moodboardHashes: Buffer[] = []; // SHA256 hashes for shuffle hints
let canvasW: number | undefined;
let canvasH: number | undefined;
for (let i = 0; i < referenceBuffers.length; i++) {
const refBuf = referenceBuffers[i];
const pngBuf = await imgToPng(refBuf);
if (i === 0) {
try {
const meta = await imgGetSize(pngBuf);
canvasW = meta.width;
canvasH = meta.height;
} catch {}
}
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), `generate-image-plugin-edit-${i}-`)
);
__registerTmp(tmpDir);
const pngPath = path.join(tmpDir, "input.png");
const imgBinPath = path.join(tmpDir, "image.bin");
fs.writeFileSync(pngPath, pngBuf);
const args = ["--in", pngPath, "--out", imgBinPath];
const proc = spawnSync(
encoderCmd.cmd,
[...encoderCmd.argsPrefix, ...args],
{
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env },
}
);
if (proc.error) {
__cleanupTmp();
return {
isError: true,
errorMessage: `Encoder failed for reference ${i + 1}: ${
proc.error.message
}`,
};
}
if (typeof proc.status === "number" && proc.status !== 0) {
const stdout = proc.stdout ? proc.stdout.toString("utf8") : "";
const stderr = proc.stderr ? proc.stderr.toString("utf8") : "";
logErr(
`[edit] encoder exit=${proc.status} ref=${i + 1} cmd=${
encoderCmd.cmd
} ${[...encoderCmd.argsPrefix, ...args].join(" ")}`
);
if (stdout.trim())
logFile(`[edit] encoder stdout ref=${i + 1}: ${stdout.trim()}`);
if (stderr.trim())
logErr(`[edit] encoder stderr ref=${i + 1}: ${stderr.trim()}`);
__cleanupTmp();
return {
isError: true,
errorMessage: `Encoder failed for reference ${i + 1} (exit ${
proc.status
})${stderr.trim() ? `: ${stderr.trim()}` : ""}`,
};
}
if (!fs.existsSync(imgBinPath) || fs.statSync(imgBinPath).size === 0) {
const stdout = proc.stdout ? proc.stdout.toString("utf8") : "";
const stderr = proc.stderr ? proc.stderr.toString("utf8") : "";
if (stdout.trim())
logFile(`[edit] encoder stdout ref=${i + 1}: ${stdout.trim()}`);
if (stderr.trim())
logErr(`[edit] encoder stderr ref=${i + 1}: ${stderr.trim()}`);
__cleanupTmp();
return {
isError: true,
errorMessage: `Encoder produced no output for reference ${i + 1}`,
};
}
const blob = fs.readFileSync(imgBinPath);
contentsTensors.push(blob);
// Compute SHA256 hash for each tensor
const tensorHash = crypto.createHash("sha256").update(blob).digest();
// First image is the canvas (primary image)
if (i === 0) {
primaryHash = tensorHash;
} else {
// Moodboard images: collect hashes for shuffle hints
moodboardHashes.push(tensorHash);
}
// Cleanup temp files
try {
fs.unlinkSync(pngPath);
} catch {}
try {
fs.unlinkSync(imgBinPath);
} catch {}
}
logFile(
`[${overlayMode === "img2img" ? "image2image" : "edit"}] encoded ${contentsTensors.length} reference images for multi-ref`
);
// Match img2img behavior: internal processing size follows the actual canvas tensor size.
// This keeps edit and img2img identical except for moodboard hints.
if (canvasW && canvasH) {
(params as any).original_width = canvasW;
(params as any).original_height = canvasH;
const hasW =
typeof (params as any).width === "number" &&
Number.isFinite((params as any).width);
const hasH =
typeof (params as any).height === "number" &&
Number.isFinite((params as any).height);
if (!hasW) (params as any).width = canvasW;
if (!hasH) (params as any).height = canvasH;
const hasTW =
typeof (params as any).target_width === "number" &&
Number.isFinite((params as any).target_width);
const hasTH =
typeof (params as any).target_height === "number" &&
Number.isFinite((params as any).target_height);
if (!hasTW) (params as any).target_width = (params as any).width;
if (!hasTH) (params as any).target_height = (params as any).height;
}
// Build FlatBuffer config
let configBytes: Buffer | undefined = undefined;
let payloadWidth: number | undefined;
let payloadHeight: number | undefined;
let payloadSteps: number | undefined;
let payloadSeed: number | undefined;
let payloadSeedMode: string | undefined;
let payloadModel: string | undefined;
let payloadNumFrames: number = 1;
let payloadFps: number = 24;
try {
const allowedKeys = new Set(Object.keys(baseDefaults));
const filtered: Record<string, any> = {};
for (const [k, v] of Object.entries(params)) {
if (allowedKeys.has(k)) filtered[k] = v as any;
}
const modelId = (params as any).model as ModelId | undefined;
const {
source,
presetName,
params: overlayParams,
} = getEffectiveOverlay(modelId, overlayMode);
// Size must be controlled by tool/core (or defaults), not by overlays.
// Batch must be controlled by tool/core via variants parameter.
// Upscaler must be controlled by tool/core via _dt_needs_upscaler decision.
const overlayParamsNoSize = (() => {
if (!overlayParams) return overlayParams;
const o: any = { ...(overlayParams as any) };
delete o.width;
delete o.height;
delete o.batch_count;
delete o.batch_size;
delete o.batchCount;
delete o.batchSize;
delete o.upscaler;
delete o.upscaler_scale;
delete o.upscalerScale;
delete o.fps;
return o;
})();
// Capture overlay info for result metadata
overlaySource = source;
overlayPreset = presetName;
if (modelId === "auto") {
delete filtered.model;
}
if (overlayParams) {
logFile(
`Applying ${source} overlay for '${modelId ?? "auto"}' (${overlayMode === "img2img" ? "image2image" : "edit"}, gRPC)${
presetName ? ` preset=${presetName}` : ""
}`
);
delete filtered.model;
}
// Log overlay source to audit
audit.setOutput({
overlay_source: source,
...(presetName && { overlay_preset: presetName }),
});
const effective = {
...baseDefaults,
...(overlayParamsNoSize || {}),
...filtered,
seed: baseDefaults.seed,
} as Record<string, any>;
defaultsUsed =
overlayMode === "edit"
? "defaultParamsDrawThingsEdit"
: "defaultParamsDrawThingsImg2Img";
overlayLookupMode = overlayMode;
strengthUsed =
typeof effective.strength === "number" && Number.isFinite(effective.strength)
? effective.strength
: undefined;
stepsUsed =
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined;
samplerUsed =
typeof effective.sampler === "string" && String(effective.sampler).trim()
? String(effective.sampler)
: undefined;
guidanceScaleUsed =
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined;
shiftUsed =
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined;
resolutionDependentShiftUsed =
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined;
audit.setOutput({
overlay_lookup_mode: overlayMode,
defaults_used:
overlayMode === "edit"
? "defaultParamsDrawThingsEdit"
: "defaultParamsDrawThingsImg2Img",
strength_used:
typeof effective.strength === "number" && Number.isFinite(effective.strength)
? effective.strength
: undefined,
steps_used:
typeof effective.steps === "number" && Number.isFinite(effective.steps)
? effective.steps
: undefined,
sampler_used:
typeof effective.sampler === "string" && String(effective.sampler).trim()
? String(effective.sampler)
: undefined,
guidance_scale_used:
typeof effective.guidance_scale === "number" &&
Number.isFinite(effective.guidance_scale)
? effective.guidance_scale
: undefined,
shift_used:
typeof effective.shift === "number" && Number.isFinite(effective.shift)
? effective.shift
: undefined,
resolution_dependent_shift_used:
typeof effective.resolution_dependent_shift === "boolean"
? effective.resolution_dependent_shift
: undefined,
});
// Extract LoRAs for audit: user > overlay > defaults
try {
const userSpecifiedLoras = Object.prototype.hasOwnProperty.call(
filtered,
"loras"
);
const overlaySelected = !!overlayParams;
if (userSpecifiedLoras) {
const ls = Array.isArray(filtered.loras) ? filtered.loras : [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else if (overlaySelected) {
const ls = Array.isArray((overlayParams as any)?.loras)
? (overlayParams as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
} else {
// Fallback: extract LoRAs from effective params (includes defaults)
const ls = Array.isArray((effective as any)?.loras)
? (effective as any).loras
: [];
requiredLoraFiles = ls
.map((x: any) => x?.file)
.filter((x: any) => typeof x === "string" && x.trim())
.map((x: any) => String(x).trim());
}
} catch {}
const { buildDtGenerationConfiguration } = await import(
"./drawThingsConfigMapper.js"
);
const cfg = buildDtGenerationConfiguration(effective);
configBytes = cfg.bytes && cfg.bytes.length ? cfg.bytes : undefined;
try {
const view = (cfg as any)?.view || {};
payloadWidth = view.width;
payloadHeight = view.height;
payloadSteps = view.steps;
payloadSeed = view.seed;
payloadSeedMode = view.seed_mode;
payloadModel = String(effective.model || "");
payloadNumFrames = typeof effective.num_frames === "number" ? effective.num_frames : 1;
payloadFps = (typeof filtered.fps === "number" ? filtered.fps : undefined)
?? getDefaultFpsForModel(payloadModel)
?? (typeof effective.fps === "number" ? effective.fps : 24);
} catch {}
} catch {}
const sharedSecret = process.env.DRAWTHINGS_SHARED_SECRET;
// Build shuffle hints for moodboard images
// CRITICAL: Moodboard images must be referenced via hints with hintType="shuffle"
// See: moodboard_grpc_research.md for details
// Weight distribution: 1/n per image (matches Draw Things client behavior)
const moodboardWeight =
moodboardHashes.length > 0 ? 1.0 / moodboardHashes.length : 1.0;
const shuffleHints: Array<{ tensor: Buffer; weight: number }> =
moodboardHashes.map((hash) => ({
tensor: hash, // SHA256 hash (32 bytes) referencing tensor in contents[]
weight: moodboardWeight,
}));
const request: any = {
prompt: String(params.prompt || baseDefaults.prompt),
negativePrompt: String((params as any).negative_prompt || ""),
scaleFactor: 1,
keywords: [],
user: "generate-image-plugin",
device: "LAPTOP",
chunked: true,
...(sharedSecret ? { sharedSecret } : {}),
image: primaryHash, // Hash of first (canvas) image
contents: contentsTensors, // All reference tensors (Content-Addressable Storage)
// Moodboard images as shuffle hints
...(shuffleHints.length > 0
? {
hints: [
{
hintType: "shuffle", // MUST be "shuffle" for moodboard/reference images
tensors: shuffleHints,
},
],
}
: {}),
};
if (configBytes) request.configuration = configBytes;
logFile(
`[edit] request built: canvas=1, moodboard=${shuffleHints.length}, total_contents=${contentsTensors.length}`
);
const call = this.client[this.methodGenerate]?.bind(this.client);
if (typeof call !== "function")
throw new Error(`gRPC method not found: ${this.methodGenerate}`);
const startTime = Date.now();
const imagesBuffers: Buffer[] = [];
const audioBuffers: Buffer[] = [];
let chunkAccum: Buffer | null = null;
let previewBuf: Buffer | null = null;
let lastSeenStep = 0;
const handleData = (resp: any) => {
try {
if (resp?.currentSignpost?.sampling?.step) {
const currentStep = resp.currentSignpost.sampling.step;
lastSeenStep = currentStep;
console.info(
`gRPC progress: sampling step=${currentStep}`
);
logFile(
`edit progress: sampling step=${currentStep}`
);
// Invoke progress callback if provided
if (onProgress) {
try {
onProgress(
currentStep,
payloadSteps,
`Sampling step ${currentStep}${payloadSteps ? `/${payloadSteps}` : ""}`
);
} catch {}
}
} else if (resp?.currentSignpost != null && onProgress) {
const sp = resp.currentSignpost;
let label: string | undefined;
if (lastSeenStep === 0) {
if (sp.textEncoded != null) label = "Loading...";
else if (sp.imageEncoded != null) label = "Processing...";
} else {
label = "Finishing...";
}
if (label) {
try { onProgress(-1, payloadSteps, label); } catch {}
}
}
if (
resp?.generatedImages &&
Array.isArray(resp.generatedImages)
) {
const list = resp.generatedImages.map((b: any) =>
Buffer.isBuffer(b) ? b : Buffer.from(b)
);
const cs = resp?.chunkState;
const isMore = cs === 1 || cs === "MORE_CHUNKS";
const isLast = cs === 0 || cs === "LAST_CHUNK";
if (isMore) {
if (list.length > 0) {
chunkAccum = chunkAccum
? Buffer.concat([chunkAccum, list[0]])
: Buffer.from(list[0]);
}
} else if (isLast && list.length > 0) {
if (chunkAccum) {
const full = Buffer.concat([chunkAccum, list[0]]);
imagesBuffers.push(full);
chunkAccum = null;
} else {
for (const buf of list) imagesBuffers.push(buf);
}
} else {
for (const buf of list) imagesBuffers.push(buf);
}
}
if (resp?.previewImage) {
const b = resp.previewImage;
previewBuf = Buffer.isBuffer(b) ? b : Buffer.from(b);
}
if (resp?.generatedAudio && Array.isArray(resp.generatedAudio)) {
for (const a of resp.generatedAudio) {
const ab = Buffer.isBuffer(a) ? a : Buffer.from(a);
if (ab.length > 0) audioBuffers.push(ab);
}
}
} catch {}
};
await new Promise<void>((resolve, reject) => {
try {
const stream = call(request);
stream.on("data", handleData);
stream.on("error", (err: any) => reject(err));
stream.on("end", () => resolve());
} catch (e) {
reject(e);
}
});
const endTime = Date.now();
// Decode tensor to PNG
const resolveDecoderCmd = (): ExternalToolCmd | null => {
const envCmd = process.env.DT_DECODER_CMD;
const projectRoot2 = __projectRoot;
const candidates = buildTensorHelperCandidates(
projectRoot2,
"GRPCBin2PNG",
envCmd
);
for (const c of candidates) {
try {
const abs = path.isAbsolute(c) ? c : path.resolve(projectRoot2, c);
if (!fs.existsSync(abs)) continue;
const origin: ExternalToolCmd["origin"] =
c === envCmd ? "env" : "auto";
if (abs.endsWith(".js")) {
return {
cmd: process.execPath,
argsPrefix: [abs],
display: abs,
impl: "ts",
origin,
};
}
try {
const st = fs.statSync(abs);
if ((st.mode & 0o111) === 0) {
try {
fs.chmodSync(abs, 0o755);
} catch {}
}
} catch {}
return {
cmd: abs,
argsPrefix: [],
display: abs,
impl: "mac-bin",
origin,
};
} catch {}
}
return null;
};
const decoderCmd = resolveDecoderCmd();
logToolSelection("edit decode", decoderCmd);
let primaryBuf: Buffer | null = null;
if (decoderCmd && imagesBuffers.length > 0) {
const pngBuffers: Buffer[] = [];
for (let i = 0; i < imagesBuffers.length; i++) {
const rawBuf = imagesBuffers[i];
try {
const tmpDir = fs.mkdtempSync(
path.join(os.tmpdir(), `generate-image-plugin-dec-${i}-`)
);
__registerTmp(tmpDir);
const inPath = path.join(tmpDir, "input.bin");
const outPath = path.join(tmpDir, "output.png");
fs.writeFileSync(inPath, rawBuf);
const proc = spawnSync(
decoderCmd.cmd,
[...decoderCmd.argsPrefix, "--in", inPath, "--out", outPath],
{
encoding: "buffer",
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env },
}
);
if (
!proc.error &&
fs.existsSync(outPath) &&
fs.statSync(outPath).size > 0
) {
pngBuffers.push(fs.readFileSync(outPath));
}
} catch {}
}
if (pngBuffers.length > 0) {
primaryBuf = pngBuffers[0];
imagesBuffers.length = 0;
for (const b of pngBuffers) imagesBuffers.push(b);
}
}
if (!primaryBuf && previewBuf) {
primaryBuf = previewBuf;
}
if (!primaryBuf)
throw new Error("gRPC edit stream yielded no image data");
const dataUrl = `data:image/png;base64,${primaryBuf.toString("base64")}`;
logFile(
`edit completed: refs=${referenceBuffers.length}, images=${imagesBuffers.length}, bytes=${primaryBuf.length}`
);
const effW = Number(payloadWidth ?? (params as any).width ?? 1024);
const effH = Number(payloadHeight ?? (params as any).height ?? 1024);
const result: BackendGenerateResult = {
isError: false,
imageBuffer: primaryBuf,
imageData: dataUrl,
images: imagesBuffers.length
? imagesBuffers.map(
(b) => `data:image/png;base64,${b.toString("base64")}`
)
: undefined,
metadata: {
alt:
overlayMode === "img2img"
? `Image-to-image from prompt: ${request.prompt}`
: `Edit from prompt: ${request.prompt}`,
inference_time_ms: endTime - startTime,
model: payloadModel,
width: effW,
height: effH,
requested_dimensions: {
width: effW,
height: effH,
},
steps: Number.isFinite(Number(payloadSteps))
? Number(payloadSteps)
: undefined,
seed:
typeof payloadSeed === "number" && Number.isFinite(payloadSeed)
? payloadSeed
: undefined,
seed_mode:
typeof payloadSeedMode === "string" && payloadSeedMode.trim()
? payloadSeedMode
: undefined,
prompt_used: String(request.prompt || ""),
reference_count: referenceBuffers.length,
transport: "grpc",
stream_chunks: imagesBuffers.length,
num_frames: payloadNumFrames,
fps: payloadFps,
// Overlay source tracking for audit
overlay_source: overlaySource,
...(overlayPreset && { overlay_preset: overlayPreset }),
...(defaultsUsed && { defaults_used: defaultsUsed }),
...(overlayLookupMode && { overlay_lookup_mode: overlayLookupMode }),
i2i_profile: i2iProfileUsed,
...(typeof strengthUsed === "number" && { strength_used: strengthUsed }),
...(typeof stepsUsed === "number" && { steps_used: stepsUsed }),
...(samplerUsed && { sampler_used: samplerUsed }),
...(typeof guidanceScaleUsed === "number" && {
guidance_scale_used: guidanceScaleUsed,
}),
...(typeof shiftUsed === "number" && { shift_used: shiftUsed }),
...(typeof resolutionDependentShiftUsed === "boolean" && {
resolution_dependent_shift_used: resolutionDependentShiftUsed,
}),
...(requiredLoraFiles &&
requiredLoraFiles.length > 0 && { loras_used: requiredLoraFiles }),
},
audioBuffers: audioBuffers.length > 0 ? audioBuffers : undefined,
};
__cleanupTmp();
return result;
} catch (e) {
return this.toBackendError(e);
}
}
}
export default DrawThingsGrpcService;