src / toolsProvider.ts
import { tool, Tool, ToolsProviderController } from "@lmstudio/sdk";
import { spawn } from "child_process";
import { mkdir, stat } from "fs/promises";
import {
basename,
extname,
isAbsolute,
join,
normalize,
relative,
resolve,
} from "path";
import { fileURLToPath } from "url";
import { z } from "zod";
import { configSchematics, DEFAULT_ANALYSIS_PROMPT } from "./config";
import { rememberGeneratedImage } from "./mediaHistory";
const OUTPUT_SUBDIR_PATTERN = /^[A-Za-z0-9._/-]+$/;
const IMAGE_INPUT_EXTENSIONS = new Set([
".avif",
".bmp",
".gif",
".jpeg",
".jpg",
".png",
".tif",
".tiff",
".webp",
]);
const PROCESS_OUTPUT_LIMIT = 4000;
const POSTER_TIMEOUT_MS = 30000;
const ANALYSIS_MAX_TOKENS_HARD_CAP = 2048;
type ProcessResult = {
code: number | null;
stdout: string;
stderr: string;
timedOut: boolean;
aborted: boolean;
launchError?: string;
};
type CommonGenerationOptions = {
prompt: string;
model?: string;
negativePrompt?: string;
image?: string;
width?: number;
height?: number;
steps?: number;
cfg?: number;
seed?: number;
strength?: number;
configJson?: string;
configFile?: string;
analyzeGeneratedMedia?: boolean;
analysisPrompt?: string;
};
type MediaAnalysisStatus = "disabled" | "pending" | "completed" | "unavailable";
type MediaAnalysisSource = "image" | "video_poster";
type MediaAnalysisResult = {
status: MediaAnalysisStatus;
source: MediaAnalysisSource;
prompt: string;
text?: string;
error?: string;
};
type MediaAnalysisRequest = {
enabled: boolean;
source: MediaAnalysisSource;
prompt: string;
maxTokens: number;
};
type ResolvedGenerationOptions = {
args: string[];
model: string;
advancedConfigSource?: "json" | "file";
};
type InputImageReference = {
relativePath: string;
absolutePath: string;
};
type VideoJobStatus = "running" | "completed" | "failed";
type VideoGenerationJob = {
id: string;
status: VideoJobStatus;
prompt: string;
model: string;
startedAt: string;
completedAt?: string;
outputPath: string;
absoluteOutputPath: string;
posterPath?: string;
absolutePosterPath?: string;
posterMarkdown?: string;
absolutePosterMarkdown?: string;
opened?: boolean;
openError?: string;
analysis: MediaAnalysisResult;
error?: string;
metadata: {
model: string;
videoFormat: string;
outputFormat: string;
advancedConfigSource?: "json" | "file";
inputImagePath?: string;
sizeBytes?: number;
};
inputImagePath?: string;
absoluteInputImagePath?: string;
};
const videoJobs = new Map<string, VideoGenerationJob>();
export async function toolsProvider(ctl: ToolsProviderController): Promise<Tool[]> {
const generateImageTool = tool({
name: "generate_image",
description:
"Generate an image with Draw Things CLI and return local Markdown that can be embedded in chat. Provide image to do image-to-image from an uploaded or previously generated image path.",
parameters: {
prompt: z.string().min(1).describe("Positive prompt for image generation."),
model: z
.string()
.optional()
.describe("Optional model override. Uses the plugin-configured model when omitted."),
negativePrompt: z
.string()
.optional()
.describe("Optional negative prompt override."),
width: z
.number()
.int()
.min(64)
.max(4096)
.optional()
.describe("Optional output width. Must be a multiple of 64."),
height: z
.number()
.int()
.min(64)
.max(4096)
.optional()
.describe("Optional output height. Must be a multiple of 64."),
steps: z.number().int().min(1).max(300).optional().describe("Optional sampling steps."),
cfg: z.number().min(0).max(50).optional().describe("Optional CFG guidance scale."),
seed: z.number().int().min(0).max(2147483647).optional().describe("Optional random seed."),
strength: z
.number()
.min(0)
.max(1)
.optional()
.describe("Optional denoising strength for img2img."),
image: z
.string()
.optional()
.describe("Optional input image path for image-to-image. Use an exact path listed in the chat's Draw Things image inputs block, or another image path inside the LM Studio working directory."),
analyzeGeneratedMedia: z
.boolean()
.optional()
.describe("Optional override for automatic analysis of the generated image."),
analysisPrompt: z
.string()
.optional()
.describe("Optional prompt override for analyzing the generated image."),
configJson: z
.string()
.optional()
.describe("Optional JSGenerationConfiguration JSON override. Mutually exclusive with configFile."),
configFile: z
.string()
.optional()
.describe("Optional JSON override file inside the LM Studio working directory. Mutually exclusive with configJson."),
},
implementation: async (params, context) => {
const { status, warn, signal } = context;
status("Preparing Draw Things image generation...");
try {
const workingDirectory = ctl.getWorkingDirectory();
const config = ctl.getPluginConfig(configSchematics) as { get: (key: any) => unknown };
const outputRoot = await resolveOutputDirectory(
workingDirectory,
getConfigString(config, "outputSubdirectory"),
);
const outputFileName = `draw-things-${Date.now()}-${randomSuffix()}.png`;
const outputPath = join(outputRoot.absolutePath, outputFileName);
const outputRelativePath = toWorkingDirectoryRelativePath(workingDirectory, outputPath);
const resolved = await buildGenerationArgs(workingDirectory, config, params, outputPath);
const inputImage = await resolveOptionalInputImageReference(workingDirectory, params.image);
if (inputImage) {
resolved.args.push("--image", inputImage.absolutePath);
}
status(`Generating image with ${resolved.model}...`);
const cliPath = getRequiredConfigString(config, "drawThingsCliPath", "Draw Things CLI Path");
const result = await runProcess(cliPath, resolved.args, workingDirectory, getTimeoutMs(config), signal);
const failure = processFailureMessage("Draw Things image generation", result, resolved.args);
if (failure) return failure;
const outputStats = await stat(outputPath).catch(() => null);
if (!outputStats?.isFile()) {
return buildMissingOutputMessage("image", outputRelativePath, result);
}
const analysis = await analyzeGeneratedMediaIfRequested({
ctl,
config,
params,
source: "image",
absolutePath: outputPath,
generationPrompt: params.prompt,
status,
warn,
});
status("Image generated successfully.");
rememberGeneratedImage({
path: outputRelativePath,
absolutePath: outputPath,
});
const analysisText = getCompletedAnalysisText(analysis);
return {
type: "image",
count: 1,
images: [``],
imageMarkdown: ``,
outputPath: outputRelativePath,
...(inputImage ? { inputImagePath: inputImage.relativePath } : {}),
...(inputImage ? { absoluteInputImagePath: inputImage.absolutePath } : {}),
analysis,
...(analysisText ? { analysisText } : {}),
metadata: {
model: resolved.model,
sizeBytes: outputStats.size,
advancedConfigSource: resolved.advancedConfigSource,
...(inputImage ? { inputImagePath: inputImage.relativePath } : {}),
},
hint: buildImageResultHint(analysis),
};
} catch (error: any) {
warn(`Draw Things image generation failed: ${error?.message || String(error)}`);
return `Error: ${error?.message || String(error)}`;
}
},
});
const generateVideoTool = tool({
name: "generate_video",
description:
"Generate a video with Draw Things CLI. Provide image to do image-to-video from an uploaded or previously generated image path. Background mode returns a job id immediately to avoid LM Studio tool-call timeouts; completed videos open automatically in the desktop default app.",
parameters: {
prompt: z.string().min(1).describe("Positive prompt for video generation."),
model: z
.string()
.optional()
.describe("Optional model override. Uses the plugin-configured model when omitted."),
negativePrompt: z
.string()
.optional()
.describe("Optional negative prompt override."),
width: z
.number()
.int()
.min(64)
.max(4096)
.optional()
.describe("Optional output width. Must be a multiple of 64."),
height: z
.number()
.int()
.min(64)
.max(4096)
.optional()
.describe("Optional output height. Must be a multiple of 64."),
steps: z.number().int().min(1).max(300).optional().describe("Optional sampling steps."),
cfg: z.number().min(0).max(50).optional().describe("Optional CFG guidance scale."),
seed: z.number().int().min(0).max(2147483647).optional().describe("Optional random seed."),
image: z
.string()
.optional()
.describe("Optional input image path for image-to-video. Use an exact path listed in the chat's Draw Things image inputs block, or another image path inside the LM Studio working directory."),
frames: z
.number()
.int()
.min(1)
.max(512)
.optional()
.describe("Optional frame count for video-capable models."),
videoFormat: z
.enum(["h264", "hevc", "prores4444", "prores422hq"])
.optional()
.describe("Optional video export format override."),
outputFormat: z.enum(["mp4", "mov"]).optional().describe("Optional output container. Defaults to mp4."),
analyzeGeneratedMedia: z
.boolean()
.optional()
.describe("Optional override for automatic analysis of the generated video poster frame."),
analysisPrompt: z
.string()
.optional()
.describe("Optional prompt override for analyzing the generated video poster frame."),
configJson: z
.string()
.optional()
.describe("Optional JSGenerationConfiguration JSON override. Mutually exclusive with configFile."),
configFile: z
.string()
.optional()
.describe("Optional JSON override file inside the LM Studio working directory. Mutually exclusive with configJson."),
},
implementation: async (params, context) => {
const { status, warn, signal } = context;
status("Preparing Draw Things video generation...");
try {
const workingDirectory = ctl.getWorkingDirectory();
const config = ctl.getPluginConfig(configSchematics) as { get: (key: any) => unknown };
const outputRoot = await resolveOutputDirectory(
workingDirectory,
getConfigString(config, "outputSubdirectory"),
);
const videoFormat = getEffectiveVideoFormat(config, params.videoFormat);
const extension = getVideoOutputExtension(videoFormat, params.outputFormat);
const baseName = `draw-things-${Date.now()}-${randomSuffix()}`;
const outputPath = join(outputRoot.absolutePath, `${baseName}.${extension}`);
const outputRelativePath = toWorkingDirectoryRelativePath(workingDirectory, outputPath);
const resolved = await buildGenerationArgs(workingDirectory, config, params, outputPath);
const inputImage = await resolveOptionalInputImageReference(workingDirectory, params.image);
if (inputImage) {
resolved.args.push("--image", inputImage.absolutePath);
}
const frames = getEffectiveNumber(params.frames, getConfigNumber(config, "frames"), -1);
if (frames !== undefined) {
validatePositiveInteger("frames", frames);
resolved.args.push("--frames", String(frames));
}
if (videoFormat !== "auto") {
resolved.args.push("--video-format", videoFormat);
}
const cliPath = getRequiredConfigString(config, "drawThingsCliPath", "Draw Things CLI Path");
const videoExecutionMode = getVideoExecutionMode(config);
const expectedPosterAbsolutePath = join(outputRoot.absolutePath, `${baseName}-poster.png`);
const analysisRequest = buildMediaAnalysisRequest(config, params, "video_poster");
if (videoExecutionMode === "background") {
const job = createVideoJob({
prompt: params.prompt,
model: resolved.model,
outputPath: outputRelativePath,
absoluteOutputPath: outputPath,
videoFormat,
outputFormat: extension,
advancedConfigSource: resolved.advancedConfigSource,
inputImage,
analysisRequest,
});
videoJobs.set(job.id, job);
startBackgroundVideoJob({
ctl,
job,
command: cliPath,
args: resolved.args,
cwd: workingDirectory,
timeoutMs: getTimeoutMs(config),
ffmpegPath: cleanString(getConfigString(config, "ffmpegPath")),
posterAbsolutePath: expectedPosterAbsolutePath,
analysisRequest,
generationPrompt: params.prompt,
});
status(`Started background video generation with ${resolved.model}.`);
return {
type: "video_job",
jobId: job.id,
status: job.status,
outputPath: job.outputPath,
absoluteOutputPath: job.absoluteOutputPath,
...(job.inputImagePath ? { inputImagePath: job.inputImagePath } : {}),
...(job.absoluteInputImagePath ? { absoluteInputImagePath: job.absoluteInputImagePath } : {}),
expectedPosterPath: toWorkingDirectoryRelativePath(workingDirectory, expectedPosterAbsolutePath),
expectedAbsolutePosterPath: expectedPosterAbsolutePath,
analysis: job.analysis,
metadata: job.metadata,
hint: buildVideoJobStartedHint(job.analysis),
};
}
status(`Generating video with ${resolved.model}...`);
const result = await runProcess(cliPath, resolved.args, workingDirectory, getTimeoutMs(config), signal);
const failure = processFailureMessage("Draw Things video generation", result, resolved.args);
if (failure) return failure;
const outputStats = await stat(outputPath).catch(() => null);
if (!outputStats?.isFile()) {
return buildMissingOutputMessage("video", outputRelativePath, result);
}
let posterMarkdown: string | undefined;
let absolutePosterMarkdown: string | undefined;
let posterPath: string | undefined;
let absolutePosterPath: string | undefined;
let posterError: string | undefined;
const ffmpegPath = cleanString(getConfigString(config, "ffmpegPath"));
if (ffmpegPath) {
const posterResult = await createVideoPoster(
ffmpegPath,
outputPath,
expectedPosterAbsolutePath,
workingDirectory,
signal,
);
if (posterResult.ok) {
posterPath = toWorkingDirectoryRelativePath(workingDirectory, expectedPosterAbsolutePath);
absolutePosterPath = expectedPosterAbsolutePath;
posterMarkdown = ``;
absolutePosterMarkdown = ``;
} else {
posterError = posterResult.message;
warn(`Could not create video poster: ${posterResult.message}`);
}
} else {
posterError = "FFmpeg Path is not configured, so no video poster frame was created for analysis.";
}
status("Video generated successfully.");
const openResult = await openLocalFile(outputPath, workingDirectory, signal);
const openFailure = processFailureMessage("Opening generated video", openResult, [outputPath]);
if (openFailure) {
warn(openFailure);
}
const analysis = await analyzeGeneratedMediaIfRequested({
ctl,
config,
params,
source: "video_poster",
absolutePath: absolutePosterPath,
missingSourceError: posterError || "Video poster frame is unavailable for analysis.",
generationPrompt: params.prompt,
status,
warn,
});
const analysisText = getCompletedAnalysisText(analysis);
return {
type: "video",
count: 1,
opened: !openFailure,
openError: openFailure || undefined,
posterMarkdown,
absolutePosterMarkdown,
outputPath: outputRelativePath,
absoluteOutputPath: outputPath,
...(inputImage ? { inputImagePath: inputImage.relativePath } : {}),
...(inputImage ? { absoluteInputImagePath: inputImage.absolutePath } : {}),
posterPath,
absolutePosterPath,
analysis,
...(analysisText ? { analysisText } : {}),
metadata: {
model: resolved.model,
sizeBytes: outputStats.size,
videoFormat,
outputFormat: extension,
advancedConfigSource: resolved.advancedConfigSource,
...(inputImage ? { inputImagePath: inputImage.relativePath } : {}),
},
hint: buildVideoResultHint(analysis),
};
} catch (error: any) {
warn(`Draw Things video generation failed: ${error?.message || String(error)}`);
return `Error: ${error?.message || String(error)}`;
}
},
});
const openGeneratedMediaTool = tool({
name: "open_generated_media",
description:
"Open a generated Draw Things image or video in the desktop default app. Use this when LM Studio does not make a local media link clickable.",
parameters: {
path: z
.string()
.min(1)
.describe("Generated media path from generate_image/generate_video. Accepts relative paths, absolute paths inside the working directory, or file:// URLs."),
},
implementation: async ({ path }, { status, warn, signal }) => {
status("Opening generated media...");
try {
const workingDirectory = ctl.getWorkingDirectory();
const absolutePath = await resolveExistingFileInsideWorkingDirectory(
workingDirectory,
normalizeMediaPathInput(path),
"path",
);
const result = await openLocalFile(absolutePath, workingDirectory, signal);
const failure = processFailureMessage("Opening generated media", result, [absolutePath]);
if (failure) {
warn(failure);
return failure;
}
return {
opened: true,
path: toWorkingDirectoryRelativePath(workingDirectory, absolutePath),
absolutePath,
};
} catch (error: any) {
warn(`Could not open generated media: ${error?.message || String(error)}`);
return `Error: ${error?.message || String(error)}`;
}
},
});
const checkVideoGenerationTool = tool({
name: "check_video_generation",
description:
"Check a background Draw Things video generation job. Use after generate_video returns a video_job result.",
parameters: {
jobId: z
.string()
.optional()
.describe("Job id returned by generate_video when video generation runs in background mode."),
path: z
.string()
.optional()
.describe("Optional generated video path to inspect if the job id is not available."),
},
implementation: async ({ jobId, path }, { status, warn }) => {
status("Checking video generation...");
try {
const workingDirectory = ctl.getWorkingDirectory();
const config = ctl.getPluginConfig(configSchematics) as { get: (key: any) => unknown };
const cleanJobId = cleanString(jobId);
if (cleanJobId) {
const job = videoJobs.get(cleanJobId);
if (!job) {
return `Error: no background video job found for id '${cleanJobId}'. The plugin may have been restarted.`;
}
return buildVideoJobPayload(job);
}
const cleanPath = cleanString(path);
if (cleanPath) {
const absolutePath = await resolveExistingFileInsideWorkingDirectory(
workingDirectory,
normalizeMediaPathInput(cleanPath),
"path",
);
const metadata = await stat(absolutePath);
const relativePath = toWorkingDirectoryRelativePath(workingDirectory, absolutePath);
const posterAbsolutePath = withPosterSuffix(absolutePath);
const posterStats = await stat(posterAbsolutePath).catch(() => null);
const posterPath = posterStats?.isFile()
? toWorkingDirectoryRelativePath(workingDirectory, posterAbsolutePath)
: undefined;
const analysis = await analyzeGeneratedMediaIfRequested({
ctl,
config,
params: {},
source: "video_poster",
absolutePath: posterStats?.isFile() ? posterAbsolutePath : undefined,
missingSourceError: "No video poster frame exists for this video path.",
generationPrompt: "Unknown; this video was checked by path.",
status,
warn,
});
const analysisText = getCompletedAnalysisText(analysis);
return {
type: "video",
status: "completed",
outputPath: relativePath,
absoluteOutputPath: absolutePath,
posterPath,
absolutePosterPath: posterStats?.isFile() ? posterAbsolutePath : undefined,
posterMarkdown: posterPath ? `` : undefined,
absolutePosterMarkdown: posterStats?.isFile()
? ``
: undefined,
analysis,
...(analysisText ? { analysisText } : {}),
metadata: {
sizeBytes: metadata.size,
},
hint: buildVideoResultHint(analysis),
};
}
const recentJobs = [...videoJobs.values()]
.sort((a, b) => b.startedAt.localeCompare(a.startedAt))
.slice(0, 10)
.map(({ id, status, startedAt, completedAt, outputPath, absoluteOutputPath, error }) => ({
id,
status,
startedAt,
completedAt,
outputPath,
absoluteOutputPath,
error,
}));
return {
type: "video_jobs",
count: recentJobs.length,
jobs: recentJobs,
hint: "Call check_video_generation with a jobId to get the current result for a background video job.",
};
} catch (error: any) {
warn(`Could not check video generation: ${error?.message || String(error)}`);
return `Error: ${error?.message || String(error)}`;
}
},
});
return [generateImageTool, generateVideoTool, checkVideoGenerationTool, openGeneratedMediaTool];
}
async function analyzeGeneratedMediaIfRequested({
ctl,
config,
params,
source,
absolutePath,
missingSourceError,
generationPrompt,
status,
warn,
}: {
ctl: ToolsProviderController;
config: { get: (key: any) => unknown };
params: Pick<CommonGenerationOptions, "analyzeGeneratedMedia" | "analysisPrompt">;
source: MediaAnalysisSource;
absolutePath?: string;
missingSourceError?: string;
generationPrompt: string;
status: (text: string) => void;
warn: (text: string) => void;
}): Promise<MediaAnalysisResult> {
const request = buildMediaAnalysisRequest(config, params, source);
return await analyzeGeneratedMedia({
ctl,
request,
absolutePath,
missingSourceError,
generationPrompt,
status,
warn,
});
}
async function analyzeGeneratedMedia({
ctl,
request,
absolutePath,
missingSourceError,
generationPrompt,
status,
warn,
}: {
ctl: ToolsProviderController;
request: MediaAnalysisRequest;
absolutePath?: string;
missingSourceError?: string;
generationPrompt: string;
status?: (text: string) => void;
warn?: (text: string) => void;
}): Promise<MediaAnalysisResult> {
if (!request.enabled) {
return {
status: "disabled",
source: request.source,
prompt: request.prompt,
};
}
if (!absolutePath) {
return {
status: "unavailable",
source: request.source,
prompt: request.prompt,
error: missingSourceError || `${formatAnalysisSource(request.source)} is unavailable for analysis.`,
};
}
try {
status?.(`Analyzing generated ${formatAnalysisSource(request.source)}...`);
const model = await ctl.client.llm.model();
if (!model.vision) {
return {
status: "unavailable",
source: request.source,
prompt: request.prompt,
error: "The currently loaded model does not support vision. Load a vision model to analyze generated media.",
};
}
const fileHandle = await ctl.client.files.prepareImage(absolutePath);
const result = await model.respond(
[
{
role: "user",
content: buildAnalysisPrompt(request, generationPrompt),
images: [fileHandle],
},
],
{
maxTokens: request.maxTokens,
},
);
const text = normalizeAnalysisText(result.content);
if (!text) {
return {
status: "unavailable",
source: request.source,
prompt: request.prompt,
error: "The vision model returned an empty analysis.",
};
}
return {
status: "completed",
source: request.source,
prompt: request.prompt,
text,
};
} catch (error: any) {
const message = error?.message || String(error);
warn?.(`Generated media analysis unavailable: ${message}`);
return {
status: "unavailable",
source: request.source,
prompt: request.prompt,
error: message,
};
}
}
function buildMediaAnalysisRequest(
config: { get: (key: any) => unknown },
params: Pick<CommonGenerationOptions, "analyzeGeneratedMedia" | "analysisPrompt">,
source: MediaAnalysisSource,
): MediaAnalysisRequest {
const enabled =
typeof params.analyzeGeneratedMedia === "boolean"
? params.analyzeGeneratedMedia
: getConfigBoolean(config, "analyzeGeneratedMedia", true);
const prompt =
getEffectiveString(params.analysisPrompt, getConfigString(config, "analysisPrompt")) ||
DEFAULT_ANALYSIS_PROMPT;
return {
enabled,
source,
prompt,
maxTokens: getAnalysisMaxTokens(config),
};
}
function buildInitialAnalysisResult(request: MediaAnalysisRequest): MediaAnalysisResult {
return {
status: request.enabled ? "pending" : "disabled",
source: request.source,
prompt: request.prompt,
};
}
function buildUnavailableAnalysisResult(request: MediaAnalysisRequest, error: string): MediaAnalysisResult {
if (!request.enabled) {
return {
status: "disabled",
source: request.source,
prompt: request.prompt,
};
}
return {
status: "unavailable",
source: request.source,
prompt: request.prompt,
error,
};
}
function buildAnalysisPrompt(request: MediaAnalysisRequest, generationPrompt: string): string {
return [
"You are a vision assistant analyzing media generated by Draw Things.",
"Give a concise final answer only. Do not provide hidden reasoning or step-by-step chain-of-thought.",
"If uncertain, state uncertainty briefly.",
"",
`Generated media type: ${formatAnalysisSource(request.source)}.`,
`Generation prompt: ${generationPrompt.trim()}`,
"",
`Analysis request: ${request.prompt}`,
].join("\n");
}
function normalizeAnalysisText(content: unknown): string {
return typeof content === "string" ? content.trim() : String(content ?? "").trim();
}
function getCompletedAnalysisText(analysis: MediaAnalysisResult): string | undefined {
return analysis.status === "completed" ? cleanString(analysis.text) || undefined : undefined;
}
function buildImageResultHint(analysis: MediaAnalysisResult): string {
return [
"The generated image is already stored locally and renderable. Embed imageMarkdown, or images[0], in the assistant response with standard Markdown. The user will not see it unless you embed it.",
buildAnalysisHint(analysis),
].join(" ");
}
function buildVideoJobStartedHint(analysis: MediaAnalysisResult): string {
return [
"Video generation is running in the background to avoid LM Studio tool-call timeouts. The video will open automatically when complete. Do not provide clickable video links. Tell the user the job has started and include the jobId. To check completion, call check_video_generation with this jobId.",
buildAnalysisHint(analysis),
].join(" ");
}
function buildVideoResultHint(analysis: MediaAnalysisResult): string {
return [
"The generated video has been opened automatically in the desktop default app when possible. LM Studio escapes raw HTML video tags and does not make local video links clickable, so do not print HTML, videoMarkdown, file URLs, or clickable video links. Include posterMarkdown when present plus absoluteOutputPath.",
buildAnalysisHint(analysis),
].join(" ");
}
function buildAnalysisHint(analysis: MediaAnalysisResult): string {
if (analysis.status === "completed") {
return "Generated media analysis is available in analysisText and analysis.text; use it as visual context when responding.";
}
if (analysis.status === "pending") {
return "Generated media analysis is pending until the generated frame is available.";
}
if (analysis.status === "disabled") {
return "Generated media analysis was disabled for this tool call.";
}
return `Generated media analysis is unavailable${analysis.error ? `: ${analysis.error}` : "."}`;
}
function formatAnalysisSource(source: MediaAnalysisSource): string {
return source === "video_poster" ? "video poster frame" : "image";
}
async function buildGenerationArgs(
workingDirectory: string,
config: { get: (key: any) => unknown },
params: CommonGenerationOptions,
outputPath: string,
): Promise<ResolvedGenerationOptions> {
const model = resolveModel(config, params.model);
const args = [
"generate",
"--model",
model,
"--prompt",
params.prompt.trim(),
];
const advancedConfig = await resolveAdvancedConfig(workingDirectory, config, params);
if (advancedConfig.source === "json") {
args.push("--config-json", advancedConfig.value);
} else if (advancedConfig.source === "file") {
args.push("--config-file", advancedConfig.value);
}
const negativePrompt = getEffectiveString(
params.negativePrompt,
getConfigString(config, "negativePrompt"),
);
if (negativePrompt !== undefined) {
args.push("--negative-prompt", negativePrompt);
}
const width = getEffectiveNumber(params.width, getConfigNumber(config, "width"), -1);
const height = getEffectiveNumber(params.height, getConfigNumber(config, "height"), -1);
if (width !== undefined) {
validateDimension("width", width);
args.push("--width", String(width));
}
if (height !== undefined) {
validateDimension("height", height);
args.push("--height", String(height));
}
const steps = getEffectiveNumber(params.steps, getConfigNumber(config, "steps"), -1);
if (steps !== undefined) {
validatePositiveInteger("steps", steps);
args.push("--steps", String(steps));
}
const cfg = getEffectiveNumber(params.cfg, getConfigNumber(config, "cfg"), -1);
if (cfg !== undefined) {
validateRange("cfg", cfg, 0, 50);
args.push("--cfg", String(cfg));
}
const seed = getEffectiveNumber(params.seed, getConfigNumber(config, "seed"), -1);
if (seed !== undefined) {
validatePositiveInteger("seed", seed, true);
args.push("--seed", String(seed));
}
const strength = getEffectiveNumber(params.strength, getConfigNumber(config, "strength"), -1);
if (strength !== undefined) {
validateRange("strength", strength, 0, 1);
args.push("--strength", String(strength));
}
args.push(
"--output",
outputPath,
"--offline",
"--no-download-missing",
"--disable-preview",
);
return {
args,
model,
advancedConfigSource: advancedConfig.source,
};
}
function createVideoJob({
prompt,
model,
outputPath,
absoluteOutputPath,
videoFormat,
outputFormat,
advancedConfigSource,
inputImage,
analysisRequest,
}: {
prompt: string;
model: string;
outputPath: string;
absoluteOutputPath: string;
videoFormat: string;
outputFormat: string;
advancedConfigSource?: "json" | "file";
inputImage?: InputImageReference;
analysisRequest: MediaAnalysisRequest;
}): VideoGenerationJob {
return {
id: `video-${Date.now()}-${randomSuffix()}`,
status: "running",
prompt,
model,
startedAt: new Date().toISOString(),
outputPath,
absoluteOutputPath,
inputImagePath: inputImage?.relativePath,
absoluteInputImagePath: inputImage?.absolutePath,
analysis: buildInitialAnalysisResult(analysisRequest),
metadata: {
model,
videoFormat,
outputFormat,
advancedConfigSource,
...(inputImage ? { inputImagePath: inputImage.relativePath } : {}),
},
};
}
function startBackgroundVideoJob({
ctl,
job,
command,
args,
cwd,
timeoutMs,
ffmpegPath,
posterAbsolutePath,
analysisRequest,
generationPrompt,
}: {
ctl: ToolsProviderController;
job: VideoGenerationJob;
command: string;
args: string[];
cwd: string;
timeoutMs: number;
ffmpegPath: string;
posterAbsolutePath: string;
analysisRequest: MediaAnalysisRequest;
generationPrompt: string;
}): void {
void (async () => {
const result = await runProcess(command, args, cwd, timeoutMs);
const failure = processFailureMessage("Draw Things background video generation", result, args);
if (failure) {
job.status = "failed";
job.error = failure;
job.analysis = buildUnavailableAnalysisResult(
analysisRequest,
"Video generation failed before a poster frame could be analyzed.",
);
job.completedAt = new Date().toISOString();
return;
}
const outputStats = await stat(job.absoluteOutputPath).catch(() => null);
if (!outputStats?.isFile()) {
job.status = "failed";
job.error = buildMissingOutputMessage("video", job.outputPath, result);
job.analysis = buildUnavailableAnalysisResult(
analysisRequest,
"Video generation did not create an output file, so no poster frame could be analyzed.",
);
job.completedAt = new Date().toISOString();
return;
}
job.metadata.sizeBytes = outputStats.size;
let posterError: string | undefined;
if (ffmpegPath) {
const posterResult = await createVideoPoster(
ffmpegPath,
job.absoluteOutputPath,
posterAbsolutePath,
cwd,
);
if (posterResult.ok) {
job.absolutePosterPath = posterAbsolutePath;
job.posterPath = toWorkingDirectoryRelativePath(cwd, posterAbsolutePath);
job.posterMarkdown = ``;
job.absolutePosterMarkdown = ``;
} else {
posterError = posterResult.message;
}
} else {
posterError = "FFmpeg Path is not configured, so no video poster frame was created for analysis.";
}
const openResult = await openLocalFile(job.absoluteOutputPath, cwd);
const openFailure = processFailureMessage("Opening generated video", openResult, [job.absoluteOutputPath]);
job.opened = !openFailure;
job.openError = openFailure || undefined;
job.analysis = await analyzeGeneratedMedia({
ctl,
request: analysisRequest,
absolutePath: job.absolutePosterPath,
missingSourceError: posterError || "Video poster frame is unavailable for analysis.",
generationPrompt,
});
job.status = "completed";
job.completedAt = new Date().toISOString();
})();
}
function buildVideoJobPayload(job: VideoGenerationJob): Record<string, unknown> {
if (job.status === "running") {
return {
type: "video_job",
jobId: job.id,
status: job.status,
startedAt: job.startedAt,
outputPath: job.outputPath,
absoluteOutputPath: job.absoluteOutputPath,
...(job.inputImagePath ? { inputImagePath: job.inputImagePath } : {}),
...(job.absoluteInputImagePath ? { absoluteInputImagePath: job.absoluteInputImagePath } : {}),
analysis: job.analysis,
metadata: job.metadata,
hint: buildVideoJobStartedHint(job.analysis),
};
}
if (job.status === "failed") {
return {
type: "video_job",
jobId: job.id,
status: job.status,
startedAt: job.startedAt,
completedAt: job.completedAt,
outputPath: job.outputPath,
absoluteOutputPath: job.absoluteOutputPath,
...(job.inputImagePath ? { inputImagePath: job.inputImagePath } : {}),
...(job.absoluteInputImagePath ? { absoluteInputImagePath: job.absoluteInputImagePath } : {}),
analysis: job.analysis,
error: job.error,
};
}
const analysisText = getCompletedAnalysisText(job.analysis);
return {
type: "video",
jobId: job.id,
status: job.status,
count: 1,
opened: job.opened,
openError: job.openError,
posterMarkdown: job.posterMarkdown,
absolutePosterMarkdown: job.absolutePosterMarkdown,
outputPath: job.outputPath,
absoluteOutputPath: job.absoluteOutputPath,
...(job.inputImagePath ? { inputImagePath: job.inputImagePath } : {}),
...(job.absoluteInputImagePath ? { absoluteInputImagePath: job.absoluteInputImagePath } : {}),
posterPath: job.posterPath,
absolutePosterPath: job.absolutePosterPath,
analysis: job.analysis,
...(analysisText ? { analysisText } : {}),
metadata: job.metadata,
hint: buildVideoResultHint(job.analysis),
};
}
async function resolveOptionalInputImageReference(
workingDirectory: string,
inputPath?: string,
): Promise<InputImageReference | undefined> {
const cleanedPath = cleanString(inputPath);
if (!cleanedPath) return undefined;
const absolutePath = await resolveExistingFileInsideWorkingDirectory(
workingDirectory,
normalizeMediaPathInput(cleanedPath),
"image",
);
if (!IMAGE_INPUT_EXTENSIONS.has(extname(absolutePath).toLowerCase())) {
throw new Error(`image must use one of these extensions: ${[...IMAGE_INPUT_EXTENSIONS].join(", ")}`);
}
return {
absolutePath,
relativePath: toWorkingDirectoryRelativePath(workingDirectory, absolutePath),
};
}
function resolveModel(config: { get: (key: any) => unknown }, overrideModel?: string): string {
const toolModel = cleanString(overrideModel);
if (toolModel) return toolModel;
const customModel = cleanString(getConfigString(config, "customModel"));
if (customModel) return customModel;
const configuredModel = cleanString(getConfigString(config, "model"));
if (!configuredModel) {
throw new Error("No Draw Things model configured.");
}
return configuredModel;
}
async function resolveAdvancedConfig(
workingDirectory: string,
config: { get: (key: any) => unknown },
params: Pick<CommonGenerationOptions, "configJson" | "configFile">,
): Promise<{ source?: "json" | "file"; value: string }> {
const configJson = getEffectiveString(params.configJson, getConfigString(config, "configJson"));
const configFile = getEffectiveString(params.configFile, getConfigString(config, "configFile"));
if (configJson && configFile) {
throw new Error("configJson and configFile are mutually exclusive. Clear one before generating.");
}
if (configJson) {
try {
JSON.parse(configJson);
} catch (error: any) {
throw new Error(`configJson is not valid JSON: ${error?.message || String(error)}`);
}
return { source: "json", value: configJson };
}
if (configFile) {
const resolvedConfigFile = await resolveExistingFileInsideWorkingDirectory(
workingDirectory,
configFile,
"configFile",
);
if (extname(resolvedConfigFile).toLowerCase() !== ".json") {
throw new Error("configFile must point to a .json file.");
}
return { source: "file", value: resolvedConfigFile };
}
return { value: "" };
}
async function resolveOutputDirectory(
workingDirectory: string,
configuredSubdirectory: string,
): Promise<{ absolutePath: string; relativePath: string }> {
const rawSubdirectory = cleanString(configuredSubdirectory) || "draw-things";
const normalizedSubdirectory = normalize(rawSubdirectory).replace(/\\/g, "/").replace(/^\.\/+/, "");
if (isAbsolute(normalizedSubdirectory)) {
throw new Error("Output Subdirectory must be relative to the LM Studio working directory.");
}
if (!OUTPUT_SUBDIR_PATTERN.test(normalizedSubdirectory)) {
throw new Error("Output Subdirectory may only contain letters, numbers, dots, underscores, hyphens, and slashes.");
}
const absolutePath = resolve(workingDirectory, normalizedSubdirectory);
if (!isPathInside(workingDirectory, absolutePath)) {
throw new Error("Output Subdirectory must stay inside the LM Studio working directory.");
}
await mkdir(absolutePath, { recursive: true });
return {
absolutePath,
relativePath: toWorkingDirectoryRelativePath(workingDirectory, absolutePath),
};
}
async function resolveExistingFileInsideWorkingDirectory(
workingDirectory: string,
inputPath: string,
label: string,
): Promise<string> {
const cleanedPath = cleanString(inputPath);
if (!cleanedPath) {
throw new Error(`${label} path is empty.`);
}
const absolutePath = isAbsolute(cleanedPath)
? resolve(cleanedPath)
: resolve(workingDirectory, cleanedPath);
if (!isPathInside(workingDirectory, absolutePath)) {
throw new Error(`${label} must be inside the LM Studio working directory.`);
}
const metadata = await stat(absolutePath).catch(() => null);
if (!metadata?.isFile()) {
throw new Error(`${label} file not found: ${cleanedPath}`);
}
return absolutePath;
}
function normalizeMediaPathInput(inputPath: string): string {
const cleanedPath = cleanString(inputPath);
if (!cleanedPath.toLowerCase().startsWith("file://")) {
return cleanedPath;
}
try {
return fileURLToPath(cleanedPath);
} catch (error: any) {
throw new Error(`Invalid file URL: ${error?.message || String(error)}`);
}
}
function isPathInside(root: string, target: string): boolean {
const resolvedRoot = resolve(root);
const resolvedTarget = resolve(target);
const relativePath = relative(resolvedRoot, resolvedTarget);
return relativePath === "" || (!relativePath.startsWith("..") && !isAbsolute(relativePath));
}
function toWorkingDirectoryRelativePath(workingDirectory: string, absolutePath: string): string {
const relativePath = relative(workingDirectory, absolutePath).replace(/\\/g, "/");
return relativePath === "" ? basename(absolutePath) : relativePath;
}
function getEffectiveString(overrideValue: string | undefined, configuredValue: string): string | undefined {
const cleanedOverride = cleanString(overrideValue);
if (cleanedOverride) return cleanedOverride;
const cleanedConfigured = cleanString(configuredValue);
return cleanedConfigured || undefined;
}
function getEffectiveNumber(
overrideValue: number | undefined,
configuredValue: number,
autoValue: number,
): number | undefined {
if (typeof overrideValue === "number") return overrideValue;
if (configuredValue !== autoValue) return configuredValue;
return undefined;
}
function getEffectiveVideoFormat(
config: { get: (key: any) => unknown },
overrideValue?: "h264" | "hevc" | "prores4444" | "prores422hq",
): "auto" | "h264" | "hevc" | "prores4444" | "prores422hq" {
if (overrideValue) return overrideValue;
const configuredValue = getConfigString(config, "videoFormat");
if (
configuredValue === "h264" ||
configuredValue === "hevc" ||
configuredValue === "prores4444" ||
configuredValue === "prores422hq"
) {
return configuredValue;
}
return "auto";
}
function getVideoExecutionMode(config: { get: (key: any) => unknown }): "background" | "foreground" {
const configuredValue = getConfigString(config, "videoExecutionMode");
return configuredValue === "foreground" ? "foreground" : "background";
}
function getVideoOutputExtension(
videoFormat: "auto" | "h264" | "hevc" | "prores4444" | "prores422hq",
requestedFormat?: "mp4" | "mov",
): "mp4" | "mov" {
if (videoFormat === "prores4444" || videoFormat === "prores422hq") {
return "mov";
}
return requestedFormat || "mp4";
}
function getConfigString(config: { get: (key: any) => unknown }, key: string): string {
const value = config.get(key);
return typeof value === "string" ? value.trim() : String(value ?? "").trim();
}
function getRequiredConfigString(
config: { get: (key: any) => unknown },
key: string,
displayName: string,
): string {
const value = getConfigString(config, key);
if (!value) {
throw new Error(`${displayName} is required.`);
}
return value;
}
function getConfigNumber(config: { get: (key: any) => unknown }, key: string): number {
const value = config.get(key);
return typeof value === "number" && Number.isFinite(value) ? value : Number(value);
}
function getConfigBoolean(config: { get: (key: any) => unknown }, key: string, fallback: boolean): boolean {
const value = config.get(key);
if (typeof value === "boolean") return value;
if (typeof value === "string") {
const normalized = value.trim().toLowerCase();
if (normalized === "true") return true;
if (normalized === "false") return false;
}
return fallback;
}
function getTimeoutMs(config: { get: (key: any) => unknown }): number {
const timeoutMs = getConfigNumber(config, "timeoutMs");
if (!Number.isFinite(timeoutMs) || timeoutMs < 1000) {
return 7200000;
}
return timeoutMs;
}
function getAnalysisMaxTokens(config: { get: (key: any) => unknown }): number {
const configuredMaxTokens = getConfigNumber(config, "analysisMaxTokens");
if (!Number.isFinite(configuredMaxTokens)) {
return 512;
}
return Math.max(1, Math.min(ANALYSIS_MAX_TOKENS_HARD_CAP, Math.trunc(configuredMaxTokens)));
}
function cleanString(value?: string): string {
return typeof value === "string" ? value.trim() : "";
}
function validateDimension(name: string, value: number): void {
validatePositiveInteger(name, value);
if (value % 64 !== 0) {
throw new Error(`${name} must be a multiple of 64.`);
}
}
function validatePositiveInteger(name: string, value: number, allowZero = false): void {
if (!Number.isInteger(value) || value < (allowZero ? 0 : 1)) {
throw new Error(`${name} must be ${allowZero ? "0 or a positive" : "a positive"} integer.`);
}
}
function validateRange(name: string, value: number, min: number, max: number): void {
if (!Number.isFinite(value) || value < min || value > max) {
throw new Error(`${name} must be between ${min} and ${max}.`);
}
}
function randomSuffix(): string {
return Math.random().toString(36).slice(2, 8);
}
async function runProcess(
command: string,
args: string[],
cwd: string,
timeoutMs: number,
signal?: AbortSignal,
): Promise<ProcessResult> {
return await new Promise((resolvePromise) => {
let stdout = "";
let stderr = "";
let settled = false;
let timedOut = false;
let aborted = false;
let timeout: ReturnType<typeof setTimeout> | undefined;
let abortHandler: (() => void) | undefined;
const child = spawn(command, args, {
cwd,
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
shell: false,
});
const settle = (result: ProcessResult) => {
if (settled) return;
settled = true;
if (timeout) clearTimeout(timeout);
if (abortHandler) signal?.removeEventListener("abort", abortHandler);
resolvePromise(result);
};
const killChild = () => {
if (!child.killed) {
child.kill("SIGTERM");
}
setTimeout(() => {
if (!settled) {
child.kill("SIGKILL");
}
}, 2000).unref();
};
timeout = setTimeout(() => {
timedOut = true;
killChild();
}, timeoutMs);
abortHandler = () => {
aborted = true;
killChild();
};
if (signal?.aborted) {
abortHandler();
} else {
signal?.addEventListener("abort", abortHandler);
}
child.stdout?.on("data", (chunk: Buffer) => {
stdout = appendLimited(stdout, chunk.toString("utf8"), PROCESS_OUTPUT_LIMIT);
});
child.stderr?.on("data", (chunk: Buffer) => {
stderr = appendLimited(stderr, chunk.toString("utf8"), PROCESS_OUTPUT_LIMIT);
});
child.on("error", (error) => {
settle({
code: null,
stdout: stdout.trim(),
stderr: stderr.trim(),
timedOut,
aborted,
launchError: error.message,
});
});
child.on("close", (code) => {
settle({
code,
stdout: stdout.trim(),
stderr: stderr.trim(),
timedOut,
aborted,
});
});
});
}
function appendLimited(current: string, next: string, limit: number): string {
const combined = current + next;
if (combined.length <= limit) return combined;
return combined.slice(0, limit) + "\n...[Output Truncated]";
}
function processFailureMessage(label: string, result: ProcessResult, args: string[]): string | null {
if (result.aborted) return `${label} aborted by user.`;
if (result.timedOut) return `${label} timed out.`;
if (result.launchError) return `Error launching ${label}: ${result.launchError}`;
if (result.code === 0) return null;
return [
`${label} failed with exit code ${result.code ?? "unknown"}.`,
result.stderr ? `STDERR:\n${result.stderr}` : "",
result.stdout ? `STDOUT:\n${result.stdout}` : "",
`ARGS:\n${formatArgsForDebug(args)}`,
]
.filter(Boolean)
.join("\n\n");
}
function buildMissingOutputMessage(kind: "image" | "video", outputPath: string, result: ProcessResult): string {
return [
`Draw Things reported success but no ${kind} file was created at ${outputPath}.`,
result.stderr ? `STDERR:\n${result.stderr}` : "",
result.stdout ? `STDOUT:\n${result.stdout}` : "",
]
.filter(Boolean)
.join("\n\n");
}
function formatArgsForDebug(args: string[]): string {
return args.map((arg) => JSON.stringify(arg)).join(" ");
}
function withPosterSuffix(videoPath: string): string {
const extension = extname(videoPath);
return `${videoPath.slice(0, videoPath.length - extension.length)}-poster.png`;
}
async function createVideoPoster(
ffmpegPath: string,
videoPath: string,
posterPath: string,
cwd: string,
signal?: AbortSignal,
): Promise<{ ok: true } | { ok: false; message: string }> {
const result = await runProcess(
ffmpegPath,
["-y", "-i", videoPath, "-frames:v", "1", posterPath],
cwd,
POSTER_TIMEOUT_MS,
signal,
);
const failure = processFailureMessage("FFmpeg poster extraction", result, [
"-y",
"-i",
videoPath,
"-frames:v",
"1",
posterPath,
]);
if (failure) return { ok: false, message: failure };
const posterStats = await stat(posterPath).catch(() => null);
if (!posterStats?.isFile()) {
return { ok: false, message: "ffmpeg completed but did not create a poster image." };
}
return { ok: true };
}
async function openLocalFile(
absolutePath: string,
cwd: string,
signal?: AbortSignal,
): Promise<ProcessResult> {
if (process.platform === "darwin") {
return await runProcess("open", [absolutePath], cwd, 10000, signal);
}
if (process.platform === "win32") {
return await runProcess("cmd.exe", ["/c", "start", "", absolutePath], cwd, 10000, signal);
}
return await runProcess("xdg-open", [absolutePath], cwd, 10000, signal);
}