src / tools / inspectImages.ts
src / tools / inspectImages.ts
import { tool, ToolsProviderController } from "@lmstudio/sdk";
import * as fs from "fs";
import * as path from "path";
import { z } from "zod";
import { Logger } from "../utils/Logger";
import { ImageDownloader } from "../image_search/utils/ImageDownloader";
import { CacheManager, generateUrlHash } from "../image_search/utils/CacheManager";
const defaultImageDownloader = new ImageDownloader();
const defaultCacheManager = new CacheManager();
/**
* Creates the inspect_images tool instance.
* Sends image(s) (local file paths or remote HTTP/HTTPS URLs) and an optional instruction
* to the currently loaded vision-capable model, returning a string array containing relative Markdown image tags and the vision analysis text.
*/
export function createInspectImagesTool(
ctl?: ToolsProviderController,
downloader: ImageDownloader = defaultImageDownloader,
cacheManager: CacheManager = defaultCacheManager
) {
return tool({
name: "Inspect Image",
description:
"Processes image(s) from the chat history or external image URLs, encodes them using the vision model, and returns the textual analysis along with the image(s). \nDisplay the Image(s) in the response as the exact Markdown format ``.",
parameters: {
images: z
.array(z.string())
.describe("Array of local image file paths, filenames, or remote HTTP(S) image URLs to inspect"),
instruction: z
.string()
.optional()
.describe(
"Optional instruction or question about the images (default: 'Describe these images in detail.')"
),
},
implementation: async ({ images, instruction }) => {
// 1. Controller & Parameter Validation
if (!ctl) {
throw new Error("ToolsProviderController is required for Inspect Image tool.");
}
if (!images || !Array.isArray(images) || images.length === 0) {
return "Error: The 'images' parameter must contain at least one local image file path or remote image URL.";
}
const workingDirectory = ctl.getWorkingDirectory() || process.cwd();
// 2. Resolve & Verify Image Files / Remote URLs
const fileHandles = [];
const displayPaths: string[] = [];
const failedImageNotices: string[] = [];
for (const imgPath of images) {
if (!imgPath || typeof imgPath !== "string" || !imgPath.trim()) {
continue;
}
const trimmedPath = imgPath.trim();
let resolvedPath: string;
if (trimmedPath.startsWith("http://") || trimmedPath.startsWith("https://")) {
// Remote Image URL Handling via ImageDownloader & CacheManager
const urlHash = generateUrlHash(trimmedPath);
const cachedPath = await cacheManager.findCachedFile(urlHash, workingDirectory);
if (cachedPath) {
resolvedPath = cachedPath;
Logger.debug("[InspectImages]", `Cache hit for remote URL: ${trimmedPath} -> ${cachedPath}`);
} else {
Logger.info("[InspectImages]", `Downloading remote image URL: ${trimmedPath}`);
const downloadRes = await downloader.downloadSingleImage(
trimmedPath,
`img_${urlHash}`,
workingDirectory
);
if (!downloadRes) {
failedImageNotices.push(`Could not download or validate remote image URL '${trimmedPath}'.`);
continue;
}
resolvedPath = downloadRes.filepath;
}
} else {
// Local File Path Handling (Preserved 1-to-1)
resolvedPath = path.isAbsolute(trimmedPath)
? trimmedPath
: path.resolve(workingDirectory, trimmedUrlOrPath(trimmedPath));
if (!fs.existsSync(resolvedPath)) {
failedImageNotices.push(`Image file not found at path '${trimmedPath}' (resolved to '${resolvedPath}').`);
continue;
}
}
try {
const handle = await ctl.client.files.prepareImage(resolvedPath);
fileHandles.push(handle);
// Format relative forward-slash path for LM Studio chat Markdown image rendering
let relPath =
workingDirectory && resolvedPath.startsWith(workingDirectory)
? path.relative(workingDirectory, resolvedPath)
: path.basename(resolvedPath);
relPath = relPath.replace(/\\/g, "/");
displayPaths.push(relPath);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
Logger.error("[InspectImages]", `Failed to prepare image '${resolvedPath}': ${msg}`);
failedImageNotices.push(`Error preparing image '${trimmedPath}': ${msg}`);
}
}
if (fileHandles.length === 0) {
return `Error: No valid image files or URLs could be processed for analysis. Details:\n- ${failedImageNotices.join("\n- ")}`;
}
// 3. Obtain Active LM Studio Vision Model
let model;
try {
model = await ctl.client.llm.model();
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
return `Error connecting to loaded model in LM Studio: ${msg}`;
}
if (!model) {
return "Error: No LLM is currently loaded in LM Studio. Please load a vision-capable model.";
}
if (!model.vision) {
const modelName = model.displayName || model.identifier || "active model";
return `Error: The currently loaded model (${modelName}) does not support vision capabilities. Please load a vision-capable model (e.g. Qwen-VL, LLaVA, Florence) in LM Studio.`;
}
// 4. Send Images & Prompt to Vision Model
const promptText = instruction && instruction.trim()
? instruction.trim()
: "Describe these images in detail.";
try {
Logger.info(
"[InspectImages]",
`Sending ${fileHandles.length} image(s) to vision model '${model.displayName}' with prompt: "${promptText}"`
);
const prediction = await model.respond([
{
role: "user",
content: promptText,
images: fileHandles,
},
]);
const analysisText =
prediction.content && prediction.content.trim()
? prediction.content.trim()
: "No analysis output was produced by the vision model.";
// Format array return matching search_images contract (string[])
// Returning string[] enables LM Studio's Native Tool UI parser to render each image artifact
const imageMarkdownItems = displayPaths.map((p, idx) => ``);
// Verbatim preservation of critical formatting hint
const formattingHint =
"\n\n**(CRITICAL) Formatting Hint:** Place the image reference(s) provided above immediately before, adjacent to the relevant explanation in your final response. DO NOT OMIT THE IMAGE REFERENCE(S).";
let diagnosticNotice = "";
if (failedImageNotices.length > 0) {
diagnosticNotice = `\n\n*(Note: The following requested image(s) could not be loaded: ${failedImageNotices.join("; ")})*`;
}
return [
...imageMarkdownItems,
`**Analysis:**\n${analysisText}${diagnosticNotice}${formattingHint}`,
];
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
Logger.error("[InspectImages]", `Vision model analysis failed: ${msg}`);
return `Error during vision model analysis: ${msg}`;
}
},
});
}
function trimmedUrlOrPath(p: string): string {
// Strip relative markdown prefix if present (e.g. "./img.jpg" -> "img.jpg")
return p.replace(/^(\.\/|\.\\)/, "");
}
/**
* Default singleton instance placeholder.
*/
export const inspectImagesTool = createInspectImagesTool();
import { tool, ToolsProviderController } from "@lmstudio/sdk";
import * as fs from "fs";
import * as path from "path";
import { z } from "zod";
import { Logger } from "../utils/Logger";
import { ImageDownloader } from "../image_search/utils/ImageDownloader";
import { CacheManager, generateUrlHash } from "../image_search/utils/CacheManager";
const defaultImageDownloader = new ImageDownloader();
const defaultCacheManager = new CacheManager();
/**
* Creates the inspect_images tool instance.
* Sends image(s) (local file paths or remote HTTP/HTTPS URLs) and an optional instruction
* to the currently loaded vision-capable model, returning a string array containing relative Markdown image tags and the vision analysis text.
*/
export function createInspectImagesTool(
ctl?: ToolsProviderController,
downloader: ImageDownloader = defaultImageDownloader,
cacheManager: CacheManager = defaultCacheManager
) {
return tool({
name: "Inspect Image",
description:
"Processes image(s) from the chat history or external image URLs, encodes them using the vision model, and returns the textual analysis along with the image(s). \nDisplay the Image(s) in the response as the exact Markdown format ``.",
parameters: {
images: z
.array(z.string())
.describe("Array of local image file paths, filenames, or remote HTTP(S) image URLs to inspect"),
instruction: z
.string()
.optional()
.describe(
"Optional instruction or question about the images (default: 'Describe these images in detail.')"
),
},
implementation: async ({ images, instruction }) => {
// 1. Controller & Parameter Validation
if (!ctl) {
throw new Error("ToolsProviderController is required for Inspect Image tool.");
}
if (!images || !Array.isArray(images) || images.length === 0) {
return "Error: The 'images' parameter must contain at least one local image file path or remote image URL.";
}
const workingDirectory = ctl.getWorkingDirectory() || process.cwd();
// 2. Resolve & Verify Image Files / Remote URLs
const fileHandles = [];
const displayPaths: string[] = [];
const failedImageNotices: string[] = [];
for (const imgPath of images) {
if (!imgPath || typeof imgPath !== "string" || !imgPath.trim()) {
continue;
}
const trimmedPath = imgPath.trim();
let resolvedPath: string;
if (trimmedPath.startsWith("http://") || trimmedPath.startsWith("https://")) {
// Remote Image URL Handling via ImageDownloader & CacheManager
const urlHash = generateUrlHash(trimmedPath);
const cachedPath = await cacheManager.findCachedFile(urlHash, workingDirectory);
if (cachedPath) {
resolvedPath = cachedPath;
Logger.debug("[InspectImages]", `Cache hit for remote URL: ${trimmedPath} -> ${cachedPath}`);
} else {
Logger.info("[InspectImages]", `Downloading remote image URL: ${trimmedPath}`);
const downloadRes = await downloader.downloadSingleImage(
trimmedPath,
`img_${urlHash}`,
workingDirectory
);
if (!downloadRes) {
failedImageNotices.push(`Could not download or validate remote image URL '${trimmedPath}'.`);
continue;
}
resolvedPath = downloadRes.filepath;
}
} else {
// Local File Path Handling (Preserved 1-to-1)
resolvedPath = path.isAbsolute(trimmedPath)
? trimmedPath
: path.resolve(workingDirectory, trimmedUrlOrPath(trimmedPath));
if (!fs.existsSync(resolvedPath)) {
failedImageNotices.push(`Image file not found at path '${trimmedPath}' (resolved to '${resolvedPath}').`);
continue;
}
}
try {
const handle = await ctl.client.files.prepareImage(resolvedPath);
fileHandles.push(handle);
// Format relative forward-slash path for LM Studio chat Markdown image rendering
let relPath =
workingDirectory && resolvedPath.startsWith(workingDirectory)
? path.relative(workingDirectory, resolvedPath)
: path.basename(resolvedPath);
relPath = relPath.replace(/\\/g, "/");
displayPaths.push(relPath);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
Logger.error("[InspectImages]", `Failed to prepare image '${resolvedPath}': ${msg}`);
failedImageNotices.push(`Error preparing image '${trimmedPath}': ${msg}`);
}
}
if (fileHandles.length === 0) {
return `Error: No valid image files or URLs could be processed for analysis. Details:\n- ${failedImageNotices.join("\n- ")}`;
}
// 3. Obtain Active LM Studio Vision Model
let model;
try {
model = await ctl.client.llm.model();
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
return `Error connecting to loaded model in LM Studio: ${msg}`;
}
if (!model) {
return "Error: No LLM is currently loaded in LM Studio. Please load a vision-capable model.";
}
if (!model.vision) {
const modelName = model.displayName || model.identifier || "active model";
return `Error: The currently loaded model (${modelName}) does not support vision capabilities. Please load a vision-capable model (e.g. Qwen-VL, LLaVA, Florence) in LM Studio.`;
}
// 4. Send Images & Prompt to Vision Model
const promptText = instruction && instruction.trim()
? instruction.trim()
: "Describe these images in detail.";
try {
Logger.info(
"[InspectImages]",
`Sending ${fileHandles.length} image(s) to vision model '${model.displayName}' with prompt: "${promptText}"`
);
const prediction = await model.respond([
{
role: "user",
content: promptText,
images: fileHandles,
},
]);
const analysisText =
prediction.content && prediction.content.trim()
? prediction.content.trim()
: "No analysis output was produced by the vision model.";
// Format array return matching search_images contract (string[])
// Returning string[] enables LM Studio's Native Tool UI parser to render each image artifact
const imageMarkdownItems = displayPaths.map((p, idx) => ``);
// Verbatim preservation of critical formatting hint
const formattingHint =
"\n\n**(CRITICAL) Formatting Hint:** Place the image reference(s) provided above immediately before, adjacent to the relevant explanation in your final response. DO NOT OMIT THE IMAGE REFERENCE(S).";
let diagnosticNotice = "";
if (failedImageNotices.length > 0) {
diagnosticNotice = `\n\n*(Note: The following requested image(s) could not be loaded: ${failedImageNotices.join("; ")})*`;
}
return [
...imageMarkdownItems,
`**Analysis:**\n${analysisText}${diagnosticNotice}${formattingHint}`,
];
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
Logger.error("[InspectImages]", `Vision model analysis failed: ${msg}`);
return `Error during vision model analysis: ${msg}`;
}
},
});
}
function trimmedUrlOrPath(p: string): string {
// Strip relative markdown prefix if present (e.g. "./img.jpg" -> "img.jpg")
return p.replace(/^(\.\/|\.\\)/, "");
}
/**
* Default singleton instance placeholder.
*/
export const inspectImagesTool = createInspectImagesTool();