vlm-tools

import { type PromptPreprocessor } from "@lmstudio/sdk";

export const imageAttachmentPromptPreprocessor: PromptPreprocessor = async (ctl, userMessage) => {
  const consumedImages = userMessage.consumeFiles(ctl.client, file => file.isImage());

  if (consumedImages.length === 0) {
    return userMessage;
  }

  const imageLines = new Array<string>();

  for (const [imageIndex, image] of consumedImages.entries()) {
    const imagePath = await image.getFilePath();

    imageLines.push(`Image ${imageIndex + 1} attached at absolute path: ${imagePath}`);
  }

  userMessage.appendText([
    "",
    "Attached image files were converted to local file paths for tool use.",
    ...imageLines,
    "When calling vlm-tools image tools, use the exact absolute path shown above.",
    "For bounding boxes, call the bounding_boxes tool with that path and an explicit mode.",
    "Use mode=\"qwen\" when the user asks for Qwen mode; otherwise use mode=\"gemma\".",
  ].join("\n"));

  return userMessage;
};