Forked from vadimfedenko/analyze-images
Forked from vadimfedenko/analyze-images
src / promptPreprocessor.ts
src / promptPreprocessor.ts
import type { PromptPreprocessor } from "@lmstudio/sdk";
import { realpath } from "node:fs/promises";
import { configSchematics } from "./config";
import { httpUrl, readLimited, selectPageImages } from "./pageImages";
import { candidateBytes, fileDigest, hashBytes, historyDigests, recentCandidates, IMAGE_LIMIT } from "./attachmentIdentity";
import { compatibleImage } from "./compatibleImage";
const trigger = /^@inject-url(?:\s+|$)/;
const extensions: Record<string, string> = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif" };
export const preprocess: PromptPreprocessor = async (ctl, original) => {
if (!original.isUserMessage()) return original;
const config = ctl.getPluginConfig(configSchematics);
const text = original.getText();
const explicit = trigger.test(text);
const inMessage = text.match(/https?:\/\/[^\s<>"\u3000]+/i)?.[0].replace(/[)\]、。.,!?]+$/, "");
const page = explicit ? inMessage ?? config.get("pageUrl").trim() : config.get("autoAttach") ? inMessage : undefined;
const maxImages = config.get("maxImages");
const startImage = config.get("startImage");
if (!Number.isInteger(maxImages) || maxImages < 1 || maxImages > 4 || !Number.isInteger(startImage) || startImage < 1 || startImage > 64) throw new Error("Invalid image attachment limits.");
if (explicit && !page) throw new Error("@inject-url requires a URL or Default page URL.");
ctl.guardAbort();
const result = original.asMutableCopy();
const budget = { remaining: 32 * 1024 * 1024 };
const current = new Set<string>();
const identifiers = new Set<string>();
let removed = 0;
let unreadableManual = false;
await result.consumeFilesAsync(ctl.client, async file => {
if (!file.isImage()) return false;
const digest = await fileDigest(file, budget);
const duplicate = identifiers.has(file.identifier) || (digest !== undefined && current.has(digest));
identifiers.add(file.identifier);
if (digest) current.add(digest); else unreadableManual = true;
if (duplicate) removed++;
return duplicate;
});
let history;
try { history = await ctl.pullHistory(); }
catch (error) { ctl.debug("Attachment history unavailable; automatic selection deferred", String(error)); }
const candidates = config.get("carryAnalyzed") && history ? recentCandidates(history) : [];
if (!page && !candidates.length && !removed) return original;
const seen = history ? await historyDigests(history, ctl.client, budget) : new Set<string>();
const pending = candidates.filter(item => !current.has(item.sha256) && !seen.has(item.sha256));
if (!page && !pending.length && !removed) return original;
const status = ctl.createStatus({status: "loading", text: "Preparing image attachments..."});
if (unreadableManual) {
status.setState({status: "error", text: "Manual images retained. Automatic attachment deferred because image comparison was unavailable."});
return removed ? result : original;
}
const signal = AbortSignal.any([ctl.abortSignal, AbortSignal.timeout(45_000)]);
const notes: string[] = [];
let count = 0;
let totalBytes = 0;
let failures = 0;
let skipped = 0;
const manualCount = result.getFiles(ctl.client).filter(file => file.isImage()).length;
const slots = Math.max(0, maxImages - manualCount);
const before = result.getFiles(ctl.client).length;
async function attach(bytes: Buffer, extension: string, source: string, respectHistory: boolean, identity = hashBytes(bytes)): Promise<void> {
if (current.has(identity) || (respectHistory && seen.has(identity))) { skipped++; return; }
if (count >= slots) return;
if (bytes.length > IMAGE_LIMIT || totalBytes + bytes.length > 16 * 1024 * 1024) throw new Error("Automatic image size limit reached.");
signal.throwIfAborted(); ctl.guardAbort();
const file = await compatibleImage(ctl.client, `aia-${identity}.${extension}`, bytes);
if (totalBytes + file.sizeBytes > 16 * 1024 * 1024) throw new Error("Converted images exceed automatic image size limit.");
signal.throwIfAborted(); ctl.guardAbort();
result.appendFile(file);
current.add(identity); totalBytes += file.sizeBytes; count++;
notes.push(`Attachment ${before + count}: ${source}`);
}
// Priority: current manual images, pending Analyze targets, then page images.
if (pending.length && slots > 0) {
try {
const root = ctl.getWorkingDirectory();
const rootHash = hashBytes(Buffer.from(await realpath(root)));
for (const item of pending) {
if (count >= slots) break;
if (item.rootHash !== rootHash || current.has(item.sha256) || seen.has(item.sha256)) continue;
try {
const input = await candidateBytes(root, item.relativePath);
if (hashBytes(input.bytes) !== item.sha256) throw new Error("Analyze target changed since analysis.");
let bytes = input.bytes;
let extension = input.extension.slice(1);
if (!["png", "jpg", "jpeg", "webp"].includes(extension)) {
const { default: sharp } = await import("sharp");
bytes = await sharp(bytes).png().toBuffer(); extension = "png";
}
await attach(bytes, extension, `previous Analyze target: ${item.relativePath}`, true, item.sha256);
} catch (error) {
if (ctl.abortSignal.aborted) throw error;
failures++; ctl.debug("Analyze image remains pending", item.relativePath, String(error));
}
}
} catch (error) {
if (ctl.abortSignal.aborted) throw error;
failures++; ctl.debug("Analyze attachment deferred", String(error));
}
}
if (page && count < slots && (history || explicit)) {
try {
const pageUrl = httpUrl(page);
const response = await fetch(pageUrl, {signal});
const html = (await readLimited(response, 3 * 1024 * 1024)).toString("utf8");
const finalUrl = response.url || pageUrl;
const selected = selectPageImages(html, finalUrl).slice(startImage - 1, startImage - 1 + maxImages);
if (!selected.length) throw new Error("No page images found at the selected position.");
for (const image of selected) {
if (count >= slots) break;
const downloaded = await fetch(image.url, {signal});
const mime = downloaded.headers.get("content-type")?.split(";")[0].trim().toLowerCase();
if (!downloaded.ok || !mime || !extensions[mime]) {
await downloaded.body?.cancel(); throw new Error(`Image fetch failed: ${downloaded.status}, ${mime}`);
}
const bytes = await readLimited(downloaded, IMAGE_LIMIT);
await attach(bytes, extensions[mime], `${image.source}; ${downloaded.url || image.url}`, !explicit);
}
ctl.debug("[analyze-images-advanced] Page image selection", {pageUrl: finalUrl, selected});
} catch (error) {
if (ctl.abortSignal.aborted || explicit) {
status.setState({status: ctl.abortSignal.aborted ? "canceled" : "error", text: "Automatic attachment failed; manual attachment is available."});
throw error;
}
failures++; ctl.debug("Page attachment failed; manual attachment is available", String(error));
}
}
if (page && !history && !explicit) failures++;
ctl.guardAbort();
if (explicit) result.replaceText(text.replace(trigger, "").trim() || "Describe the attached images.");
if (notes.length) result.appendText(`\n\n[Automatically attached images]\n${notes.join("\n")}`);
status.setState({status: failures ? "error" : "done", text: failures
? `Attached ${count} images. Some automatic images could not be attached; manual attachment is available.`
: `Attached ${count} images; skipped ${skipped + removed} duplicates.`});
ctl.debug("[analyze-images-advanced] Attachment summary", {count, removed, skipped, failures});
return count || removed || explicit ? result : original;
};
import type { PromptPreprocessor } from "@lmstudio/sdk";
import { realpath } from "node:fs/promises";
import { configSchematics } from "./config";
import { httpUrl, readLimited, selectPageImages } from "./pageImages";
import { candidateBytes, fileDigest, hashBytes, historyDigests, recentCandidates, IMAGE_LIMIT } from "./attachmentIdentity";
import { compatibleImage } from "./compatibleImage";
const trigger = /^@inject-url(?:\s+|$)/;
const extensions: Record<string, string> = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif" };
export const preprocess: PromptPreprocessor = async (ctl, original) => {
if (!original.isUserMessage()) return original;
const config = ctl.getPluginConfig(configSchematics);
const text = original.getText();
const explicit = trigger.test(text);
const inMessage = text.match(/https?:\/\/[^\s<>"\u3000]+/i)?.[0].replace(/[)\]、。.,!?]+$/, "");
const page = explicit ? inMessage ?? config.get("pageUrl").trim() : config.get("autoAttach") ? inMessage : undefined;
const maxImages = config.get("maxImages");
const startImage = config.get("startImage");
if (!Number.isInteger(maxImages) || maxImages < 1 || maxImages > 4 || !Number.isInteger(startImage) || startImage < 1 || startImage > 64) throw new Error("Invalid image attachment limits.");
if (explicit && !page) throw new Error("@inject-url requires a URL or Default page URL.");
ctl.guardAbort();
const result = original.asMutableCopy();
const budget = { remaining: 32 * 1024 * 1024 };
const current = new Set<string>();
const identifiers = new Set<string>();
let removed = 0;
let unreadableManual = false;
await result.consumeFilesAsync(ctl.client, async file => {
if (!file.isImage()) return false;
const digest = await fileDigest(file, budget);
const duplicate = identifiers.has(file.identifier) || (digest !== undefined && current.has(digest));
identifiers.add(file.identifier);
if (digest) current.add(digest); else unreadableManual = true;
if (duplicate) removed++;
return duplicate;
});
let history;
try { history = await ctl.pullHistory(); }
catch (error) { ctl.debug("Attachment history unavailable; automatic selection deferred", String(error)); }
const candidates = config.get("carryAnalyzed") && history ? recentCandidates(history) : [];
if (!page && !candidates.length && !removed) return original;
const seen = history ? await historyDigests(history, ctl.client, budget) : new Set<string>();
const pending = candidates.filter(item => !current.has(item.sha256) && !seen.has(item.sha256));
if (!page && !pending.length && !removed) return original;
const status = ctl.createStatus({status: "loading", text: "Preparing image attachments..."});
if (unreadableManual) {
status.setState({status: "error", text: "Manual images retained. Automatic attachment deferred because image comparison was unavailable."});
return removed ? result : original;
}
const signal = AbortSignal.any([ctl.abortSignal, AbortSignal.timeout(45_000)]);
const notes: string[] = [];
let count = 0;
let totalBytes = 0;
let failures = 0;
let skipped = 0;
const manualCount = result.getFiles(ctl.client).filter(file => file.isImage()).length;
const slots = Math.max(0, maxImages - manualCount);
const before = result.getFiles(ctl.client).length;
async function attach(bytes: Buffer, extension: string, source: string, respectHistory: boolean, identity = hashBytes(bytes)): Promise<void> {
if (current.has(identity) || (respectHistory && seen.has(identity))) { skipped++; return; }
if (count >= slots) return;
if (bytes.length > IMAGE_LIMIT || totalBytes + bytes.length > 16 * 1024 * 1024) throw new Error("Automatic image size limit reached.");
signal.throwIfAborted(); ctl.guardAbort();
const file = await compatibleImage(ctl.client, `aia-${identity}.${extension}`, bytes);
if (totalBytes + file.sizeBytes > 16 * 1024 * 1024) throw new Error("Converted images exceed automatic image size limit.");
signal.throwIfAborted(); ctl.guardAbort();
result.appendFile(file);
current.add(identity); totalBytes += file.sizeBytes; count++;
notes.push(`Attachment ${before + count}: ${source}`);
}
// Priority: current manual images, pending Analyze targets, then page images.
if (pending.length && slots > 0) {
try {
const root = ctl.getWorkingDirectory();
const rootHash = hashBytes(Buffer.from(await realpath(root)));
for (const item of pending) {
if (count >= slots) break;
if (item.rootHash !== rootHash || current.has(item.sha256) || seen.has(item.sha256)) continue;
try {
const input = await candidateBytes(root, item.relativePath);
if (hashBytes(input.bytes) !== item.sha256) throw new Error("Analyze target changed since analysis.");
let bytes = input.bytes;
let extension = input.extension.slice(1);
if (!["png", "jpg", "jpeg", "webp"].includes(extension)) {
const { default: sharp } = await import("sharp");
bytes = await sharp(bytes).png().toBuffer(); extension = "png";
}
await attach(bytes, extension, `previous Analyze target: ${item.relativePath}`, true, item.sha256);
} catch (error) {
if (ctl.abortSignal.aborted) throw error;
failures++; ctl.debug("Analyze image remains pending", item.relativePath, String(error));
}
}
} catch (error) {
if (ctl.abortSignal.aborted) throw error;
failures++; ctl.debug("Analyze attachment deferred", String(error));
}
}
if (page && count < slots && (history || explicit)) {
try {
const pageUrl = httpUrl(page);
const response = await fetch(pageUrl, {signal});
const html = (await readLimited(response, 3 * 1024 * 1024)).toString("utf8");
const finalUrl = response.url || pageUrl;
const selected = selectPageImages(html, finalUrl).slice(startImage - 1, startImage - 1 + maxImages);
if (!selected.length) throw new Error("No page images found at the selected position.");
for (const image of selected) {
if (count >= slots) break;
const downloaded = await fetch(image.url, {signal});
const mime = downloaded.headers.get("content-type")?.split(";")[0].trim().toLowerCase();
if (!downloaded.ok || !mime || !extensions[mime]) {
await downloaded.body?.cancel(); throw new Error(`Image fetch failed: ${downloaded.status}, ${mime}`);
}
const bytes = await readLimited(downloaded, IMAGE_LIMIT);
await attach(bytes, extensions[mime], `${image.source}; ${downloaded.url || image.url}`, !explicit);
}
ctl.debug("[analyze-images-advanced] Page image selection", {pageUrl: finalUrl, selected});
} catch (error) {
if (ctl.abortSignal.aborted || explicit) {
status.setState({status: ctl.abortSignal.aborted ? "canceled" : "error", text: "Automatic attachment failed; manual attachment is available."});
throw error;
}
failures++; ctl.debug("Page attachment failed; manual attachment is available", String(error));
}
}
if (page && !history && !explicit) failures++;
ctl.guardAbort();
if (explicit) result.replaceText(text.replace(trigger, "").trim() || "Describe the attached images.");
if (notes.length) result.appendText(`\n\n[Automatically attached images]\n${notes.join("\n")}`);
status.setState({status: failures ? "error" : "done", text: failures
? `Attached ${count} images. Some automatic images could not be attached; manual attachment is available.`
: `Attached ${count} images; skipped ${skipped + removed} duplicates.`});
ctl.debug("[analyze-images-advanced] Attachment summary", {count, removed, skipped, failures});
return count || removed || explicit ? result : original;
};