Forked from vadimfedenko/analyze-images
Forked from vadimfedenko/analyze-images
scripts / integration-smoke.cjs
scripts / integration-smoke.cjs
const assert = require('node:assert/strict');
const http = require('node:http');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { Chat, ChatMessage, FileHandle } = require('@lmstudio/sdk');
const { preprocess } = require('../dist/promptPreprocessor');
const { selectPageImages } = require('../dist/pageImages');
const { main } = require('../dist/index');
(async () => {
const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aWQAAAABJRU5ErkJggg==', 'base64');
const uploads = await fs.mkdtemp(path.join(os.tmpdir(), 'aia-uploads-'));
await fs.writeFile(path.join(uploads, 'existing'), Buffer.concat([png, Buffer.from('manual')]));
const web = http.createServer((req, res) => {
if (req.url.startsWith('/image')) {
res.setHeader('Content-Type', 'image/png'); res.end(req.url === '/image2' ? Buffer.concat([png, Buffer.from('second')]) : png);
} else if (req.url === '/empty') {
res.end('<main>No images</main>');
} else {
const origin = `http://127.0.0.1:${web.address().port}`;
res.end(`<img src="/logo"><main><img src="/image2"><img src="/image1"></main>
<script type="application/ld+json">${JSON.stringify({'@graph': [
{'@type': 'Product', url: '/related', image: '/wrong'},
{'@type': 'Product', '@id': origin + '/page#product', image: [
{'@type': 'ImageObject', contentUrl: '/image1', caption: 'DO_NOT_LEAK'},
'/image2', '/image1'], description: 'DO_NOT_LEAK'},
]})}</script>`);
}
});
await new Promise(resolve => web.listen(0, '127.0.0.1', resolve));
const prepared = [];
const logs = [];
const states = [];
const client = { files: { getLocalFileAbsolutePath: async name => ({path: path.join(uploads, name)}), prepareImageBase64: async (name, data) => {
const bytes = Buffer.from(data, 'base64');
assert(bytes.length > 0);
prepared.push({ name, size: bytes.length });
await fs.writeFile(path.join(uploads, `prepared-${prepared.length}`), bytes);
return new FileHandle(client.files, `local:prepared-${prepared.length}`, 'image', bytes.length, name);
} } };
const settings = { autoAttach: true, carryAnalyzed: true, pageUrl: 'https://hoju.jp/rapunzel', maxImages: 3, startImage: 1 };
const ctl = {
client, abortSignal: new AbortController().signal, guardAbort() {},
pullHistory: async () => Chat.from([]),
getPluginConfig: () => ({ get: key => settings[key] }),
createStatus: state => ({ setText() {}, setState: state => states.push(state) }),
debug: (...args) => logs.push(args),
};
try {
const base = `http://127.0.0.1:${web.address().port}`;
const original = ChatMessage.from({ role: 'user', content: [
{ type: 'text', text: `${base}/page Describe images.` },
{ type: 'file', identifier: 'local:existing', name: 'existing.png', sizeBytes: 3, fileType: 'image' },
]}).asImmutableCopy();
const result = await preprocess(ctl, original);
assert.equal(result.getFiles(client).length, 3);
assert.equal(original.getFiles(client).length, 1);
assert(!result.getText().includes('DO_NOT_LEAK'));
assert(result.getText().includes(`Attachment 2: Product.image[1]; ${base}/image1`));
assert(result.getText().includes(`Attachment 3: Product.image[2]; ${base}/image2`));
assert.equal(prepared.length, 0); // No connection-scoped temporary uploads.
assert.equal(result.getFiles(client)[1].sizeBytes, png.length);
assert(result.getFiles(client).slice(1).every(file => file.identifier.startsWith('base64:')));
assert.equal(states.at(-1).status, 'done');
const priorTurn = Chat.empty(); priorTurn.append(result);
ctl.pullHistory = async () => priorTurn;
const repeated = ChatMessage.from(`${base}/page Same URL again.`);
assert.equal(await preprocess(ctl, repeated), repeated);
const replay = await preprocess(ctl, ChatMessage.from(`@inject-url ${base}/page Show again.`));
assert.equal(replay.getFiles(client).length, 2);
ctl.pullHistory = async () => Chat.from([]);
const plain = ChatMessage.from('No URL here.');
assert.equal(await preprocess(ctl, plain), plain);
settings.autoAttach = false;
const disabled = ChatMessage.from(`${base}/page`);
assert.equal(await preprocess(ctl, disabled), disabled);
settings.pageUrl = `${base}/page`; settings.startImage = 2; settings.maxImages = 1;
const second = await preprocess(ctl, ChatMessage.from('@inject-url Compare this image.'));
assert.equal(second.getFiles(client).length, 1);
assert(second.getText().includes('Product.image[2]'));
assert(!second.getText().includes('@inject-url'));
settings.autoAttach = true;
const noImages = ChatMessage.from(`${base}/empty`);
assert.equal(await preprocess(ctl, noImages), noImages);
await assert.rejects(() => preprocess(ctl, ChatMessage.from(`@inject-url ${base}/empty`)), /No page images/);
const noUploads = { ...ctl, client: {files: {prepareImageBase64: async () => {throw Error('Temporary uploads must not be used');}}} };
assert.equal((await preprocess(noUploads, ChatMessage.from(`@inject-url ${base}/page`))).getFiles(client).length, 1);
await assert.rejects(() => preprocess(ctl, ChatMessage.from(`@inject-url ${base}/empty`)), /No page images/);
assert.equal(states.at(-1).status, 'error');
const abortedSignal = new AbortController(); abortedSignal.abort();
await assert.rejects(() => preprocess({...ctl, abortSignal: abortedSignal.signal, guardAbort() {abortedSignal.signal.throwIfAborted();}}, ChatMessage.from(`${base}/page`)));
const handlers = {};
await main({
withConfigSchematics: value => {handlers.config = value;},
withGlobalConfigSchematics: value => {handlers.globalConfig = value;},
withPromptPreprocessor: value => {handlers.preprocess = value;},
withToolsProvider: value => {handlers.tools = value;},
});
assert.equal(handlers.preprocess, preprocess);
assert(handlers.config);
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'analyze-images-integration-'));
await fs.writeFile(path.join(directory, 'sample.png'), png);
await fs.writeFile(path.join(directory, 'sample.json'), JSON.stringify({caption: 'regression fixture', sourcePageUrl: base}));
const toolCalls = {status() {}, warn() {}, signal: new AbortController().signal, callId: 0};
const model = {vision: true, respond: async (messages, options) => {
assert.equal(messages[0].images.length, 1);
assert(messages[0].images[0].identifier.startsWith('base64:'));
assert(messages[0].content.includes('regression fixture'));
assert.equal(options.maxTokens, 2048);
return {content: 'mock visual analysis'};
}};
const tools = await handlers.tools({client: {...client, llm: {model: async () => model}}, getWorkingDirectory: () => directory, getPluginConfig: () => ({get: key => key === 'analysisBackend' ? 'auto' : undefined})});
assert.deepEqual(tools.map(t => t.name), ['List Local Images Advanced', 'Search Local Images Advanced', 'Analyze Local Image Advanced', 'Multi Analyze Local Images Advanced']);
const listed = await tools[0].implementation({}, toolCalls);
assert.equal(listed.images[0].name, 'sample.png');
const searched = await tools[1].implementation({query: 'regression'}, toolCalls);
assert.equal(searched.images[0].name, 'sample.png');
const analyzed = await tools[2].implementation({imageName: 'sample.png', prompt: 'describe'}, toolCalls);
assert.equal(analyzed.analysis, 'mock visual analysis');
assert.equal(analyzed._aiaImageCandidate.relativePath, 'sample.png');
const rejected = await tools[2].implementation({imageName: '../sample.png', prompt: 'describe'}, toolCalls);
assert.equal(typeof rejected, 'string'); assert(rejected.startsWith('Error:'));
console.log('PASS: combined hook registration and existing List/Search/Analyze flow, including path rejection.');
assert.deepEqual(selectPageImages('<meta property="og:image" content="/hero">', base), [{url: base + '/hero', source: 'page metadata image'}]);
console.log('PASS: actual HTTP retrieval, target Product selection, JSON-LD order/dedup, immutable input and existing attachments, caption isolation, auto/explicit modes, image index, empty page and inline attachments without temporary uploads.');
if (process.argv.includes('--live')) {
settings.startImage = 1; settings.maxImages = 2;
const live = await preprocess(ctl, ChatMessage.from('https://hoju.jp/rapunzel Describe attached images.'));
assert.equal(live.getFiles(client).length, 2);
const selected = logs.findLast(entry => entry[1]?.selected)?.[1].selected;
assert(selected[0].url.includes('138475990_00_'));
assert(selected[1].url.includes('138475990_01_'));
console.log('PASS live page: both Product images downloaded and appended in order.');
console.log(JSON.stringify({ selected, prepared: prepared.slice(-2) }, null, 2));
}
console.log('Controller/model are mocked; real SDK inline attachments and HTTP are used. LM Studio UI inference remains untested.');
} finally {
web.closeAllConnections(); await new Promise(resolve => web.close(resolve));
}
})().catch(error => { console.error(error); process.exitCode = 1; });
const assert = require('node:assert/strict');
const http = require('node:http');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { Chat, ChatMessage, FileHandle } = require('@lmstudio/sdk');
const { preprocess } = require('../dist/promptPreprocessor');
const { selectPageImages } = require('../dist/pageImages');
const { main } = require('../dist/index');
(async () => {
const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aWQAAAABJRU5ErkJggg==', 'base64');
const uploads = await fs.mkdtemp(path.join(os.tmpdir(), 'aia-uploads-'));
await fs.writeFile(path.join(uploads, 'existing'), Buffer.concat([png, Buffer.from('manual')]));
const web = http.createServer((req, res) => {
if (req.url.startsWith('/image')) {
res.setHeader('Content-Type', 'image/png'); res.end(req.url === '/image2' ? Buffer.concat([png, Buffer.from('second')]) : png);
} else if (req.url === '/empty') {
res.end('<main>No images</main>');
} else {
const origin = `http://127.0.0.1:${web.address().port}`;
res.end(`<img src="/logo"><main><img src="/image2"><img src="/image1"></main>
<script type="application/ld+json">${JSON.stringify({'@graph': [
{'@type': 'Product', url: '/related', image: '/wrong'},
{'@type': 'Product', '@id': origin + '/page#product', image: [
{'@type': 'ImageObject', contentUrl: '/image1', caption: 'DO_NOT_LEAK'},
'/image2', '/image1'], description: 'DO_NOT_LEAK'},
]})}</script>`);
}
});
await new Promise(resolve => web.listen(0, '127.0.0.1', resolve));
const prepared = [];
const logs = [];
const states = [];
const client = { files: { getLocalFileAbsolutePath: async name => ({path: path.join(uploads, name)}), prepareImageBase64: async (name, data) => {
const bytes = Buffer.from(data, 'base64');
assert(bytes.length > 0);
prepared.push({ name, size: bytes.length });
await fs.writeFile(path.join(uploads, `prepared-${prepared.length}`), bytes);
return new FileHandle(client.files, `local:prepared-${prepared.length}`, 'image', bytes.length, name);
} } };
const settings = { autoAttach: true, carryAnalyzed: true, pageUrl: 'https://hoju.jp/rapunzel', maxImages: 3, startImage: 1 };
const ctl = {
client, abortSignal: new AbortController().signal, guardAbort() {},
pullHistory: async () => Chat.from([]),
getPluginConfig: () => ({ get: key => settings[key] }),
createStatus: state => ({ setText() {}, setState: state => states.push(state) }),
debug: (...args) => logs.push(args),
};
try {
const base = `http://127.0.0.1:${web.address().port}`;
const original = ChatMessage.from({ role: 'user', content: [
{ type: 'text', text: `${base}/page Describe images.` },
{ type: 'file', identifier: 'local:existing', name: 'existing.png', sizeBytes: 3, fileType: 'image' },
]}).asImmutableCopy();
const result = await preprocess(ctl, original);
assert.equal(result.getFiles(client).length, 3);
assert.equal(original.getFiles(client).length, 1);
assert(!result.getText().includes('DO_NOT_LEAK'));
assert(result.getText().includes(`Attachment 2: Product.image[1]; ${base}/image1`));
assert(result.getText().includes(`Attachment 3: Product.image[2]; ${base}/image2`));
assert.equal(prepared.length, 0); // No connection-scoped temporary uploads.
assert.equal(result.getFiles(client)[1].sizeBytes, png.length);
assert(result.getFiles(client).slice(1).every(file => file.identifier.startsWith('base64:')));
assert.equal(states.at(-1).status, 'done');
const priorTurn = Chat.empty(); priorTurn.append(result);
ctl.pullHistory = async () => priorTurn;
const repeated = ChatMessage.from(`${base}/page Same URL again.`);
assert.equal(await preprocess(ctl, repeated), repeated);
const replay = await preprocess(ctl, ChatMessage.from(`@inject-url ${base}/page Show again.`));
assert.equal(replay.getFiles(client).length, 2);
ctl.pullHistory = async () => Chat.from([]);
const plain = ChatMessage.from('No URL here.');
assert.equal(await preprocess(ctl, plain), plain);
settings.autoAttach = false;
const disabled = ChatMessage.from(`${base}/page`);
assert.equal(await preprocess(ctl, disabled), disabled);
settings.pageUrl = `${base}/page`; settings.startImage = 2; settings.maxImages = 1;
const second = await preprocess(ctl, ChatMessage.from('@inject-url Compare this image.'));
assert.equal(second.getFiles(client).length, 1);
assert(second.getText().includes('Product.image[2]'));
assert(!second.getText().includes('@inject-url'));
settings.autoAttach = true;
const noImages = ChatMessage.from(`${base}/empty`);
assert.equal(await preprocess(ctl, noImages), noImages);
await assert.rejects(() => preprocess(ctl, ChatMessage.from(`@inject-url ${base}/empty`)), /No page images/);
const noUploads = { ...ctl, client: {files: {prepareImageBase64: async () => {throw Error('Temporary uploads must not be used');}}} };
assert.equal((await preprocess(noUploads, ChatMessage.from(`@inject-url ${base}/page`))).getFiles(client).length, 1);
await assert.rejects(() => preprocess(ctl, ChatMessage.from(`@inject-url ${base}/empty`)), /No page images/);
assert.equal(states.at(-1).status, 'error');
const abortedSignal = new AbortController(); abortedSignal.abort();
await assert.rejects(() => preprocess({...ctl, abortSignal: abortedSignal.signal, guardAbort() {abortedSignal.signal.throwIfAborted();}}, ChatMessage.from(`${base}/page`)));
const handlers = {};
await main({
withConfigSchematics: value => {handlers.config = value;},
withGlobalConfigSchematics: value => {handlers.globalConfig = value;},
withPromptPreprocessor: value => {handlers.preprocess = value;},
withToolsProvider: value => {handlers.tools = value;},
});
assert.equal(handlers.preprocess, preprocess);
assert(handlers.config);
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'analyze-images-integration-'));
await fs.writeFile(path.join(directory, 'sample.png'), png);
await fs.writeFile(path.join(directory, 'sample.json'), JSON.stringify({caption: 'regression fixture', sourcePageUrl: base}));
const toolCalls = {status() {}, warn() {}, signal: new AbortController().signal, callId: 0};
const model = {vision: true, respond: async (messages, options) => {
assert.equal(messages[0].images.length, 1);
assert(messages[0].images[0].identifier.startsWith('base64:'));
assert(messages[0].content.includes('regression fixture'));
assert.equal(options.maxTokens, 2048);
return {content: 'mock visual analysis'};
}};
const tools = await handlers.tools({client: {...client, llm: {model: async () => model}}, getWorkingDirectory: () => directory, getPluginConfig: () => ({get: key => key === 'analysisBackend' ? 'auto' : undefined})});
assert.deepEqual(tools.map(t => t.name), ['List Local Images Advanced', 'Search Local Images Advanced', 'Analyze Local Image Advanced', 'Multi Analyze Local Images Advanced']);
const listed = await tools[0].implementation({}, toolCalls);
assert.equal(listed.images[0].name, 'sample.png');
const searched = await tools[1].implementation({query: 'regression'}, toolCalls);
assert.equal(searched.images[0].name, 'sample.png');
const analyzed = await tools[2].implementation({imageName: 'sample.png', prompt: 'describe'}, toolCalls);
assert.equal(analyzed.analysis, 'mock visual analysis');
assert.equal(analyzed._aiaImageCandidate.relativePath, 'sample.png');
const rejected = await tools[2].implementation({imageName: '../sample.png', prompt: 'describe'}, toolCalls);
assert.equal(typeof rejected, 'string'); assert(rejected.startsWith('Error:'));
console.log('PASS: combined hook registration and existing List/Search/Analyze flow, including path rejection.');
assert.deepEqual(selectPageImages('<meta property="og:image" content="/hero">', base), [{url: base + '/hero', source: 'page metadata image'}]);
console.log('PASS: actual HTTP retrieval, target Product selection, JSON-LD order/dedup, immutable input and existing attachments, caption isolation, auto/explicit modes, image index, empty page and inline attachments without temporary uploads.');
if (process.argv.includes('--live')) {
settings.startImage = 1; settings.maxImages = 2;
const live = await preprocess(ctl, ChatMessage.from('https://hoju.jp/rapunzel Describe attached images.'));
assert.equal(live.getFiles(client).length, 2);
const selected = logs.findLast(entry => entry[1]?.selected)?.[1].selected;
assert(selected[0].url.includes('138475990_00_'));
assert(selected[1].url.includes('138475990_01_'));
console.log('PASS live page: both Product images downloaded and appended in order.');
console.log(JSON.stringify({ selected, prepared: prepared.slice(-2) }, null, 2));
}
console.log('Controller/model are mocked; real SDK inline attachments and HTTP are used. LM Studio UI inference remains untested.');
} finally {
web.closeAllConnections(); await new Promise(resolve => web.close(resolve));
}
})().catch(error => { console.error(error); process.exitCode = 1; });