visit-website-advanced

import { tool, Tool, ToolsProviderController } from "@lmstudio/sdk";
import { writeFile, access } from "fs/promises";
import { join } from "path";
import { z } from "zod";
import { configSchematics } from "./config";

export async function toolsProvider(ctl:ToolsProviderController):Promise<Tool[]> {
	const tools: Tool[] = [];
	const COMPACT_INLINE_LIMIT = 2;
	const COMPACT_GALLERY_LIMIT = 6;

	type FetchHTMLResult = {
		html: string;
		head: string;
		body: string;
		source: "direct" | "jina";
		finalUrl: string;
		statusCode?: number;
		serverHeader?: string;
	};
	const fetchWithRetries = async (targetUrl:string, signal:AbortSignal, headers:Record<string, string>, attempts = 2) => {
		let lastError: unknown = null;
		for(let i = 0; i < attempts; i++) {
			try {
				return await fetch(targetUrl, {
					method: "GET",
					signal,
					headers,
					redirect: "follow",
				});
			} catch(error) {
				lastError = error;
				if(signal.aborted) throw error;
				await new Promise(resolve => setTimeout(resolve, 350 * (i + 1)));
			}
		}
		throw lastError;
	}
	const lastIndexOfRegex = (text: string, regex: RegExp) => {
		const matches = [...text.matchAll(regex)];
		return matches.at(-1)?.index ?? -1;
	};
	const toParsedHtml = (html: string) => {
		const headStartMatch = html.match(/<head\b[^>]*>/i);
		const headStart = headStartMatch?.index ?? -1;

		const headEndMatch = html.match(/<\/head\s*>/i);
		const headEnd = headEndMatch?.index != null
			? headEndMatch.index + headEndMatch[0].length
			: -1;

		const head = headStart >= 0 && headEnd > headStart
			? html.substring(headStart, headEnd)
			: "";

		const bodyStartMatch = html.match(/<body\b[^>]*>/i);
		const bodyStart = bodyStartMatch?.index ?? 0;

		const bodyEnd = lastIndexOfRegex(html, /<\/body\s*>/gi);

		const body = html.substring(
			bodyStart,
			bodyEnd > bodyStart ? bodyEnd : html.length
		);

		return { head, body };
	};
	const looksBlocked = (statusCode:number, _serverHeader:string, html:string) =>
		statusCode === 401
		|| statusCode === 403
		|| statusCode === 429
		|| statusCode === 503
		|| /access denied|captcha|cf-chl|checking your browser|attention required|challenge-platform/i.test(html);
	const fetchViaJina = async (url:string, signal:AbortSignal, warn:(msg:string) => void):Promise<FetchHTMLResult | null> => {
		const jinaUrl = `https://r.jina.ai/http://${url.replace(/^https?:\/\//, "")}`;
		try {
			const response = await fetchWithRetries(jinaUrl, signal, {
				"User-Agent": "Mozilla/5.0",
				"Accept": "text/plain,text/html;q=0.9,*/*;q=0.8",
			});
			if(!response.ok) {
				warn(`Jina fallback failed: ${response.status} ${response.statusText}`);
				return null;
			}
			const text = await response.text();
			if(!text.trim()) {
				warn("Jina fallback returned empty content.");
				return null;
			}
			return {
				html: text,
				head: "",
				body: text,
				source: "jina",
				finalUrl: response.url || jinaUrl,
				statusCode: response.status,
				serverHeader: response.headers.get("server") || "",
			};
		} catch(error: any) {
			if (error instanceof DOMException && error.name === "AbortError") throw error;
			warn(`Jina fallback request failed: ${error?.message || String(error)}`);
			return null;
		}
	}
	const fetchHTML = async (url:string, signal:AbortSignal, warn:(msg:string) => void):Promise<FetchHTMLResult> => {
		const headers = spoofHeaders(url);
		let response: Response;
		try {
			response = await fetchWithRetries(url, signal, headers);
		} catch(error: any) {
			if (error instanceof DOMException && error.name === "AbortError") throw error;
			const fallback = await fetchViaJina(url, signal, warn);
			if(fallback) return fallback;
			throw new Error(`Failed to fetch website: network error (${error?.message || String(error)})`);
		}
		const html = await response.text();
		const statusCode = response.status;
		const serverHeader = response.headers.get("server") || "";
		if (!response.ok || looksBlocked(statusCode, serverHeader, html)) {
			const fallback = await fetchViaJina(url, signal, warn);
			if(fallback) return fallback;
			warn(`Failed to fetch website: ${statusCode} ${response.statusText}`);
			throw new Error(`Failed to fetch website: ${statusCode} ${response.statusText} (server: ${serverHeader || "unknown"})`);
		}
		const { head, body } = toParsedHtml(html);
		return {
			html,
			head,
			body,
			source: "direct",
			finalUrl: response.url || url,
			statusCode,
			serverHeader,
		};
	}

	const extractLinks = (body:string, url:string, maxLinks:number, searchTerms?:string[]) =>
		[...body.matchAll(/<a\s+[^>]*?href="([^"]+)"[^>]*>((?:\n|.)*?)<\/a>/g)]
			.map((match, index) => ({
				index,
				label: match[2]?.replace(/\\[ntr]|\s|<(?:[^>"]|"[^"]*")+>/g, " ").trim() || "",
				link: match[1]?.startsWith("/")
					? new URL(match[1], url).href
					: match[1],
			}))
			.filter(({ link }) => link?.startsWith("http"))
			.map((x, index, { length }) => {
				// Prioritize links fitting the search terms
				// Followed by short navigation links and content links with long labels
				// Fewer digits = more likely a navigation link than a content link
				const ratio = 1 / Math.min(1, /\d/g.exec(x.link)?.length || 1);
				const score
					= ratio * (100 - (x.label.length + x.link.length + (20 * index / length)))
					+ (1 - ratio) * x.label.split(/\s+/).length;
				return {
					...x,
					score: searchTerms?.length
						&& searchTerms.reduce((acc, term) => acc + (x.label.toLowerCase().includes(term.toLowerCase()) ? 1000 : 0), score)
						|| score,
				};
			})
			.sort((a, b) => b.score - a.score) // Sort by score in descending order
			.filter((x, i, arr) =>
				// Filter out duplicates based on link, keeping the first occurrence
				!arr.find((y, j) => j < i && y.link === x.link)
			)
			.slice(0, maxLinks) // Limit number of links
			// .sort((a, b) => a.index - b.index) // Sort by original order in the body
			.map(({ label, link }) => [label, link] as [string, string]);

	const extractImages = (body:string, url:string, maxImages:number, searchTerms?:string[]) => {
		const htmlMatches = [...body.matchAll(/<img(\s+[^>]*)/g)]
			.filter(x => x[1])
			.map(([, attributes], index) => {
				const alt = attributes.match(/\salt="([^"]+)"/)?.[1] || "";
				const src = attributes.match(/\ssrc="([^"]+)"/)?.[1]
					|| attributes.match(/\sdata-src="([^"]+)"/)?.[1]
					|| attributes.match(/\sdata-original="([^"]+)"/)?.[1]
					|| attributes.match(/\ssrcset="([^"]+)"/)?.[1]?.split(",")?.[0]?.trim()?.split(/\s+/)?.[0];
				return { index, alt, src };
			});
		const markdownMatches = [...body.matchAll(/!\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/g)]
			.map((match, index) => ({
				index: index + htmlMatches.length,
				alt: match[1] || "",
				src: match[2],
			}));
		const combined = [...htmlMatches, ...markdownMatches]
			.map(x => ({
				...x,
				src: x.src?.startsWith("/") ? new URL(x.src, url).href : x.src,
			}))
			.filter(({ src }) => src && src.startsWith("http"))
			.map(x => ({
				...x,
				score: searchTerms?.length
					? searchTerms.reduce((acc, term) => acc + (x.alt.toLowerCase().includes(term.toLowerCase()) ? 1000 : 0), x.alt.length)
					: x.alt.length,
			}))
			.sort((a, b) => b.score - a.score)
			.filter((x, i, arr) => !arr.find((y, j) => j < i && y.src === x.src))
			.slice(0, maxImages)
			.sort((a, b) => a.index - b.index);
		return combined.map(({ src, alt }) => [alt, src] as [string, string]);
	}

	const downloadImagesTool = tool({
		name: "Download Images",
		description: "Use with remote HTTP(S) URLs or websiteURL.",
		parameters: {
			imageURLs: z.array(z.string()).optional().describe("List of image references to view. Supports HTTP(S) URLs, local file paths, and markdown image links."),
			websiteURL: z.string().url().optional().describe("The URL of the website, whose images to view."),
			maxImages: z.number().int().min(1).max(200).optional().describe("Maximum number of images to view when websiteURL is provided."),
			imageMetadata: z.array(z.object({
				url: z.string(),
				alt: z.string().optional(),
				caption: z.string().optional(),
				sourcePageUrl: z.string().optional(),
				source: z.string().optional(),
			})).optional().describe("Optional metadata for downloaded images."),
		},
		implementation: async ({ imageURLs, websiteURL, maxImages, imageMetadata }, { status, warn, signal }) => {
			try {
				maxImages = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("maxImages"), -1)
					?? maxImages
					?? 10;

				const metadataForImages = (imageMetadata || [])
					.map((item: any) => ({
						...item,
						url: normalizeImageReference(item.url) || item.url,
					}));

				const imageURLsToDownload = (imageURLs || [])
					.map(normalizeImageReference)
					.filter((x): x is string => !!x);

				if (imageURLsToDownload.length === 0 && !websiteURL) {
					return [];
				}

				if(websiteURL) {
					status("Fetching image URLs from website...");
					const { body } = await fetchHTML(websiteURL, signal, warn);
					const images = extractImages(body, websiteURL, maxImages);

					imageURLsToDownload.push(...images.map(x => x[1]));

					metadataForImages.push(
						...images.map(([alt, src]) => ({
							url: src,
							alt,
							sourcePageUrl: websiteURL,
							source: "download-images",
						}))
					);
				}

				status("Downloading images...");
				const workingDirectory = ctl.getWorkingDirectory();
				const timestamp = Date.now();
				const downloadPromises = imageURLsToDownload.map(async (url:string, i:number) => {
					if (isLocalImagePath(url)) {
						const safeLocal = await resolveLocalImagePathForCurrentWorkingDirectory(url, workingDirectory);
						if (!safeLocal) {
							return null;
						}
						return { fullPath: safeLocal, thumbPath: safeLocal }; // Already local, no resize needed here
					}
					
					const index = i + 1;
					try {
						const headers = spoofHeaders(url);
						const imageResponse = await fetch(url, {
							method: "GET",
							signal,
							headers,
						});
						if (!imageResponse.ok) {
							warn(`Failed to fetch image ${index}: ${imageResponse.statusText}`);
							return null; // Skip this image if download fails
						}
						const bytes = await imageResponse.bytes();
						if (bytes.length === 0) {
							warn(`Image ${index} is empty: ${url}`);
							return null; // Skip empty images
						}
						// save the image to a file in the working directory
						const fileExtension = /image\/([\w]+)/.exec(imageResponse.headers.get('content-type') || '')?.[1]
							|| /\.([\w]+)(?:\?.*)$/.exec(url)?.[1] // Extract extension from URL if content type is not available
							|| 'jpg'; // Default to jpg if no content type
						const fileName = `${timestamp}-${index}.${fileExtension}`;
						const filePath = join(workingDirectory, fileName);
						const localPath = fileName;

						await writeFile(filePath, bytes, 'binary');

						const thumbFileName = `${timestamp}-${index}-thumb.webp`;
						const thumbFilePath = join(workingDirectory, thumbFileName);
						const thumbCreated = await createThumbnailWebp(bytes, thumbFilePath);
						const thumbLocalPath = thumbCreated ? thumbFileName : localPath;

						const metadata = metadataForImages.find((item: any) => item.url === url);
						if (metadata) {
							const metadataFileName = `${timestamp}-${index}.json`;
							const metadataFilePath = join(workingDirectory, metadataFileName);

							await writeFile(
								metadataFilePath,
								JSON.stringify(
									{
										...metadata,
										localImage: localPath,
										localThumbnail: thumbLocalPath,
										localMetadata: metadataFileName,
										downloadedAt: new Date().toISOString(),
									},
									null,
									2
								),
								"utf8"
							);
						}

						return {
							fullPath: localPath,
							thumbPath: thumbLocalPath,
						};
					} catch (error: any) {
						if (error instanceof DOMException && error.name === "AbortError")
							return null; // Skip if download was aborted
						warn(`Error fetching image ${index}: ${error.message}`);
						return null; // Skip this image on error
					}
				});
				const downloadedImageEntries = (await Promise.all(downloadPromises))
					.filter((x): x is { fullPath: string; thumbPath: string } => !!x);
				if (downloadedImageEntries.length === 0) {
					// Avoid returning remote URLs as if they were downloaded.
					return {
						count: 0,
						images: [],
						compactGalleryMarkdown: "",
						hint: "No images were downloaded successfully. The listed source URLs may be blocked or unavailable.",
					};
				}

				status(`Downloaded ${downloadedImageEntries.length} images successfully.`);

				return buildImagePresentationPayload(downloadedImageEntries, COMPACT_INLINE_LIMIT, COMPACT_GALLERY_LIMIT);
			} catch (error: any) {
				if (error instanceof DOMException && error.name === "AbortError") {
					return "Image download aborted by user.";
				}
				console.error(error);
				warn(`Error during image download: ${error.message}`);
				return `Error: ${error.message}`;
			}
		}
	});

	const decodeHtmlEntities = (text = "") =>
		text
			.replace(/&nbsp;/g, " ")
			.replace(/&quot;/g, '"')
			.replace(/&#34;/g, '"')
			.replace(/&#x22;/gi, '"')
			.replace(/&amp;/g, "&")
			.replace(/&#38;/g, "&")
			.replace(/&#x26;/gi, "&")
			.replace(/&lt;/g, "<")
			.replace(/&gt;/g, ">")
			.replace(/&#39;/g, "'")
			.replace(/&apos;/g, "'")
			.replace(/&#x27;/gi, "'");

	const htmlToText = (html = "") =>
		decodeHtmlEntities(
			html
				.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
				.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
				.replace(/<[^>]+>/g, " ")
				.replace(/\s+/g, " ")
				.trim()
		);

	const extractFirstTagText = (html = "", tagName) => {
		const match = html.match(
			new RegExp(`<${tagName}\\b[^>]*>([\\s\\S]*?)<\\/${tagName}>`, "i")
		);

		return match ? htmlToText(match[1]) : "";
	};

	const extractTagTexts = (html = "", tagName) => {
		const matches = [
			...html.matchAll(
				new RegExp(`<${tagName}\\b[^>]*>([\\s\\S]*?)<\\/${tagName}>`, "gi")
			)
		];

		return matches
			.map((match) => htmlToText(match[1]))
			.filter(Boolean);
	};

	const getAttr = (tag: string, attrName: string) => {
		const match = tag.match(
			new RegExp(`\\b${attrName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i")
		);

		return decodeHtmlEntities(match?.[1] || match?.[2] || match?.[3] || "");
	};

	const extractMetaContent = (html: string, attrName: string, attrValue: string) => {
		const metaTags = html.match(/<meta\b[^>]*>/gi) || [];

		const tag = metaTags.find(
			(metaTag) => getAttr(metaTag, attrName).toLowerCase() === attrValue.toLowerCase()
		);

		return tag ? getAttr(tag, "content").trim() : "";
	};

	const extractJsonLd = (html = "", warn?: (msg: string) => void): unknown[] => {
		const scripts = [
			...html.matchAll(
				/<script\b(?=[^>]*type\s*=\s*(?:"[^"]*application\/ld\+json[^"]*"|'[^']*application\/ld\+json[^']*'|[^\s>]*application\/ld\+json[^\s>]*))[^>]*>([\s\S]*?)<\/script\s*>/gi
			)
		];

		const jsonLd: unknown[] = [];

		for (const script of scripts) {
			const raw = decodeHtmlEntities(script[1].trim());

			if (!raw) continue;

			try {
				jsonLd.push(JSON.parse(raw));
			} catch (error: any) {
				warn?.(`Invalid JSON-LD skipped: ${error.message}`);
				jsonLd.push(raw);
			}
		}

		return jsonLd;
	};

	const extractHeadingTree = (html = "", limit = 100) => {
		const headingMatches = [
			...html.matchAll(/<h([1-3])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi)
		];

		const root = [];
		const stack = [];
		let count = 0;

		for (const match of headingMatches) {
			if (count >= limit) break;

			const level = Number(match[1]);
			const text = htmlToText(match[2]);

			if (!text) continue;

			count++;

			const node = {
				level,
				text,
				children: []
			};

			while (stack.length && stack.at(-1).level >= level) {
				stack.pop();
			}

			if (stack.length) {
				stack.at(-1).children.push(node);
			} else {
				root.push(node);
			}

			stack.push(node);
		}

		return root;
	};

	const visitWebsiteTool = tool({
		name: "Visit Website Advanced",
		description: "Visit a website and return enhanced metadata, description, heading structure, JSON-LD, links, images, and text content. Images are automatically downloaded and viewable.",
		parameters: {
			url: z.string().url().describe("The URL of the website to visit"),
			findInPage: z.array(z.string()).optional().describe("Highly recommended! Optional search terms to prioritize which links, images, and content to return."),
			maxLinks: z.number().int().min(0).max(200).optional().describe("Maximum number of links to extract from the page."),
			maxImages: z.number().int().min(0).max(200).optional().describe("Maximum number of images to extract from the page."),
			contentLimit: z.number().int().min(0).max(10_000).optional().describe("Maximum text content length to extract from the page."),
		},
		implementation: async ({ url, maxLinks, maxImages, contentLimit, findInPage: searchTerms }, context) => {
			const { status, warn, signal } = context;
			status("Visiting website...");

			try {
				maxLinks = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("maxLinks"), -1)
					?? maxLinks
					?? 40;
				maxImages = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("maxImages"), -1)
					?? maxImages
					?? 10;
				contentLimit = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("contentLimit"), -1)
					?? contentLimit
					?? 2000;

				const { html, head, body, source, finalUrl, statusCode, serverHeader } =
				await fetchHTML(url, signal, warn);

				const metadataHtml = html || `${head || ""}\n${body || ""}`;
				const bodyHtml = body || metadataHtml;
				status("Website visited successfully.");

				const title =
					extractFirstTagText(metadataHtml, "title")
					|| extractMetaContent(metadataHtml, "property", "og:title")
					|| extractMetaContent(metadataHtml, "name", "twitter:title")
					|| "";

				const description =
					extractMetaContent(metadataHtml, "name", "description")
					|| extractMetaContent(metadataHtml, "property", "og:description")
					|| extractMetaContent(metadataHtml, "name", "twitter:description")
					|| "";

				const h1s = extractTagTexts(bodyHtml, "h1");
				const h2s = extractTagTexts(bodyHtml, "h2");
				const h3s = extractTagTexts(bodyHtml, "h3");

				const h1 = h1s[0] || "";
				const h2 = h2s[0] || "";
				const h3 = h3s[0] || "";

				const headingTree = extractHeadingTree(bodyHtml);
				const jsonLd = extractJsonLd(metadataHtml, warn);
				const links = maxLinks && extractLinks(bodyHtml, url, maxLinks, searchTerms);
				const imagesToFetch = maxImages ? extractImages(bodyHtml, url, maxImages, searchTerms) : [];
				const imageMetadata = imagesToFetch.map(([alt, src]) => ({
					url: src,
					alt,
					sourcePageUrl: url,
					source: "visit-website-advanced",
				}));

				const imagesPayload = maxImages &&
					await downloadImagesTool.implementation({
						imageURLs: imagesToFetch.map(x => x[1]),
						imageMetadata,
					}, context) as any;
				const imageMarkdowns: string[] = Array.isArray(imagesPayload)
					? imagesPayload
					: imagesPayload?.images || [];
				const images = maxImages &&
					imageMarkdowns.map((markdown, index) => [imagesToFetch[index]?.[0] || "", markdown] as [string, string]);

				// Extract text content from the body HTML.
				const allContent = contentLimit && bodyHtml
					.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove script tags
					.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove style tags
					.replace(/<[^>]+>/g, '') // Remove all HTML tags
					.replace(/\s+/g, ' ') // Normalize whitespace
					.trim() || '';
				
				let content = "";
				if(searchTerms?.length && contentLimit < allContent.length) {
					const padding = `.{0,${contentLimit / (searchTerms.length * 2)}}`;
					const escapeRegExp = (text: string) =>
						text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
					const matches = searchTerms
						.map(term => new RegExp(padding + escapeRegExp(term) + padding, "gi").exec(allContent))
						.filter((match): match is RegExpExecArray => !!match)
						.sort((a, b) => a.index - b.index);
					let nextMinIndex = 0;
					for(const match of matches) {
						// Ensure we don't return duplicates by merging overlapping matches
						content += match.index >= nextMinIndex
							// The Match does not overlap with the previous one
							? match[0]
							// The match overlaps so we just extend the content to include it
							: match[0].slice(nextMinIndex - match.index);
						nextMinIndex = match.index + match[0].length;
					}
				}
				else content = allContent.slice(0, contentLimit) // Limit text length
					
				return {
					url,
					title,
					...(description ? { description } : {}),
					h1,
					h2,
					h3,
					...(headingTree.length ? { headingTree } : {}),
					...(jsonLd.length ? { jsonLd } : {}),
					fetch: { source, finalUrl, statusCode, server: serverHeader || undefined },
					...(links ? { links } : {}),
					...(images ? { images } : {}),
					...(imagesPayload?.compactGalleryMarkdown ? { compactGalleryMarkdown: imagesPayload.compactGalleryMarkdown } : {}),
					...(imagesPayload?.hint ? { hint: imagesPayload.hint } : {}),
					...(content ? { content } : {}),
				};
			} catch (error: any) {
				if (error instanceof DOMException && error.name === "AbortError") {
					return "Website visit aborted by user.";
				}
				console.error(error);
				warn(`Error during website visit: ${error.message}`);
				return `Error: ${error.message}`;
			}
		},
	});


	tools.push(visitWebsiteTool);
	tools.push(downloadImagesTool);
	return tools;
}

const undefinedIfAuto = (value: unknown, autoValue: unknown) =>
	value === autoValue ? undefined : value as undefined;

const spoofedUserAgents = [
	// Random spoofed realistic user agents for DuckDuckGo
	"Mozilla/5.0 (Linux; Android 10; SM-M515F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Mobile Safari/537.36",
	"Mozilla/5.0 (Linux; Android 6.0; E5533) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.101 Mobile Safari/537.36",
	"Mozilla/5.0 (Linux; Android 8.1.0; AX1082) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.83 Mobile Safari/537.36",
	"Mozilla/5.0 (Linux; Android 8.1.0; TM-MID1020A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.96 Safari/537.36",
	"Mozilla/5.0 (Linux; Android 9; POT-LX1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Mobile Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:97.0) Gecko/20100101 Firefox/97.0",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.71",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36 Edg/98.0.1108.62",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36",
	"Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0",
	"Opera/9.80 (Android 7.0; Opera Mini/36.2.2254/119.132; U; id) Presto/2.12.423 Version/12.16",
]

function spoofHeaders(url:string) {
	const domain = new URL(url).hostname;
	return {
		'User-Agent': spoofedUserAgents[Math.floor(Math.random() * spoofedUserAgents.length)],
		'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
		'Accept-Language': 'en-US,en;q=0.9',
		'Accept-Encoding': 'gzip, deflate, br',
		'Referer': 'https://' + domain + '/',
		'Connection': 'keep-alive',
		'Upgrade-Insecure-Requests': '1',
		'Sec-Fetch-Dest': 'document',
		'Sec-Fetch-Mode': 'navigate',
		'Sec-Fetch-Site': 'same-origin',
		'Sec-Fetch-User': '?1',
		'Cache-Control': 'max-age=0',
	};
}

function buildImagePresentationPayload(
	localImages: Array<{ fullPath: string; thumbPath: string }>,
	inlineLimit = 2,
	galleryLimit = 6,
) {
	const thumbImages = localImages.map((x, i) => `![Image ${i + 1}](${x.thumbPath})`);
	const galleryImages = thumbImages.slice(0, galleryLimit);
	const hint = "These images are already downloaded and renderable (stored locally). Use compactGalleryMarkdown when you want a compact table gallery; otherwise, please embed images where they fit with ![](imagefile). The user will not see images unless you embed them. Only standard Markdown (no HTML <img>). Do NOT call Download Images as these are already downloaded local files";

	return {
		count: localImages.length,
		...(thumbImages.length <= inlineLimit ? { images: thumbImages } : {}),
		compactGalleryMarkdown: buildCompactGalleryMarkdown(galleryImages),
		hint,
		thumbnailsUsed: true,
	};
}

function normalizeImageReference(input: string): string | null {
	const trimmed = input.trim();
	if (!trimmed) return null;
	const markdownMatch = trimmed.match(/!\[[^\]]*\]\(([^)]+)\)/);
	const extracted = markdownMatch?.[1]?.trim() || trimmed;
	if (/^https?:\/\//i.test(extracted)) return extracted;
	if (extracted.startsWith("/")) return extracted;
	if (/^[a-zA-Z]:[\\/]/.test(extracted)) return extracted;
	return null;
}

function buildCompactGalleryMarkdown(imageMarkdowns: string[]): string {
	if (imageMarkdowns.length === 0) return "";
	const rows: string[] = ["| 1 | 2 |", "| --- | --- |"];
	for (let i = 0; i < imageMarkdowns.length; i += 2) {
		const left = imageMarkdowns[i] || "";
		const right = imageMarkdowns[i + 1] || "";
		rows.push(`| ${left} | ${right} |`);
	}
	return rows.join("\n");
}

const THUMB_MAX_WIDTH = 360;
const THUMB_WEBP_QUALITY = 60;

function isLocalImagePath(value: string): boolean {
	return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") || /^[a-zA-Z]:[\\/]/.test(value);
}

async function createThumbnailWebp(
	inputBytes: Uint8Array,
	outputPath: string,
): Promise<boolean> {
	try {
		const sharpModule = await import("sharp");
		const sharp = (sharpModule as any).default || sharpModule;
		const outputBytes = await sharp(Buffer.from(inputBytes))
			.resize({ width: THUMB_MAX_WIDTH, fit: "inside", withoutEnlargement: true })
			.webp({ quality: THUMB_WEBP_QUALITY })
			.toBuffer();
		if (!outputBytes?.length) {
			return false;
		}
		await writeFile(outputPath, outputBytes, "binary");
		return true;
	} catch {
		return false;
	}
}

async function resolveLocalImagePathForCurrentWorkingDirectory(
	inputPath: string,
	workingDirectory: string,
): Promise<string | null> {
	if (!inputPath) return null;
	const normalized = inputPath.replace(/\\/g, "/");
	const filename = normalized.split("/").pop() || "";
	const candidates = [inputPath, normalized, filename].filter(Boolean);
	for (const candidate of candidates) {
		const absoluteCandidate = candidate === filename
			? join(workingDirectory, filename)
			: toAbsolutePath(candidate, workingDirectory);
		try {
			await access(absoluteCandidate);
			return candidate === filename ? filename : normalized;
		} catch {
			// Try next candidate
		}
	}
	return null;
}

function toAbsolutePath(pathLike: string, workingDirectory: string): string {
	if (/^[a-zA-Z]:[\\/]/.test(pathLike)) return pathLike;
	if (pathLike.startsWith("/")) {
		if (/^\/Users\//.test(pathLike)) {
			return pathLike.replace(/^\//, "").replace(/\//g, "\\");
		}
		return join(workingDirectory, pathLike);
	}
	return join(workingDirectory, pathLike);
}