src / image_search / utils / ImageDownloader.ts
src / image_search / utils / ImageDownloader.ts
import * as fs from "fs";
import * as path from "path";
import { HTTPClient } from "../../http/HTTPClient";
import { ImageSearchResult, LocalImage } from "../models";
import { CacheManager, generateUrlHash } from "./CacheManager";
export const MAX_IMAGE_BYTES = 50 * 1024 * 1024; // 50 MB byte cap
export const DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 15.0;
export const MIME_TYPE_MAP: Record<string, string> = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/png": ".png",
"image/gif": ".gif",
"image/webp": ".webp",
"image/svg+xml": ".svg",
"image/bmp": ".bmp",
"image/x-ms-bmp": ".bmp",
"image/vnd.microsoft.icon": ".ico",
"image/x-icon": ".ico",
"image/tiff": ".tiff",
"image/avif": ".avif",
};
/**
* Sniff magic bytes and determine correct image file extension.
* Rejects HTML pages (e.g. anti-bot/404 pages) and non-image buffers.
*/
export function determineExtension(
content: Buffer,
contentTypeHeader?: string,
url: string = ""
): string | null {
if (!content || content.length === 0) {
return null;
}
// Reject HTML error/anti-bot pages immediately
const snippet = content.subarray(0, 300).toString("utf-8").toLowerCase().trim();
if (
snippet.startsWith("<!doctype html") ||
snippet.startsWith("<html") ||
snippet.startsWith("<head") ||
snippet.includes("<body")
) {
return null;
}
// 1. Magic byte signature sniffing (High confidence)
if (content.length >= 3 && content[0] === 0xff && content[1] === 0xd8 && content[2] === 0xff) {
return ".jpg";
}
if (
content.length >= 8 &&
content[0] === 0x89 &&
content[1] === 0x50 &&
content[2] === 0x4e &&
content[3] === 0x47 &&
content[4] === 0x0d &&
content[5] === 0x0a &&
content[6] === 0x1a &&
content[7] === 0x0a
) {
return ".png";
}
if (
content.length >= 6 &&
(content.toString("ascii", 0, 6) === "GIF87a" || content.toString("ascii", 0, 6) === "GIF89a")
) {
return ".gif";
}
if (content.length >= 2 && content.toString("ascii", 0, 2) === "BM") {
return ".bmp";
}
if (
content.length >= 12 &&
content.toString("ascii", 0, 4) === "RIFF" &&
content.toString("ascii", 8, 12) === "WEBP"
) {
return ".webp";
}
if (
content.length >= 4 &&
((content[0] === 0x00 && content[1] === 0x00 && content[2] === 0x01 && content[3] === 0x00) ||
(content[0] === 0x00 && content[1] === 0x00 && content[2] === 0x02 && content[3] === 0x00))
) {
return ".ico";
}
if (
content.length >= 4 &&
(content.toString("ascii", 0, 4) === "II*\0" || content.toString("ascii", 0, 4) === "MM\0*")
) {
return ".tiff";
}
if (snippet.includes("<svg") || snippet.includes("<?xml")) {
return ".svg";
}
// 2. Content-Type header match fallback
if (contentTypeHeader) {
const mime = contentTypeHeader.split(";")[0].trim().toLowerCase();
if (MIME_TYPE_MAP[mime]) {
return MIME_TYPE_MAP[mime];
}
}
// 3. URL extension fallback
if (url) {
try {
const parsedUrl = new URL(url);
const urlExt = path.extname(parsedUrl.pathname).toLowerCase();
if (
[".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico", ".tiff", ".avif"].includes(urlExt)
) {
return urlExt === ".jpeg" ? ".jpg" : urlExt;
}
} catch {
// Ignore URL parsing error
}
}
return null;
}
export interface ImageDownloaderOptions {
timeoutSeconds?: number;
cacheManager?: CacheManager;
}
export interface DownloadResult {
filename: string;
filepath: string;
extension: string;
bytes: number;
}
/**
* Isolated Image Downloader responsible for single-image network fetches, format sniffing, and atomic disk writes.
* Mapped from Python `utils/image_downloader.py` ImageDownloader.
*/
export class ImageDownloader {
private timeoutSeconds: number;
private cacheManager: CacheManager;
constructor(options: ImageDownloaderOptions = {}) {
this.timeoutSeconds = options.timeoutSeconds ?? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.cacheManager = options.cacheManager ?? new CacheManager();
}
/**
* Process a list of raw search results into downloaded, locally cached LocalImage objects.
* Performs cache hits check first, single cache cleanup pass, and sequential image downloading.
*/
public async processResults(
results: ImageSearchResult[],
providerName: string,
workingDir: string
): Promise<LocalImage[]> {
if (!results || results.length === 0) {
return [];
}
try {
await fs.promises.mkdir(workingDir, { recursive: true });
} catch {
return [];
}
// Single-pass cache cleanup ONCE during batch image processing
await this.cacheManager.cleanupCache(workingDir);
const httpClient = new HTTPClient({ timeoutSeconds: this.timeoutSeconds });
const localImages: LocalImage[] = [];
for (let i = 0; i < results.length; i++) {
const result = results[i];
const targetUrl = result.imageUrl || result.thumbnailUrl;
if (!targetUrl) continue;
const urlHash = generateUrlHash(targetUrl);
const prefix = `img_${urlHash}`;
// 1. Check if image file already exists in working directory cache
const cachedFilePath = await this.cacheManager.findCachedFile(urlHash, workingDir);
if (cachedFilePath) {
const filename = path.basename(cachedFilePath);
localImages.push(
new LocalImage({
title: result.title,
localFilename: filename,
localPath: cachedFilePath,
originalUrl: targetUrl,
provider: providerName,
width: result.width,
height: result.height,
sourceUrl: result.sourceUrl,
sourceDomain: result.sourceDomain,
})
);
continue;
}
// 2. Download missing image
const downloadRes = await this.downloadSingleImage(
targetUrl,
prefix,
workingDir,
httpClient
);
if (downloadRes) {
localImages.push(
new LocalImage({
title: result.title,
localFilename: downloadRes.filename,
localPath: downloadRes.filepath,
originalUrl: targetUrl,
provider: providerName,
width: result.width,
height: result.height,
sourceUrl: result.sourceUrl,
sourceDomain: result.sourceDomain,
})
);
}
}
return localImages;
}
/**
* Fetch a single remote image URL and write it atomically to disk after verifying magic bytes.
*
* @param url Remote image URL.
* @param baseFilename Filename prefix (without extension).
* @param targetDir Target directory to write the image into.
* @param httpClient Optional pre-created HTTPClient instance (for batch reuse).
*/
public async downloadSingleImage(
url: string,
baseFilename: string,
targetDir: string,
httpClient?: HTTPClient
): Promise<DownloadResult | null> {
const client = httpClient ?? new HTTPClient({ timeoutSeconds: this.timeoutSeconds });
const headers: Record<string, string> = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
};
let response;
try {
response = await client.get(url, { headers });
} catch {
return null;
}
if (response.statusCode !== 200) {
return null;
}
const buffer = response.buffer;
if (!buffer || buffer.length === 0 || buffer.length > MAX_IMAGE_BYTES) {
return null;
}
const contentType = response.headers["content-type"] || response.headers["Content-Type"] || "";
const ext = determineExtension(buffer, contentType, url);
if (!ext) {
return null;
}
const finalFilename = `${baseFilename}${ext}`;
const tmpFilename = `.tmp_${finalFilename}`;
const finalPath = path.join(targetDir, finalFilename);
const tmpPath = path.join(targetDir, tmpFilename);
try {
await fs.promises.mkdir(targetDir, { recursive: true });
await fs.promises.writeFile(tmpPath, buffer);
// Reopen file immediately to verify written byte count & magic bytes
const writtenFileBuffer = await fs.promises.readFile(tmpPath);
if (writtenFileBuffer.length !== buffer.length) {
await fs.promises.unlink(tmpPath);
return null;
}
await fs.promises.rename(tmpPath, finalPath);
return {
filename: finalFilename,
filepath: path.resolve(finalPath),
extension: ext,
bytes: buffer.length,
};
} catch {
if (fs.existsSync(tmpPath)) {
try {
await fs.promises.unlink(tmpPath);
} catch {
// Ignore cleanup error
}
}
return null;
}
}
}
import * as fs from "fs";
import * as path from "path";
import { HTTPClient } from "../../http/HTTPClient";
import { ImageSearchResult, LocalImage } from "../models";
import { CacheManager, generateUrlHash } from "./CacheManager";
export const MAX_IMAGE_BYTES = 50 * 1024 * 1024; // 50 MB byte cap
export const DEFAULT_DOWNLOAD_TIMEOUT_SECONDS = 15.0;
export const MIME_TYPE_MAP: Record<string, string> = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/png": ".png",
"image/gif": ".gif",
"image/webp": ".webp",
"image/svg+xml": ".svg",
"image/bmp": ".bmp",
"image/x-ms-bmp": ".bmp",
"image/vnd.microsoft.icon": ".ico",
"image/x-icon": ".ico",
"image/tiff": ".tiff",
"image/avif": ".avif",
};
/**
* Sniff magic bytes and determine correct image file extension.
* Rejects HTML pages (e.g. anti-bot/404 pages) and non-image buffers.
*/
export function determineExtension(
content: Buffer,
contentTypeHeader?: string,
url: string = ""
): string | null {
if (!content || content.length === 0) {
return null;
}
// Reject HTML error/anti-bot pages immediately
const snippet = content.subarray(0, 300).toString("utf-8").toLowerCase().trim();
if (
snippet.startsWith("<!doctype html") ||
snippet.startsWith("<html") ||
snippet.startsWith("<head") ||
snippet.includes("<body")
) {
return null;
}
// 1. Magic byte signature sniffing (High confidence)
if (content.length >= 3 && content[0] === 0xff && content[1] === 0xd8 && content[2] === 0xff) {
return ".jpg";
}
if (
content.length >= 8 &&
content[0] === 0x89 &&
content[1] === 0x50 &&
content[2] === 0x4e &&
content[3] === 0x47 &&
content[4] === 0x0d &&
content[5] === 0x0a &&
content[6] === 0x1a &&
content[7] === 0x0a
) {
return ".png";
}
if (
content.length >= 6 &&
(content.toString("ascii", 0, 6) === "GIF87a" || content.toString("ascii", 0, 6) === "GIF89a")
) {
return ".gif";
}
if (content.length >= 2 && content.toString("ascii", 0, 2) === "BM") {
return ".bmp";
}
if (
content.length >= 12 &&
content.toString("ascii", 0, 4) === "RIFF" &&
content.toString("ascii", 8, 12) === "WEBP"
) {
return ".webp";
}
if (
content.length >= 4 &&
((content[0] === 0x00 && content[1] === 0x00 && content[2] === 0x01 && content[3] === 0x00) ||
(content[0] === 0x00 && content[1] === 0x00 && content[2] === 0x02 && content[3] === 0x00))
) {
return ".ico";
}
if (
content.length >= 4 &&
(content.toString("ascii", 0, 4) === "II*\0" || content.toString("ascii", 0, 4) === "MM\0*")
) {
return ".tiff";
}
if (snippet.includes("<svg") || snippet.includes("<?xml")) {
return ".svg";
}
// 2. Content-Type header match fallback
if (contentTypeHeader) {
const mime = contentTypeHeader.split(";")[0].trim().toLowerCase();
if (MIME_TYPE_MAP[mime]) {
return MIME_TYPE_MAP[mime];
}
}
// 3. URL extension fallback
if (url) {
try {
const parsedUrl = new URL(url);
const urlExt = path.extname(parsedUrl.pathname).toLowerCase();
if (
[".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico", ".tiff", ".avif"].includes(urlExt)
) {
return urlExt === ".jpeg" ? ".jpg" : urlExt;
}
} catch {
// Ignore URL parsing error
}
}
return null;
}
export interface ImageDownloaderOptions {
timeoutSeconds?: number;
cacheManager?: CacheManager;
}
export interface DownloadResult {
filename: string;
filepath: string;
extension: string;
bytes: number;
}
/**
* Isolated Image Downloader responsible for single-image network fetches, format sniffing, and atomic disk writes.
* Mapped from Python `utils/image_downloader.py` ImageDownloader.
*/
export class ImageDownloader {
private timeoutSeconds: number;
private cacheManager: CacheManager;
constructor(options: ImageDownloaderOptions = {}) {
this.timeoutSeconds = options.timeoutSeconds ?? DEFAULT_DOWNLOAD_TIMEOUT_SECONDS;
this.cacheManager = options.cacheManager ?? new CacheManager();
}
/**
* Process a list of raw search results into downloaded, locally cached LocalImage objects.
* Performs cache hits check first, single cache cleanup pass, and sequential image downloading.
*/
public async processResults(
results: ImageSearchResult[],
providerName: string,
workingDir: string
): Promise<LocalImage[]> {
if (!results || results.length === 0) {
return [];
}
try {
await fs.promises.mkdir(workingDir, { recursive: true });
} catch {
return [];
}
// Single-pass cache cleanup ONCE during batch image processing
await this.cacheManager.cleanupCache(workingDir);
const httpClient = new HTTPClient({ timeoutSeconds: this.timeoutSeconds });
const localImages: LocalImage[] = [];
for (let i = 0; i < results.length; i++) {
const result = results[i];
const targetUrl = result.imageUrl || result.thumbnailUrl;
if (!targetUrl) continue;
const urlHash = generateUrlHash(targetUrl);
const prefix = `img_${urlHash}`;
// 1. Check if image file already exists in working directory cache
const cachedFilePath = await this.cacheManager.findCachedFile(urlHash, workingDir);
if (cachedFilePath) {
const filename = path.basename(cachedFilePath);
localImages.push(
new LocalImage({
title: result.title,
localFilename: filename,
localPath: cachedFilePath,
originalUrl: targetUrl,
provider: providerName,
width: result.width,
height: result.height,
sourceUrl: result.sourceUrl,
sourceDomain: result.sourceDomain,
})
);
continue;
}
// 2. Download missing image
const downloadRes = await this.downloadSingleImage(
targetUrl,
prefix,
workingDir,
httpClient
);
if (downloadRes) {
localImages.push(
new LocalImage({
title: result.title,
localFilename: downloadRes.filename,
localPath: downloadRes.filepath,
originalUrl: targetUrl,
provider: providerName,
width: result.width,
height: result.height,
sourceUrl: result.sourceUrl,
sourceDomain: result.sourceDomain,
})
);
}
}
return localImages;
}
/**
* Fetch a single remote image URL and write it atomically to disk after verifying magic bytes.
*
* @param url Remote image URL.
* @param baseFilename Filename prefix (without extension).
* @param targetDir Target directory to write the image into.
* @param httpClient Optional pre-created HTTPClient instance (for batch reuse).
*/
public async downloadSingleImage(
url: string,
baseFilename: string,
targetDir: string,
httpClient?: HTTPClient
): Promise<DownloadResult | null> {
const client = httpClient ?? new HTTPClient({ timeoutSeconds: this.timeoutSeconds });
const headers: Record<string, string> = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
};
let response;
try {
response = await client.get(url, { headers });
} catch {
return null;
}
if (response.statusCode !== 200) {
return null;
}
const buffer = response.buffer;
if (!buffer || buffer.length === 0 || buffer.length > MAX_IMAGE_BYTES) {
return null;
}
const contentType = response.headers["content-type"] || response.headers["Content-Type"] || "";
const ext = determineExtension(buffer, contentType, url);
if (!ext) {
return null;
}
const finalFilename = `${baseFilename}${ext}`;
const tmpFilename = `.tmp_${finalFilename}`;
const finalPath = path.join(targetDir, finalFilename);
const tmpPath = path.join(targetDir, tmpFilename);
try {
await fs.promises.mkdir(targetDir, { recursive: true });
await fs.promises.writeFile(tmpPath, buffer);
// Reopen file immediately to verify written byte count & magic bytes
const writtenFileBuffer = await fs.promises.readFile(tmpPath);
if (writtenFileBuffer.length !== buffer.length) {
await fs.promises.unlink(tmpPath);
return null;
}
await fs.promises.rename(tmpPath, finalPath);
return {
filename: finalFilename,
filepath: path.resolve(finalPath),
extension: ext,
bytes: buffer.length,
};
} catch {
if (fs.existsSync(tmpPath)) {
try {
await fs.promises.unlink(tmpPath);
} catch {
// Ignore cleanup error
}
}
return null;
}
}
}