src / image_search / utils / CacheManager.ts
src / image_search / utils / CacheManager.ts
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
export const DEFAULT_IMAGE_CACHE_MAX_BYTES = 500 * 1024 * 1024; // 500 MB
export const DEFAULT_IMAGE_CACHE_MAX_FILES = 1000;
export const DEFAULT_IMAGE_CACHE_EXPIRATION_DAYS = 30.0;
/**
* Generate a deterministic 12-character SHA-256 hash for a given URL.
* Mapped from Python `utils/image_downloader.py` generate_url_hash().
*/
export function generateUrlHash(url: string): string {
return crypto.createHash("sha256").update(url, "utf-8").digest("hex").substring(0, 12);
}
export interface CacheManagerOptions {
maxBytes?: number;
maxFiles?: number;
expirationDays?: number;
}
/**
* Isolated cache manager responsible for SHA-256 URL hashing, cache hit verification, and LRU/TTL cleanup.
*/
export class CacheManager {
private maxBytes: number;
private maxFiles: number;
private expirationDays: number;
constructor(options: CacheManagerOptions = {}) {
this.maxBytes = options.maxBytes ?? DEFAULT_IMAGE_CACHE_MAX_BYTES;
this.maxFiles = options.maxFiles ?? DEFAULT_IMAGE_CACHE_MAX_FILES;
this.expirationDays = options.expirationDays ?? DEFAULT_IMAGE_CACHE_EXPIRATION_DAYS;
}
/**
* Search for an existing cached image file by URL hash in the target working directory.
*/
public async findCachedFile(urlHash: string, workingDir: string): Promise<string | null> {
if (!workingDir || !fs.existsSync(workingDir)) {
return null;
}
const prefix = `img_${urlHash}.`;
try {
const files = await fs.promises.readdir(workingDir);
for (const file of files) {
if (file.startsWith(prefix) && !file.endsWith(".tmp")) {
const fullPath = path.join(workingDir, file);
const stat = await fs.promises.stat(fullPath);
if (stat.isFile() && stat.size > 0) {
// Touch mtime to mark recently accessed for LRU
try {
const now = new Date();
await fs.promises.utimes(fullPath, now, now);
} catch {
// Ignore utimes errors
}
return path.resolve(fullPath);
}
}
}
} catch {
return null;
}
return null;
}
/**
* Perform isolated LRU and TTL cleanup targeting ONLY 'img_*' files in workingDir.
*/
public async cleanupCache(workingDir: string): Promise<void> {
if (!workingDir || !fs.existsSync(workingDir)) {
return;
}
const nowSeconds = Date.now() / 1000;
const expirationSeconds = this.expirationDays * 86400;
try {
const files = await fs.promises.readdir(workingDir);
// Clean up stale temporary files (.tmp_img_*) older than 1 hour
for (const file of files) {
if (file.startsWith(".tmp_img_")) {
const fullPath = path.join(workingDir, file);
try {
const stat = await fs.promises.stat(fullPath);
if (nowSeconds - stat.mtimeMs / 1000 > 3600) {
await fs.promises.unlink(fullPath);
}
} catch {
// Ignore unlink errors
}
}
}
// Collect all 'img_*' cache files
const cacheFiles: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
for (const file of files) {
if (file.startsWith("img_") && !file.startsWith(".tmp_")) {
const fullPath = path.join(workingDir, file);
try {
const stat = await fs.promises.stat(fullPath);
if (stat.isFile()) {
cacheFiles.push({
filePath: fullPath,
mtimeMs: stat.mtimeMs,
size: stat.size,
});
}
} catch {
continue;
}
}
}
if (cacheFiles.length === 0) {
return;
}
// 1. TTL Expiration Cleanup (> 30 days)
const retainedFiles: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
for (const item of cacheFiles) {
const fileAge = nowSeconds - item.mtimeMs / 1000;
if (fileAge > expirationSeconds && fileAge > 60) {
try {
await fs.promises.unlink(item.filePath);
} catch {
// Ignore removal errors
}
} else {
retainedFiles.push(item);
}
}
// 2. LRU Eviction by count and total size
let totalFiles = retainedFiles.length;
let totalSize = retainedFiles.reduce((acc, f) => acc + f.size, 0);
if (totalFiles > this.maxFiles || totalSize > this.maxBytes) {
// Sort oldest modified first
retainedFiles.sort((a, b) => a.mtimeMs - b.mtimeMs);
for (const item of retainedFiles) {
if (totalFiles <= this.maxFiles && totalSize <= this.maxBytes) {
break;
}
if (nowSeconds - item.mtimeMs / 1000 <= 60) {
continue;
}
try {
await fs.promises.unlink(item.filePath);
totalFiles -= 1;
totalSize -= item.size;
} catch {
// Ignore removal errors
}
}
}
} catch {
// Ignore top-level cache cleanup errors
}
}
}
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
export const DEFAULT_IMAGE_CACHE_MAX_BYTES = 500 * 1024 * 1024; // 500 MB
export const DEFAULT_IMAGE_CACHE_MAX_FILES = 1000;
export const DEFAULT_IMAGE_CACHE_EXPIRATION_DAYS = 30.0;
/**
* Generate a deterministic 12-character SHA-256 hash for a given URL.
* Mapped from Python `utils/image_downloader.py` generate_url_hash().
*/
export function generateUrlHash(url: string): string {
return crypto.createHash("sha256").update(url, "utf-8").digest("hex").substring(0, 12);
}
export interface CacheManagerOptions {
maxBytes?: number;
maxFiles?: number;
expirationDays?: number;
}
/**
* Isolated cache manager responsible for SHA-256 URL hashing, cache hit verification, and LRU/TTL cleanup.
*/
export class CacheManager {
private maxBytes: number;
private maxFiles: number;
private expirationDays: number;
constructor(options: CacheManagerOptions = {}) {
this.maxBytes = options.maxBytes ?? DEFAULT_IMAGE_CACHE_MAX_BYTES;
this.maxFiles = options.maxFiles ?? DEFAULT_IMAGE_CACHE_MAX_FILES;
this.expirationDays = options.expirationDays ?? DEFAULT_IMAGE_CACHE_EXPIRATION_DAYS;
}
/**
* Search for an existing cached image file by URL hash in the target working directory.
*/
public async findCachedFile(urlHash: string, workingDir: string): Promise<string | null> {
if (!workingDir || !fs.existsSync(workingDir)) {
return null;
}
const prefix = `img_${urlHash}.`;
try {
const files = await fs.promises.readdir(workingDir);
for (const file of files) {
if (file.startsWith(prefix) && !file.endsWith(".tmp")) {
const fullPath = path.join(workingDir, file);
const stat = await fs.promises.stat(fullPath);
if (stat.isFile() && stat.size > 0) {
// Touch mtime to mark recently accessed for LRU
try {
const now = new Date();
await fs.promises.utimes(fullPath, now, now);
} catch {
// Ignore utimes errors
}
return path.resolve(fullPath);
}
}
}
} catch {
return null;
}
return null;
}
/**
* Perform isolated LRU and TTL cleanup targeting ONLY 'img_*' files in workingDir.
*/
public async cleanupCache(workingDir: string): Promise<void> {
if (!workingDir || !fs.existsSync(workingDir)) {
return;
}
const nowSeconds = Date.now() / 1000;
const expirationSeconds = this.expirationDays * 86400;
try {
const files = await fs.promises.readdir(workingDir);
// Clean up stale temporary files (.tmp_img_*) older than 1 hour
for (const file of files) {
if (file.startsWith(".tmp_img_")) {
const fullPath = path.join(workingDir, file);
try {
const stat = await fs.promises.stat(fullPath);
if (nowSeconds - stat.mtimeMs / 1000 > 3600) {
await fs.promises.unlink(fullPath);
}
} catch {
// Ignore unlink errors
}
}
}
// Collect all 'img_*' cache files
const cacheFiles: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
for (const file of files) {
if (file.startsWith("img_") && !file.startsWith(".tmp_")) {
const fullPath = path.join(workingDir, file);
try {
const stat = await fs.promises.stat(fullPath);
if (stat.isFile()) {
cacheFiles.push({
filePath: fullPath,
mtimeMs: stat.mtimeMs,
size: stat.size,
});
}
} catch {
continue;
}
}
}
if (cacheFiles.length === 0) {
return;
}
// 1. TTL Expiration Cleanup (> 30 days)
const retainedFiles: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
for (const item of cacheFiles) {
const fileAge = nowSeconds - item.mtimeMs / 1000;
if (fileAge > expirationSeconds && fileAge > 60) {
try {
await fs.promises.unlink(item.filePath);
} catch {
// Ignore removal errors
}
} else {
retainedFiles.push(item);
}
}
// 2. LRU Eviction by count and total size
let totalFiles = retainedFiles.length;
let totalSize = retainedFiles.reduce((acc, f) => acc + f.size, 0);
if (totalFiles > this.maxFiles || totalSize > this.maxBytes) {
// Sort oldest modified first
retainedFiles.sort((a, b) => a.mtimeMs - b.mtimeMs);
for (const item of retainedFiles) {
if (totalFiles <= this.maxFiles && totalSize <= this.maxBytes) {
break;
}
if (nowSeconds - item.mtimeMs / 1000 <= 60) {
continue;
}
try {
await fs.promises.unlink(item.filePath);
totalFiles -= 1;
totalSize -= item.size;
} catch {
// Ignore removal errors
}
}
}
} catch {
// Ignore top-level cache cleanup errors
}
}
}