find-image

import { GgufMultimodalEmbeddingBackend } from "./multimodalEmbeddingClientGguf.js";
import { EMBEDDING_INSTRUCTIONS } from "./embeddingInstructions.js";

export type MultimodalEmbeddingBackendKind = "lmstudio-api" | "integrated-local-gguf";

export interface MultimodalEmbeddingHealth {
  ready: boolean;
  backend: MultimodalEmbeddingBackendKind;
  model?: string;
  dimension?: number;
  modelLoaded?: boolean;
  error?: string;
}

export interface MultimodalEmbeddingResult {
  embedding: number[];
  model: string;
  dimension: number;
  model_load_seconds?: number;
}

export interface MultimodalEmbeddingBackend {
  /**
   * Idempotently makes this backend's engine the one actually running
   * behind the shared service URL — adopting it if already running with a
   * matching model/dimension, restarting it if settings changed, or
   * stopping a sibling backend occupying the port and starting this one.
   * This is the same on-demand switch that happens as a side effect of any
   * embed*() call; exposing it directly lets callers (e.g. inspection/CLI
   * tooling) trigger the switch without needing to perform a real embed.
   */
  ensureReady(): Promise<void>;
  health(): Promise<MultimodalEmbeddingHealth>;
  embedImagePath(imagePath: string, dimension?: number, statusLabel?: string): Promise<MultimodalEmbeddingResult>;
  embedImageBytes(bytes: Uint8Array, dimension?: number, statusLabel?: string): Promise<MultimodalEmbeddingResult>;
  embedText(text: string, dimension?: number, statusLabel?: string, instruction?: string): Promise<MultimodalEmbeddingResult>;
  /**
   * Document-side image + canonical metadata embedding. Unlike the query
   * method below, this deliberately does not add an Instruct:/Query: wrapper.
   */
  embedImageBytesAndText(bytes: Uint8Array, text: string, dimension?: number, statusLabel?: string, instruction?: string): Promise<MultimodalEmbeddingResult>;
  /**
   * Combined image+text QUERY embedding: fuses a target image and a
   * descriptive text query into a single cross-modal embedding, instead of
   * embedding the image alone. Qwen3-VL-Embedding natively supports this
   * (official docs show single calls with both "image" and "text" keys).
   * Only used for the query side of `find_image` (target + query together);
   * never used for indexing.
   */
  embedImageAndText(imagePath: string, text: string, dimension?: number, statusLabel?: string, instruction?: string): Promise<MultimodalEmbeddingResult>;
}

export interface MultimodalEmbeddingClientConfig {
  backend: MultimodalEmbeddingBackendKind;
  /** Port the local llama-server embedding process listens on (always bound to 127.0.0.1). */
  port: number;
  lmStudioUrl: string;
  modelPath: string;
  ggufBinaryPath: string;
  ggufGpuLayers?: number;
  model: string;
  dimension: number;
  contextSize: number;
  abortSignal?: AbortSignal;
  onStatus?: (message: string) => void;
}

/**
 * find-image has a single local embedding backend (native `llama-server` /
 * GGUF, see `multimodalEmbeddingClientGguf.ts`). `lmstudio-api` remains a
 * placeholder for a future LM Studio-native embedding endpoint; there is no
 * HF/Python backend and no auto-detection between multiple local backends
 * anymore.
 */
export function resolveMultimodalEmbeddingBackend(
  backend: MultimodalEmbeddingBackendKind,
  _modelPath: string,
): MultimodalEmbeddingBackendKind {
  return backend === "lmstudio-api" ? "lmstudio-api" : "integrated-local-gguf";
}

class LmStudioMultimodalEmbeddingBackend implements MultimodalEmbeddingBackend {
  constructor(private readonly config: MultimodalEmbeddingClientConfig) {}

  async health(): Promise<MultimodalEmbeddingHealth> {
    return {
      ready: false,
      backend: "lmstudio-api",
      model: this.config.model,
      dimension: this.config.dimension,
      error: "LM Studio does not currently expose a compatible multimodal image embedding endpoint.",
    };
  }

  async embedImagePath(): Promise<MultimodalEmbeddingResult> {
    throw new Error("LM Studio multimodal image embeddings are not available yet.");
  }

  async embedImageBytes(): Promise<MultimodalEmbeddingResult> {
    throw new Error("LM Studio multimodal image embeddings are not available yet.");
  }

  async embedText(_text?: string, _dimension?: number, _statusLabel?: string, _instruction?: string): Promise<MultimodalEmbeddingResult> {
    throw new Error("LM Studio multimodal text embeddings are not wired for find-image yet.");
  }

  async embedImageBytesAndText(): Promise<MultimodalEmbeddingResult> {
    throw new Error("LM Studio multimodal image embeddings are not available yet.");
  }

  async embedImageAndText(_imagePath?: string, _text?: string, _dimension?: number, _statusLabel?: string, _instruction?: string): Promise<MultimodalEmbeddingResult> {
    throw new Error("LM Studio multimodal image+text embeddings are not available yet.");
  }

  async ensureReady(): Promise<void> {
    // Nothing to start/adopt locally; the LM Studio API backend talks to a
    // server LM Studio itself manages.
  }
}

export class MultimodalEmbeddingClient implements MultimodalEmbeddingBackend {
  private readonly backend: MultimodalEmbeddingBackend;

  constructor(config: MultimodalEmbeddingClientConfig) {
    const resolvedBackend = resolveMultimodalEmbeddingBackend(config.backend, config.modelPath);
    if (resolvedBackend === "lmstudio-api") {
      this.backend = new LmStudioMultimodalEmbeddingBackend(config);
    } else {
      // Native llama-server / GGUF backend. Uses the shared multimodal
      // config surface: service port, model path, dimension, context size,
      // and GPU layers.
      this.backend = new GgufMultimodalEmbeddingBackend({
        port: config.port,
        binaryPath: config.ggufBinaryPath,
        modelDir: config.modelPath,
        dimension: config.dimension,
        contextSize: config.contextSize,
        nGpuLayers: config.ggufGpuLayers,
        abortSignal: config.abortSignal,
        onStatus: config.onStatus,
      });
    }
  }

  ensureReady(): Promise<void> {
    return this.backend.ensureReady();
  }

  health(): Promise<MultimodalEmbeddingHealth> {
    return this.backend.health();
  }

  embedImagePath(imagePath: string, dimension?: number, statusLabel?: string): Promise<MultimodalEmbeddingResult> {
    return this.backend.embedImagePath(imagePath, dimension, statusLabel);
  }

  embedImageBytes(bytes: Uint8Array, dimension?: number, statusLabel?: string): Promise<MultimodalEmbeddingResult> {
    return this.backend.embedImageBytes(bytes, dimension, statusLabel);
  }

  embedText(text: string, dimension?: number, statusLabel?: string, instruction: string = EMBEDDING_INSTRUCTIONS.textRetrieval): Promise<MultimodalEmbeddingResult> {
    return this.backend.embedText(text, dimension, statusLabel, instruction);
  }

  embedImageBytesAndText(bytes: Uint8Array, text: string, dimension?: number, statusLabel?: string, instruction?: string): Promise<MultimodalEmbeddingResult> {
    return this.backend.embedImageBytesAndText(bytes, text, dimension, statusLabel, instruction);
  }

  embedImageAndText(imagePath: string, text: string, dimension?: number, statusLabel?: string, instruction: string = EMBEDDING_INSTRUCTIONS.fusion): Promise<MultimodalEmbeddingResult> {
    return this.backend.embedImageAndText(imagePath, text, dimension, statusLabel, instruction);
  }
}