user-docs

import type { IndexedDocument, SourceAdapterContext } from "./types.js";
import { normalizeSourceDocument } from "./normalizer.js";
import { GithubMarkdownSourceAdapter } from "./adapters/githubMarkdownSourceAdapter.js";
import { HuggingFaceMarkdownSourceAdapter } from "./adapters/huggingFaceMarkdownSourceAdapter.js";
import { StaticHtmlSourceAdapter } from "./adapters/staticHtmlSourceAdapter.js";
import { LmStudioConversationSourceAdapter } from "./adapters/lmStudioConversationSourceAdapter.js";

export interface SourceRegistryConfig extends SourceAdapterContext {
  remoteSources: string[];
}

const adapters = [
  new GithubMarkdownSourceAdapter(),
  new HuggingFaceMarkdownSourceAdapter(),
  new LmStudioConversationSourceAdapter(),
  new StaticHtmlSourceAdapter(),
];

export async function loadRemoteIndexedDocuments(config: SourceRegistryConfig): Promise<IndexedDocument[]> {
  const indexed: IndexedDocument[] = [];
  const seen = new Set<string>();

  for (const rawSource of config.remoteSources) {
    const source = rawSource.trim();
    if (!source) continue;
    const adapter = adapters.find((candidate) => candidate.canHandle(source));
    if (!adapter) {
      console.warn(`[sources] no adapter for remote source: ${source}`);
      continue;
    }
    try {
      const docs = await adapter.load(source, config);
      for (const doc of docs) {
        if (seen.has(doc.sourceId)) continue;
        seen.add(doc.sourceId);
        indexed.push(normalizeSourceDocument(doc));
      }
    } catch (err) {
      console.warn(`[sources] failed to load remote source ${source}:`, String(err));
    }
  }

  return indexed;
}