import fs from "node:fs";
import path from "node:path";
import { LocalDocumentStore } from "./store";

export interface ExtractedPdfForIndex {
  readonly url: string;
  readonly title: string;
  readonly fullText: string;
}

/** Store extracted text in a durable, isolated directory and index it as one library. */
export async function indexPdfBatch(
  documents: ReadonlyArray<ExtractedPdfForIndex>,
  name: string,
  baseDir: string,
  store: LocalDocumentStore,
  persist: () => boolean,
): Promise<string> {
  const usable = documents.filter((document) => document.fullText.trim().length > 0);
  if (usable.length === 0) throw new Error("No extracted PDF text to index.");
  fs.mkdirSync(baseDir, { recursive: true });
  const folder = fs.mkdtempSync(path.join(baseDir, "batch-"));
  for (const [index, document] of usable.entries()) {
    const header = `Title: ${document.title.replace(/[\r\n]+/g, " ")}\nSource: ${document.url.replace(/[\r\n]+/g, " ")}\n\n`;
    fs.writeFileSync(path.join(folder, `${String(index + 1).padStart(3, "0")}.txt`), header + document.fullText, "utf-8");
  }
  const library = await store.indexLibrary(name, folder, "Extracted PDF batch", "reference", ["academic"]);
  if (library.fileCount !== usable.length) throw new Error(`Only ${library.fileCount}/${usable.length} PDF texts were indexed. Files remain in ${folder}.`);
  if (!persist()) throw new Error(`RAG index could not be saved. Extracted files remain in ${folder}.`);
  return library.id;
}

import fs from "node:fs";
import path from "node:path";
import { LocalDocumentStore } from "./store";

export interface ExtractedPdfForIndex {
  readonly url: string;
  readonly title: string;
  readonly fullText: string;
}

/** Store extracted text in a durable, isolated directory and index it as one library. */
export async function indexPdfBatch(
  documents: ReadonlyArray<ExtractedPdfForIndex>,
  name: string,
  baseDir: string,
  store: LocalDocumentStore,
  persist: () => boolean,
): Promise<string> {
  const usable = documents.filter((document) => document.fullText.trim().length > 0);
  if (usable.length === 0) throw new Error("No extracted PDF text to index.");
  fs.mkdirSync(baseDir, { recursive: true });
  const folder = fs.mkdtempSync(path.join(baseDir, "batch-"));
  for (const [index, document] of usable.entries()) {
    const header = `Title: ${document.title.replace(/[\r\n]+/g, " ")}\nSource: ${document.url.replace(/[\r\n]+/g, " ")}\n\n`;
    fs.writeFileSync(path.join(folder, `${String(index + 1).padStart(3, "0")}.txt`), header + document.fullText, "utf-8");
  }
  const library = await store.indexLibrary(name, folder, "Extracted PDF batch", "reference", ["academic"]);
  if (library.fileCount !== usable.length) throw new Error(`Only ${library.fileCount}/${usable.length} PDF texts were indexed. Files remain in ${folder}.`);
  if (!persist()) throw new Error(`RAG index could not be saved. Extracted files remain in ${folder}.`);
  return library.id;
}