src / documentParser.ts
import { readFile, stat } from "node:fs/promises";
import path from "node:path";
import { extractMhtmlHtml } from "./mhtmlParser";
export interface ParsedDocument {
text: string;
format: string;
metadata: Record<string, string | number>;
}
/** Extract text locally. This function does not upload document contents. */
export async function parseDocumentFile(
filePath: string,
maxBytes: number,
): Promise<ParsedDocument> {
const extension = path.extname(filePath).toLowerCase();
const buffer = await readFile(filePath);
if (buffer.byteLength > maxBytes) {
throw new Error("The document exceeds the configured file-size limit.");
}
if (buffer.byteLength === 0) {
return {
text: "",
format: extension.replace(/^\./, "") || "txt",
metadata: { characters: 0 },
};
}
let text = "";
const metadata: Record<string, string | number> = {};
switch (extension) {
case ".pdf": {
const pdfjs = await import("pdfjs-dist");
const loadingTask = pdfjs.getDocument({
data: new Uint8Array(buffer),
useSystemFonts: true,
isEvalSupported: false,
});
const pdf = await loadingTask.promise;
const pages: string[] = [];
try {
for (
let pageNumber = 1;
pageNumber <= pdf.numPages;
pageNumber += 1
) {
const page = await pdf.getPage(pageNumber);
const content = await page.getTextContent();
const pageText = content.items
.map((item) => ("str" in item ? item.str : ""))
.join(" ");
pages.push(
`\n\n--- PDF page ${pageNumber} ---\n${pageText}`,
);
page.cleanup();
}
metadata.pages = pdf.numPages;
text = pages.join("\n");
} finally {
await pdf.destroy();
}
break;
}
case ".docx": {
const mammoth = await import("mammoth");
const result = await mammoth.extractRawText({ buffer });
text = result.value;
break;
}
case ".xlsx":
case ".xls":
case ".csv": {
const XLSX = await import("xlsx");
const workbook = XLSX.read(buffer, { type: "buffer" });
text = workbook.SheetNames.map((sheetName) => {
const worksheet = workbook.Sheets[sheetName];
const csv = XLSX.utils.sheet_to_csv(worksheet);
return `\n--- Sheet: ${sheetName} ---\n${csv}`;
}).join("\n");
break;
}
case ".html":
case ".htm":
text = htmlToText(buffer.toString("utf8"));
break;
case ".mhtml":
case ".mht": {
text = htmlToText(extractMhtmlHtml(buffer));
break;
}
case ".json": {
const source = buffer.toString("utf8");
try {
text = JSON.stringify(JSON.parse(source), null, 2);
} catch {
text = source;
}
break;
}
case ".txt":
case ".text":
case ".md":
case ".markdown":
case ".tsv":
case ".log":
case ".xml":
case ".yaml":
case ".yml":
case ".jsonl":
text = buffer.toString("utf8");
break;
default:
throw new Error(
`Unsupported file type "${extension || "none"}". ` +
"Supported formats include TXT, Markdown, HTML, MHTML, MHT, " +
"PDF, DOCX, CSV, JSON, JSONL, XML, YAML, XLSX, and XLS.",
);
}
const normalizedText = normalizeText(text);
return {
text: normalizedText,
format: extension.replace(/^\./, "") || "txt",
metadata: {
characters: normalizedText.length,
},
};
}
export function htmlToText(html: string): string {
return html
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "")
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "")
.replace(/<\/(p|div|section|article|h[1-6]|li|tr)>/gi, "\n")
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<[^>]+>/g, " ")
.replace(/ /gi, " ")
.replace(/&/gi, "&")
.replace(/</gi, "<")
.replace(/>/gi, ">")
.replace(/"/gi, '"')
.replace(/'/g, "'");
}
function normalizeText(value: string): string {
return value
.replace(/\r\n?/g, "\n")
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
}
src / documentParser.ts
import { readFile, stat } from "node:fs/promises";
import path from "node:path";
import { extractMhtmlHtml } from "./mhtmlParser";
export interface ParsedDocument {
text: string;
format: string;
metadata: Record<string, string | number>;
}
/** Extract text locally. This function does not upload document contents. */
export async function parseDocumentFile(
filePath: string,
maxBytes: number,
): Promise<ParsedDocument> {
const extension = path.extname(filePath).toLowerCase();
const buffer = await readFile(filePath);
if (buffer.byteLength > maxBytes) {
throw new Error("The document exceeds the configured file-size limit.");
}
if (buffer.byteLength === 0) {
return {
text: "",
format: extension.replace(/^\./, "") || "txt",
metadata: { characters: 0 },
};
}
let text = "";
const metadata: Record<string, string | number> = {};
switch (extension) {
case ".pdf": {
const pdfjs = await import("pdfjs-dist");
const loadingTask = pdfjs.getDocument({
data: new Uint8Array(buffer),
useSystemFonts: true,
isEvalSupported: false,
});
const pdf = await loadingTask.promise;
const pages: string[] = [];
try {
for (
let pageNumber = 1;
pageNumber <= pdf.numPages;
pageNumber += 1
) {
const page = await pdf.getPage(pageNumber);
const content = await page.getTextContent();
const pageText = content.items
.map((item) => ("str" in item ? item.str : ""))
.join(" ");
pages.push(
`\n\n--- PDF page ${pageNumber} ---\n${pageText}`,
);
page.cleanup();
}
metadata.pages = pdf.numPages;
text = pages.join("\n");
} finally {
await pdf.destroy();
}
break;
}
case ".docx": {
const mammoth = await import("mammoth");
const result = await mammoth.extractRawText({ buffer });
text = result.value;
break;
}
case ".xlsx":
case ".xls":
case ".csv": {
const XLSX = await import("xlsx");
const workbook = XLSX.read(buffer, { type: "buffer" });
text = workbook.SheetNames.map((sheetName) => {
const worksheet = workbook.Sheets[sheetName];
const csv = XLSX.utils.sheet_to_csv(worksheet);
return `\n--- Sheet: ${sheetName} ---\n${csv}`;
}).join("\n");
break;
}
case ".html":
case ".htm":
text = htmlToText(buffer.toString("utf8"));
break;
case ".mhtml":
case ".mht": {
text = htmlToText(extractMhtmlHtml(buffer));
break;
}
case ".json": {
const source = buffer.toString("utf8");
try {
text = JSON.stringify(JSON.parse(source), null, 2);
} catch {
text = source;
}
break;
}
case ".txt":
case ".text":
case ".md":
case ".markdown":
case ".tsv":
case ".log":
case ".xml":
case ".yaml":
case ".yml":
case ".jsonl":
text = buffer.toString("utf8");
break;
default:
throw new Error(
`Unsupported file type "${extension || "none"}". ` +
"Supported formats include TXT, Markdown, HTML, MHTML, MHT, " +
"PDF, DOCX, CSV, JSON, JSONL, XML, YAML, XLSX, and XLS.",
);
}
const normalizedText = normalizeText(text);
return {
text: normalizedText,
format: extension.replace(/^\./, "") || "txt",
metadata: {
characters: normalizedText.length,
},
};
}
export function htmlToText(html: string): string {
return html
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "")
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "")
.replace(/<\/(p|div|section|article|h[1-6]|li|tr)>/gi, "\n")
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<[^>]+>/g, " ")
.replace(/ /gi, " ")
.replace(/&/gi, "&")
.replace(/</gi, "<")
.replace(/>/gi, ">")
.replace(/"/gi, '"')
.replace(/'/g, "'");
}
function normalizeText(value: string): string {
return value
.replace(/\r\n?/g, "\n")
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
}