rag-ultimate

export const WHITESPACE_RE = /\s+/g;
export const NON_WORD_RE = /\W+/g;
export const HEADINGS_RE = /^#{1,6}\s+/;
export const LISTS_RE = /^[-*\u2022]\s+/;
export const NUM_LIST_RE = /^\d+[.)]\s+/;
export const HEADER_RE = /^[A-Z0-9][A-Za-z0-9\s()\-_:/.]+$/;
export const TEXT_EXTS = /\.(txt|md|markdown|json|js|ts|py|java|c|cpp|cs|html|css|csv|yml|yaml|xml|log)$/i;
export const NEWLINE_RE = /\r?\n/;
export const DOUBLE_NEWLINE_RE = /\n\s*\n/g;

export function normalizeWhitespace(text: string): string {
  return text.replace(WHITESPACE_RE, " ").trim();
}

export function truncate(text: string, maxChars: number): string {
  if (text.length <= maxChars) return text;
  return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + "\u2026";
}

export function stableHash(input: string): string {
  let h = 2166136261 >>> 0;
  for (let i = 0; i < input.length; i++) {
    const code = input.codePointAt(i);
    if (code === undefined) continue;
    if (code > 0xFFFF) i++;
    h = Math.imul(h ^ code, 16777619) >>> 0;
  }
  return h.toString(16);
}