Forked from bakit/rag-ultimate
Forked from bakit/rag-ultimate
src / utils / text.ts
export const WHITESPACE_RE = /\s+/g;
export const NON_WORD_RE = /\W+/g;
export const HEADINGS_RE = /^#{1,6}\s/;
export const LISTS_RE = /^[-•]\s/;
export const NUM_LIST_RE = /^\d+[.)]\s/;
export const HEADER_RE = /^[A-Z0-9][A-Za-z0-9\s()\-_/:.]+$/;
export const TEXT_EXTS = /\.(txt|md|markdown|json|js|ts|py|java|c|cpp|cs|html|css|csv|yml|yaml|xml|log)$/i;
export const NEWLINE_RE = /\r?\n/;
export const DOUBLE_NEWLINE_RE = /\n\s*\n/g;
/** Normalize whitespace in O(n) time. */
export function normalizeWhitespace(text: string): string {
return text.replace(WHITESPACE_RE, " ").trim();
}
/** Truncate with ellipsis suffix. O(maxChars). */
export function truncate(text: string, maxChars: number): string {
if (text.length <= maxChars) return text;
return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + "\u2026";
}
/** FNV-1a-like hash for stable string hashing. O(n) time. */
export function stableHash(input: string): string {
let h = 2166136261 >>> 0;
for (let i = 0; i < input.length; i++) {
const code = input.charCodeAt(i);
if (code > 0xFFFF) {
// Include high surrogate to avoid emoji/CJK collisions.
h = Math.imul(h ^ code, 16777619) >>> 0;
i++;
continue;
}
h = Math.imul(h ^ code, 16777619) >>> 0;
}
return h.toString(16);
}
src / utils / text.ts
export const WHITESPACE_RE = /\s+/g;
export const NON_WORD_RE = /\W+/g;
export const HEADINGS_RE = /^#{1,6}\s/;
export const LISTS_RE = /^[-•]\s/;
export const NUM_LIST_RE = /^\d+[.)]\s/;
export const HEADER_RE = /^[A-Z0-9][A-Za-z0-9\s()\-_/:.]+$/;
export const TEXT_EXTS = /\.(txt|md|markdown|json|js|ts|py|java|c|cpp|cs|html|css|csv|yml|yaml|xml|log)$/i;
export const NEWLINE_RE = /\r?\n/;
export const DOUBLE_NEWLINE_RE = /\n\s*\n/g;
/** Normalize whitespace in O(n) time. */
export function normalizeWhitespace(text: string): string {
return text.replace(WHITESPACE_RE, " ").trim();
}
/** Truncate with ellipsis suffix. O(maxChars). */
export function truncate(text: string, maxChars: number): string {
if (text.length <= maxChars) return text;
return text.slice(0, Math.max(0, maxChars - 1)).trimEnd() + "\u2026";
}
/** FNV-1a-like hash for stable string hashing. O(n) time. */
export function stableHash(input: string): string {
let h = 2166136261 >>> 0;
for (let i = 0; i < input.length; i++) {
const code = input.charCodeAt(i);
if (code > 0xFFFF) {
// Include high surrogate to avoid emoji/CJK collisions.
h = Math.imul(h ^ code, 16777619) >>> 0;
i++;
continue;
}
h = Math.imul(h ^ code, 16777619) >>> 0;
}
return h.toString(16);
}