rag-ultimate

import { NON_WORD_RE, stableHash } from "../utils/text";
import { normalizeWhitespace } from "../utils/text";

const MAX_TOKEN_CACHE = 256;
const tokenCache = new Map<string, Set<string>>();

export function getTokenSet(input: string): Set<string> {
  const key = stableHash(input.substring(0, 100));
  const cached = tokenCache.get(key);
  if (cached) return cached;

  const tokens = new Set(
    normalizeWhitespace(input.toLowerCase())
      .split(NON_WORD_RE)
      .filter((t) => t.length > 2),
  );

  if (tokenCache.size >= MAX_TOKEN_CACHE) {
    const oldest = tokenCache.keys().next().value;
    if (oldest !== undefined) tokenCache.delete(oldest);
  }
  tokenCache.set(key, tokens);
  return tokens;
}