Forked from bakit/rag-ultimate
Forked from bakit/rag-ultimate
src / retrieval / queryExpansion.ts
const MAX_EXPANSION_CACHE = 512;
const expansionCache = new Map<string, string[]>();
const CONJUNCT_RE = /\b(?:and|or|vs|versus)\b/i;
function normalizeQuery(query: string): string {
return query.trim().replace(/\s+/g, " ");
}
function splitClauses(query: string): string[] {
// Split on commas, semicolons, and conjunctions to get independent clauses.
const parts = query.split(/[,;]|\b(?:and|or|vs|versus)\b/i);
return parts.map((p) => p.trim()).filter(Boolean);
}
function unique(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const v of values) {
const lower = v.toLowerCase();
if (seen.has(lower)) continue;
seen.add(lower);
result.push(v);
}
return result;
}
/**
* Expand a query into multiple variants for broader retrieval coverage.
*
* OPTIMIZATIONS over the original:
* 1. Simplified clause splitting — removed the duplicate second-half push
* that produced near-duplicate variants wasting retrieval calls.
* 2. Added length-based pruning — variants shorter than 3 chars are dropped.
* 3. Cache key uses a stronger separator (U+0001 instead of U+0000) to
* prevent edge-case collisions with queries containing null bytes.
*/
export function expandQueries(query: string, maxCount: number): string[] {
const base = normalizeQuery(query);
if (base.length < 15) {
return [base];
}
const cacheKey = base + "\u0001" + maxCount;
const cached = expansionCache.get(cacheKey);
if (cached !== undefined) return cached;
const words = base.split(/\W+/).filter(Boolean);
const focus = words.slice(0, 6).join(" ");
const clauseVariants = splitClauses(base).slice(0, 2);
const v0 = words.length > 5 ? words.slice(0, 5).join(" ") : "";
const v1 = focus && focus !== base ? focus : "";
const v2 = clauseVariants[0] ? "tell me about " + clauseVariants[0] : "";
const v3 = clauseVariants[1] ? "explain " + clauseVariants[1] : "";
const v4 = focus && focus.length < 30 ? focus + " meaning" : "";
const limited = unique([base, v0, v1, v2, v3, v4])
.filter((v) => v.length >= 3)
.slice(0, Math.max(1, maxCount));
if (limited.length === 0) {
expansionCache.set(cacheKey, [base]);
return [base];
}
if (expansionCache.size >= MAX_EXPANSION_CACHE) {
const oldest = expansionCache.keys().next().value;
if (oldest !== undefined) expansionCache.delete(oldest);
}
expansionCache.set(cacheKey, limited);
return limited;
}
src / retrieval / queryExpansion.ts
const MAX_EXPANSION_CACHE = 512;
const expansionCache = new Map<string, string[]>();
const CONJUNCT_RE = /\b(?:and|or|vs|versus)\b/i;
function normalizeQuery(query: string): string {
return query.trim().replace(/\s+/g, " ");
}
function splitClauses(query: string): string[] {
// Split on commas, semicolons, and conjunctions to get independent clauses.
const parts = query.split(/[,;]|\b(?:and|or|vs|versus)\b/i);
return parts.map((p) => p.trim()).filter(Boolean);
}
function unique(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const v of values) {
const lower = v.toLowerCase();
if (seen.has(lower)) continue;
seen.add(lower);
result.push(v);
}
return result;
}
/**
* Expand a query into multiple variants for broader retrieval coverage.
*
* OPTIMIZATIONS over the original:
* 1. Simplified clause splitting — removed the duplicate second-half push
* that produced near-duplicate variants wasting retrieval calls.
* 2. Added length-based pruning — variants shorter than 3 chars are dropped.
* 3. Cache key uses a stronger separator (U+0001 instead of U+0000) to
* prevent edge-case collisions with queries containing null bytes.
*/
export function expandQueries(query: string, maxCount: number): string[] {
const base = normalizeQuery(query);
if (base.length < 15) {
return [base];
}
const cacheKey = base + "\u0001" + maxCount;
const cached = expansionCache.get(cacheKey);
if (cached !== undefined) return cached;
const words = base.split(/\W+/).filter(Boolean);
const focus = words.slice(0, 6).join(" ");
const clauseVariants = splitClauses(base).slice(0, 2);
const v0 = words.length > 5 ? words.slice(0, 5).join(" ") : "";
const v1 = focus && focus !== base ? focus : "";
const v2 = clauseVariants[0] ? "tell me about " + clauseVariants[0] : "";
const v3 = clauseVariants[1] ? "explain " + clauseVariants[1] : "";
const v4 = focus && focus.length < 30 ? focus + " meaning" : "";
const limited = unique([base, v0, v1, v2, v3, v4])
.filter((v) => v.length >= 3)
.slice(0, Math.max(1, maxCount));
if (limited.length === 0) {
expansionCache.set(cacheKey, [base]);
return [base];
}
if (expansionCache.size >= MAX_EXPANSION_CACHE) {
const oldest = expansionCache.keys().next().value;
if (oldest !== undefined) expansionCache.delete(oldest);
}
expansionCache.set(cacheKey, limited);
return limited;
}