src / scoring / fastCriticOptimizer.ts
src / scoring / fastCriticOptimizer.ts
export interface CriticCardInput {
id: string;
url: string;
title: string;
content: string;
engine?: string;
relevanceScore?: number;
sourceTier?: string;
}
export interface FastCriticPartition {
autoApproved: Array<{
id: string;
tier: "A" | "B";
confidence: "High" | "Medium";
reason: string;
}>;
autoRejected: Array<{
id: string;
reason: string;
}>;
needsLlmGrading: Array<CriticCardInput & { compactExcerpt: string }>;
}
const JUNK_TITLE_OR_URL_REGEX =
/\b(merriam-webster\.com\/dictionary|dictionary\.cambridge\.org|collinsdictionary\.com|vocabulary\.com|\/search\?|search\s+page|login|sign\s+in)\b/i;
const CANONICAL_ENGINES = new Set([
"openalex",
"crossref",
"arxiv",
"pubmed",
"wikipedia",
]);
/**
* Partitions evidence cards BEFORE calling the LLM Critic:
* 1. Rejects dictionary/search-page noise ("Search Page", "merriam-webster.com/dictionary/original").
* 2. Auto-approves Tier-0 canonical academic/reference cards & high-relevance (rel >= 0.85) cards.
* 3. Compresses borderline cards to 450 chars so LLM Critic runs 5x faster.
*/
export function partitionCardsForFastCritic(
cards: CriticCardInput[],
topic: string
): FastCriticPartition {
const autoApproved: FastCriticPartition["autoApproved"] = [];
const autoRejected: FastCriticPartition["autoRejected"] = [];
const needsLlmGrading: FastCriticPartition["needsLlmGrading"] = [];
for (const card of cards) {
const combinedHeader = `${card.title} ${card.url}`;
// 1. Instant reject dictionary definitions & generic "Search Page" stubs seen in plugin_3.log
if (
JUNK_TITLE_OR_URL_REGEX.test(combinedHeader) ||
card.title.trim().toLowerCase() === "search page" ||
/^meaning of .+ in english/i.test(card.title.trim())
) {
autoRejected.push({
id: card.id,
reason: "heuristic reject: dictionary or search index page",
});
continue;
}
// 2. Auto-approve Tier-0 verified JSON sources (OpenAlex / Wiki / PubMed)
const engineLower = (card.engine ?? "").toLowerCase();
const isCanonicalDomain =
CANONICAL_ENGINES.has(engineLower) ||
/\b(pubmed\.ncbi\.nlm\.nih\.gov|ncbi\.nlm\.nih\.gov|doi\.org|arxiv\.org|openalex\.org)\b/i.test(
card.url
);
if (isCanonicalDomain && card.content.length >= 180) {
autoApproved.push({
id: card.id,
tier: engineLower === "openalex" || card.url.includes("pubmed") ? "A" : "B",
confidence: "High",
reason: "canonical academic/reference source",
});
continue;
}
// 3. Auto-approve high lexical relevance cards (rel >= 0.85) with substantive text
// Upgrade spec: deterministic relevance >= 0.85 AND >= 400 chars of clean prose
if ((card.relevanceScore ?? 0) >= 0.85 && card.content.length >= 400) {
autoApproved.push({
id: card.id,
tier: "B",
confidence: "High",
reason: `high deterministic relevance (${card.relevanceScore?.toFixed(2)})`,
});
continue;
}
// 4. Send only borderline cards to LLM Critic, with a tight 450-char excerpt
needsLlmGrading.push({
...card,
compactExcerpt: buildCompactCriticExcerpt(card.content, topic, 450),
});
}
return { autoApproved, autoRejected, needsLlmGrading };
}
function buildCompactCriticExcerpt(
content: string,
topic: string,
maxChars = 450
): string {
const clean = content.replace(/\s+/g, " ").trim();
if (clean.length <= maxChars) return clean;
const terms = topic
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((t) => t.length > 3);
const sentences = clean.split(/(?<=[.!?])\s+/);
const scored = sentences.map((s, idx) => {
const lower = s.toLowerCase();
const hits = terms.filter((t) => lower.includes(t)).length;
return { idx, s, hits };
});
scored.sort((a, b) => b.hits - a.hits || a.idx - b.idx);
let out = "";
for (const item of scored) {
if ((out + " " + item.s).trim().length > maxChars) break;
out = (out + " " + item.s).trim();
}
return out || clean.slice(0, maxChars);
}
export interface CriticCardInput {
id: string;
url: string;
title: string;
content: string;
engine?: string;
relevanceScore?: number;
sourceTier?: string;
}
export interface FastCriticPartition {
autoApproved: Array<{
id: string;
tier: "A" | "B";
confidence: "High" | "Medium";
reason: string;
}>;
autoRejected: Array<{
id: string;
reason: string;
}>;
needsLlmGrading: Array<CriticCardInput & { compactExcerpt: string }>;
}
const JUNK_TITLE_OR_URL_REGEX =
/\b(merriam-webster\.com\/dictionary|dictionary\.cambridge\.org|collinsdictionary\.com|vocabulary\.com|\/search\?|search\s+page|login|sign\s+in)\b/i;
const CANONICAL_ENGINES = new Set([
"openalex",
"crossref",
"arxiv",
"pubmed",
"wikipedia",
]);
/**
* Partitions evidence cards BEFORE calling the LLM Critic:
* 1. Rejects dictionary/search-page noise ("Search Page", "merriam-webster.com/dictionary/original").
* 2. Auto-approves Tier-0 canonical academic/reference cards & high-relevance (rel >= 0.85) cards.
* 3. Compresses borderline cards to 450 chars so LLM Critic runs 5x faster.
*/
export function partitionCardsForFastCritic(
cards: CriticCardInput[],
topic: string
): FastCriticPartition {
const autoApproved: FastCriticPartition["autoApproved"] = [];
const autoRejected: FastCriticPartition["autoRejected"] = [];
const needsLlmGrading: FastCriticPartition["needsLlmGrading"] = [];
for (const card of cards) {
const combinedHeader = `${card.title} ${card.url}`;
// 1. Instant reject dictionary definitions & generic "Search Page" stubs seen in plugin_3.log
if (
JUNK_TITLE_OR_URL_REGEX.test(combinedHeader) ||
card.title.trim().toLowerCase() === "search page" ||
/^meaning of .+ in english/i.test(card.title.trim())
) {
autoRejected.push({
id: card.id,
reason: "heuristic reject: dictionary or search index page",
});
continue;
}
// 2. Auto-approve Tier-0 verified JSON sources (OpenAlex / Wiki / PubMed)
const engineLower = (card.engine ?? "").toLowerCase();
const isCanonicalDomain =
CANONICAL_ENGINES.has(engineLower) ||
/\b(pubmed\.ncbi\.nlm\.nih\.gov|ncbi\.nlm\.nih\.gov|doi\.org|arxiv\.org|openalex\.org)\b/i.test(
card.url
);
if (isCanonicalDomain && card.content.length >= 180) {
autoApproved.push({
id: card.id,
tier: engineLower === "openalex" || card.url.includes("pubmed") ? "A" : "B",
confidence: "High",
reason: "canonical academic/reference source",
});
continue;
}
// 3. Auto-approve high lexical relevance cards (rel >= 0.85) with substantive text
// Upgrade spec: deterministic relevance >= 0.85 AND >= 400 chars of clean prose
if ((card.relevanceScore ?? 0) >= 0.85 && card.content.length >= 400) {
autoApproved.push({
id: card.id,
tier: "B",
confidence: "High",
reason: `high deterministic relevance (${card.relevanceScore?.toFixed(2)})`,
});
continue;
}
// 4. Send only borderline cards to LLM Critic, with a tight 450-char excerpt
needsLlmGrading.push({
...card,
compactExcerpt: buildCompactCriticExcerpt(card.content, topic, 450),
});
}
return { autoApproved, autoRejected, needsLlmGrading };
}
function buildCompactCriticExcerpt(
content: string,
topic: string,
maxChars = 450
): string {
const clean = content.replace(/\s+/g, " ").trim();
if (clean.length <= maxChars) return clean;
const terms = topic
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((t) => t.length > 3);
const sentences = clean.split(/(?<=[.!?])\s+/);
const scored = sentences.map((s, idx) => {
const lower = s.toLowerCase();
const hits = terms.filter((t) => lower.includes(t)).length;
return { idx, s, hits };
});
scored.sort((a, b) => b.hits - a.hits || a.idx - b.idx);
let out = "";
for (const item of scored) {
if ((out + " " + item.s).trim().length > maxChars) break;
out = (out + " " + item.s).trim();
}
return out || clean.slice(0, maxChars);
}