src / prepend / VisionContextDetector.ts
src / prepend / VisionContextDetector.ts
/**
* Vision Context Detector
*
* Deterministically evaluates whether a user prompt requests analysis, explanation, or interpretation
* of an existing image/diagram/screenshot/chart in the current chat.
*
* Excludes:
* 1. Fresh/unnamed chats (isFreshChat === true) where searched images cannot yet exist.
* 2. Search Images queries ("Explain X with images", "Show me images of Y").
* 3. Pure image lookups lacking analysis/explanation intent ("Find the second image").
* 4. General conceptual questions mentioning image/diagram without specific reference.
*/
const ANALYSIS_INTENT_PATTERN =
/\b(?:analyze|analyzing|analysis|explain|explaining|understand|understanding|describe|describing|interpret|interpreting|read|reading|compare|comparing|look\s+at|looking\s+at|extract|extracting|find\s+(?:the\s+)?errors?|find\s+(?:the\s+)?bugs?|find\s+what(?:\s+is|'s)?\s+wrong|what(?:'s|s|\s+is|\s+'s)?\s+wrong(?:\s+with)?|what\s+can\s+you\s+tell\s+me|tell\s+me\s+about|what\s+does|what\s+is|whats)\b/i;
const IMAGE_NOUNS_SOURCE =
"(?:images?|pics?|pictures?|photos?|diagrams?|figures?|illustrations?|screenshots?|charts?|flowcharts?|schematics?|drawings?)";
const ORDINALS_SOURCE =
"(?:\\d+(?:st|nd|rd|th)|first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth|eleventh|twelfth|thirteenth|fourteenth|fifteenth|sixteenth|seventeenth|eighteenth|nineteenth|twentieth)";
const DEMONSTRATIVES_PATTERN = new RegExp(
`\\b(?:this|that|these|those|both|all)\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const ORDINAL_REF_PATTERN = new RegExp(
`\\b(?:the\\s+)?${ORDINALS_SOURCE}\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const ORDINAL_JOINED_REF_PATTERN = new RegExp(
`\\b(?:the\\s+)?${ORDINALS_SOURCE}\\s+(?:and|or)\\s+${ORDINALS_SOURCE}\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const INDEXED_REF_PATTERN = new RegExp(
`\\b${IMAGE_NOUNS_SOURCE}\\s+(?:#?\\d+|[0-9]+)\\b`,
"i"
);
const SPATIAL_ABOVE_PATTERN = new RegExp(
`\\b(?:the\\s+)?(?:above\\s+${IMAGE_NOUNS_SOURCE}|${IMAGE_NOUNS_SOURCE}\\s+above)\\b`,
"i"
);
const PURE_LOOKUP_PATTERN = new RegExp(
`^(?:find|fetch|get|show|display|look\\s+for|search\\s+for)\\s+(?:the\\s+)?(?:${ORDINALS_SOURCE}\\s+)?${IMAGE_NOUNS_SOURCE}(?:\\s+#?\\d+)?$`,
"i"
);
export interface VisionContextOptions {
isFreshChat?: boolean;
requestId?: string;
}
function matchImageReference(text: string): string | null {
const domMatch = DEMONSTRATIVES_PATTERN.exec(text);
if (domMatch) return domMatch[0];
const ordJoinedMatch = ORDINAL_JOINED_REF_PATTERN.exec(text);
if (ordJoinedMatch) return ordJoinedMatch[0];
const ordMatch = ORDINAL_REF_PATTERN.exec(text);
if (ordMatch) return ordMatch[0];
const idxMatch = INDEXED_REF_PATTERN.exec(text);
if (idxMatch) return idxMatch[0];
const spatialMatch = SPATIAL_ABOVE_PATTERN.exec(text);
if (spatialMatch) return spatialMatch[0];
return null;
}
export function requiresVisionContext(
userText: string,
options?: VisionContextOptions
): boolean {
const text = typeof userText === "string" ? userText.trim() : "";
const isFreshChat = options?.isFreshChat === true;
const analysisMatch = text ? ANALYSIS_INTENT_PATTERN.exec(text) : null;
const hasAnalysisIntent = analysisMatch !== null;
const matchedRef = text ? matchImageReference(text) : null;
const hasImageReference = matchedRef !== null;
// Pure lookup check: e.g. "Find the second image" lacking explicit analysis/explanation keywords
const isPureLookup =
text.length > 0 &&
PURE_LOOKUP_PATTERN.test(text) &&
!/\b(?:explain|analyze|describe|interpret|error|bug|wrong|meaning|show|showing|tell)\b/i.test(text);
const finalResult =
text.length > 0 &&
!isFreshChat &&
hasAnalysisIntent &&
hasImageReference &&
!isPureLookup;
return finalResult;
}
/**
* Vision Context Detector
*
* Deterministically evaluates whether a user prompt requests analysis, explanation, or interpretation
* of an existing image/diagram/screenshot/chart in the current chat.
*
* Excludes:
* 1. Fresh/unnamed chats (isFreshChat === true) where searched images cannot yet exist.
* 2. Search Images queries ("Explain X with images", "Show me images of Y").
* 3. Pure image lookups lacking analysis/explanation intent ("Find the second image").
* 4. General conceptual questions mentioning image/diagram without specific reference.
*/
const ANALYSIS_INTENT_PATTERN =
/\b(?:analyze|analyzing|analysis|explain|explaining|understand|understanding|describe|describing|interpret|interpreting|read|reading|compare|comparing|look\s+at|looking\s+at|extract|extracting|find\s+(?:the\s+)?errors?|find\s+(?:the\s+)?bugs?|find\s+what(?:\s+is|'s)?\s+wrong|what(?:'s|s|\s+is|\s+'s)?\s+wrong(?:\s+with)?|what\s+can\s+you\s+tell\s+me|tell\s+me\s+about|what\s+does|what\s+is|whats)\b/i;
const IMAGE_NOUNS_SOURCE =
"(?:images?|pics?|pictures?|photos?|diagrams?|figures?|illustrations?|screenshots?|charts?|flowcharts?|schematics?|drawings?)";
const ORDINALS_SOURCE =
"(?:\\d+(?:st|nd|rd|th)|first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth|eleventh|twelfth|thirteenth|fourteenth|fifteenth|sixteenth|seventeenth|eighteenth|nineteenth|twentieth)";
const DEMONSTRATIVES_PATTERN = new RegExp(
`\\b(?:this|that|these|those|both|all)\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const ORDINAL_REF_PATTERN = new RegExp(
`\\b(?:the\\s+)?${ORDINALS_SOURCE}\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const ORDINAL_JOINED_REF_PATTERN = new RegExp(
`\\b(?:the\\s+)?${ORDINALS_SOURCE}\\s+(?:and|or)\\s+${ORDINALS_SOURCE}\\s+${IMAGE_NOUNS_SOURCE}\\b`,
"i"
);
const INDEXED_REF_PATTERN = new RegExp(
`\\b${IMAGE_NOUNS_SOURCE}\\s+(?:#?\\d+|[0-9]+)\\b`,
"i"
);
const SPATIAL_ABOVE_PATTERN = new RegExp(
`\\b(?:the\\s+)?(?:above\\s+${IMAGE_NOUNS_SOURCE}|${IMAGE_NOUNS_SOURCE}\\s+above)\\b`,
"i"
);
const PURE_LOOKUP_PATTERN = new RegExp(
`^(?:find|fetch|get|show|display|look\\s+for|search\\s+for)\\s+(?:the\\s+)?(?:${ORDINALS_SOURCE}\\s+)?${IMAGE_NOUNS_SOURCE}(?:\\s+#?\\d+)?$`,
"i"
);
export interface VisionContextOptions {
isFreshChat?: boolean;
requestId?: string;
}
function matchImageReference(text: string): string | null {
const domMatch = DEMONSTRATIVES_PATTERN.exec(text);
if (domMatch) return domMatch[0];
const ordJoinedMatch = ORDINAL_JOINED_REF_PATTERN.exec(text);
if (ordJoinedMatch) return ordJoinedMatch[0];
const ordMatch = ORDINAL_REF_PATTERN.exec(text);
if (ordMatch) return ordMatch[0];
const idxMatch = INDEXED_REF_PATTERN.exec(text);
if (idxMatch) return idxMatch[0];
const spatialMatch = SPATIAL_ABOVE_PATTERN.exec(text);
if (spatialMatch) return spatialMatch[0];
return null;
}
export function requiresVisionContext(
userText: string,
options?: VisionContextOptions
): boolean {
const text = typeof userText === "string" ? userText.trim() : "";
const isFreshChat = options?.isFreshChat === true;
const analysisMatch = text ? ANALYSIS_INTENT_PATTERN.exec(text) : null;
const hasAnalysisIntent = analysisMatch !== null;
const matchedRef = text ? matchImageReference(text) : null;
const hasImageReference = matchedRef !== null;
// Pure lookup check: e.g. "Find the second image" lacking explicit analysis/explanation keywords
const isPureLookup =
text.length > 0 &&
PURE_LOOKUP_PATTERN.test(text) &&
!/\b(?:explain|analyze|describe|interpret|error|bug|wrong|meaning|show|showing|tell)\b/i.test(text);
const finalResult =
text.length > 0 &&
!isFreshChat &&
hasAnalysisIntent &&
hasImageReference &&
!isPureLookup;
return finalResult;
}