// ==================== FILE: src\config.ts ====================

// src/config.ts
import { createConfigSchematics } from "@lmstudio/sdk";

export const configSchematics = createConfigSchematics()
  .field(
    "timeRange",
    "select",
    {
      displayName: "Search Time Range",
      subtitle: "Filter search results by publication date to ensure information freshness.",
      options: [
        { value: "all", displayName: "All time (default)" },
        { value: "year", displayName: "Past year" },
        { value: "month", displayName: "Past month" },
        { value: "week", displayName: "Past week" },
        { value: "day", displayName: "Past 24 hours" },
      ],
    },
    "all",
  )
  .field(
    "researchDepth",
    "select",
    {
      displayName: "Research Depth",
      subtitle:
        "Controls rounds, per-worker budgets, queries, and link-following aggressiveness. " +
        "Sources are collected adaptively with no hard cap — deeper = more sources.",
      options: [
        { value: "shallow", displayName: "Shallow — 1 round, ~10-25 sources, fast" },
        { value: "standard", displayName: "Standard — 3 rounds, ~30-60 sources (recommended)" },
        { value: "deep", displayName: "Deep — 5 rounds, ~60-120 sources, thorough" },
        { value: "deeper", displayName: "Deeper — 10 rounds, ~100-200+ sources, very thorough" },
        { value: "exhaustive", displayName: "Exhaustive — 15 rounds, 200+ sources, maximum depth" },
      ],
    },
    "standard",
  )
  .field(
    "engineSelectionMode",
    "select",
    {
      displayName: "Engine Selection Mode",
      subtitle:
        "Controls how search engines are chosen per worker. " +
        "Adaptive: DDG first, fallbacks if weak. " +
        "Benchmark: Run all engines to test performance. " +
        "Priority: Use best engines from historical data.",
      options: [
        { value: "adaptive", displayName: "Adaptive (Default)" },
        { value: "benchmark", displayName: "Benchmark (Run all)" },
        { value: "priority", displayName: "Priority (Historical)" },
      ],
    },
    "adaptive",
  )
  .field(
    "cacheDuration",
    "select",
    {
      displayName: "Persistent Cache Duration",
      subtitle: "How long visited pages and extracted content are kept in the local cache.",
      options: [
        { value: "30", displayName: "30 Days" },
        { value: "90", displayName: "90 Days" },
        { value: "180", displayName: "6 Months" },
        { value: "365", displayName: "12 Months" },
        { value: "730", displayName: "24 Months" },
      ],
    },
    "30",
  )
  .field(
    "contentLimitPerPage",
    "numeric",
    {
      displayName: "Content Per Page (chars)",
      subtitle:
        "Characters extracted per page. Higher = richer but slower. " +
        "Leave at default to auto-scale with depth preset (1000-20000)",
      min: 1000,
      max: 20000,
      int: true,
      slider: { step: 1000, min: 1000, max: 20000 },
    },
    4000,
  )
  .field(
    "enableLinkFollowing",
    "select",
    {
      displayName: "Link Following",
      subtitle: "Workers follow relevant in-page links (like citations and references)",
      options: [
        { value: "on", displayName: "On — follow top links (recommended)" },
        { value: "off", displayName: "Off — search results only" },
      ],
    },
    "on",
  )
  .field(
    "enableAIPlanning",
    "select",
    {
      displayName: "AI Query Planning",
      subtitle: "Use the loaded model for smarter queries, dynamic decomposition, and synthesis",
      options: [
        { value: "on", displayName: "On — AI-powered (best quality)" },
        { value: "off", displayName: "Off — dimension-based fallback (faster start)" },
      ],
    },
    "on",
  )
  .field(
    "safeSearch",
    "select",
    {
      displayName: "Safe Search",
      options: [
        { value: "strict", displayName: "Strict" },
        { value: "moderate", displayName: "Moderate" },
        { value: "off", displayName: "Off" },
      ],
    },
    "moderate",
  )
  .field(
    "enableLocalSources",
    "select",
    {
      displayName: "Data Sources",
      subtitle: "Choose where the research swarm should pull information from.",
      options: [
        { value: "off", displayName: "Web only" },
        { value: "local", displayName: "Local documents only" },
        { value: "web_local", displayName: "Local documents and web" },
      ],
    },
    "off",
  )
  .field(
    "maxSessionMinutes",
    "numeric",
    {
      displayName: "Max Session Time (minutes)",
      subtitle:
        "Hard cap on wall-clock time for Deep Research runs. " +
        "Set to 0 for Unlimited (runs until exhausted or stagnates).",
      min: 0,
      max: 240,
      int: true,
      slider: { step: 5, min: 0, max: 240 },
    },
    30,
  )
  .field(
    "enableAcademicAPIs",
    "select",
    {
      displayName: "Academic APIs (OpenAlex, Crossref, arXiv)",
      subtitle: "Query academic databases directly for papers and research.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "off",
  )
  .field(
    "enableYouTube",
    "select",
    {
      displayName: "YouTube Transcript Search",
      subtitle: "Search YouTube and extract video transcripts as text sources.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "off",
  )
  .field(
    "enableReferenceSearch",
    "select",
    {
      displayName: "Encyclopedia Search",
      subtitle: "Prioritize Grokipedia, Encyclopedia.com, and Britannica over Wikipedia.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "on",
  )
    .field(
    "contextBudgetMode",
    "select",
    {
      displayName: "Context Budget Mode",
      subtitle: "Controls how the plugin manages token limits for synthesis. Auto is recommended.",
      options: [
        { value: "auto", displayName: "Auto (Recommended)" },
        { value: "conservative", displayName: "Conservative" },
        { value: "manual", displayName: "Manual Override" },
      ],
    },
    "auto",
  )
  .field(
    "manualContextLimit",
    "numeric",
    {
      displayName: "Manual Context Limit (Tokens)",
      subtitle: "Only used if Mode is Manual. E.g., 8192, 16384, 32768.",
      min: 2048,
      max: 131072,
      int: true,
    },
    8192,
  )
  .field(
    "maxSynthesisInputTokens",
    "numeric",
    {
      displayName: "Max Synthesis Input Tokens",
      subtitle: "Hard cap on tokens sent to the model for final report generation.",
      min: 2000,
      max: 32000,
      int: true,
    },
    18000,
  )
    .field(
    "contextIsolation",
    "select",
    {
      displayName: "LLM Context Isolation",
      subtitle: "Controls how model context is managed. Strict isolation prevents context overflow and history bleed.",
      options: [
        { value: "strict", displayName: "Strict isolation (Recommended)" },
        { value: "worker_reuse", displayName: "Reuse within one worker only" },
        { value: "advanced_reuse", displayName: "Advanced reuse (Not recommended)" },
      ],
    },
    "strict",
  )
  .field(
    "llmCallMode",
    "select",
    {
      displayName: "LLM Call Budget",
      subtitle: "Hard limit on model calls to prevent infinite loops. Standard is highly recommended.",
      options: [
        { value: "compact", displayName: "Compact (Max 20 calls)" },
        { value: "standard", displayName: "Standard (Max 45 calls)" },
        { value: "deep", displayName: "Deep (Max 80 calls)" },
        { value: "extended", displayName: "Extended (Max 120 calls)" },
      ],
    },
    "standard",
  )
  // Add this right BEFORE the final .build()
  .field(
    "flaresolverrUrl",
    "string",
    {
      displayName: "FlareSolverr URL (Advanced)",
      subtitle: "Optional: Local endpoint to bypass strict Cloudflare blocks (e.g., http://127.0.0.1:8191/v1). Leave blank to disable.",
    },
    ""
  )  
  .build();

// ==================== FILE: src\net\ddg.ts ====================

import { fetchPage } from "./http";

export class DdgRateLimiter {
  private lastRequest = 0;
  private minDelay: number;

  constructor(minDelayMs: number = 2500) {
    this.minDelay = minDelayMs;
  }

  async acquire(): Promise<void> {
    const now = Date.now();
    const elapsed = now - this.lastRequest;
    if (elapsed < this.minDelay) {
      await new Promise((resolve) => setTimeout(resolve, this.minDelay - elapsed));
    }
    this.lastRequest = Date.now();
  }
}

export const sharedDdgLimiter = new DdgRateLimiter(2500);

export class DdgLimiterPool {
  private limiters: DdgRateLimiter[] = [];
  private currentIndex = 0;

  constructor(numLanes: number, minDelayMs: number = 2500) {
    for (let i = 0; i < numLanes; i++) {
      this.limiters.push(new DdgRateLimiter(minDelayMs));
    }
  }

  next(): DdgRateLimiter {
    const limiter = this.limiters[this.currentIndex];
    this.currentIndex = (this.currentIndex + 1) % this.limiters.length;
    return limiter;
  }
}

export function resetThrottle(): void {}

/**
 * Bulletproof multi-tier DDG Search Waterfall:
 * Tier A: DDG Lite POST endpoint (fastest, mimics ddgr)
 * Tier B: DDG HTML fallback endpoint (handles strict blocks)
 * Tier C: Graceful recovery (returns empty array instead of throwing crash errors)
 */
export async function searchDDG(
  query: string,
  maxResults: number,
  safeSearch: "strict" | "moderate" | "off" = "moderate",
  signal?: AbortSignal,
  limiter?: DdgRateLimiter,
  timeRange: string = "all"
): Promise<ReadonlyArray<{ url: string; title: string; snippet: string }>> {
  if (limiter) await limiter.acquire();
  if (signal?.aborted) throw new DOMException("Aborted", "AbortError");

  const safeParam = safeSearch === "strict" ? "1" : safeSearch === "off" ? "-1" : "0";
  const dfParam = timeRange === "all" ? "" : `&df=${timeRange}`;
  const formData = `q=${encodeURIComponent(query)}&kp=${safeParam}${dfParam}`;

  // TIER A: DDG Lite Endpoint
  try {
    const res = await fetchPage("https://lite.duckduckgo.com/lite/", signal!, {
      method: "POST",
      body: formData,
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
        "Referer": "https://lite.duckduckgo.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
      }
    });
    const hits = parseDDGResults(res.html, maxResults);
    if (hits.length > 0) return hits;
  } catch (err) {
    if (signal?.aborted) throw err;
    console.warn(`[DDG Tier A] Failed for query "${query}": ${err instanceof Error ? err.message : String(err)}. Trying Tier B...`);
  }

  // TIER B: DDG HTML Endpoint Fallback
  try {
    if (limiter) await limiter.acquire();
    const htmlFallbackUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
    const resB = await fetchPage(htmlFallbackUrl, signal!, {
      method: "GET",
      headers: {
        "Referer": "https://html.duckduckgo.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
      }
    });
    const hitsB = parseHTMLDDGResults(resB.html, maxResults);
    if (hitsB.length > 0) return hitsB;
  } catch (errB) {
    if (signal?.aborted) throw errB;
    console.warn(`[DDG Tier B] Fallback also failed for query "${query}": ${errB instanceof Error ? errB.message : String(errB)}`);
  }

  // TIER C: Graceful exit (Returns empty array so the worker's outer health system handles it cleanly without crashing)
  return [];
}

export async function searchDDGPaginated(
  query: string,
  maxResultsPerPage: number,
  pages: number,
  safeSearch: "strict" | "moderate" | "off" = "moderate",
  signal?: AbortSignal,
  limiter?: DdgRateLimiter,
  timeRange: string = "all"
): Promise<ReadonlyArray<{ url: string; title: string; snippet: string }>> {
  const allHits: { url: string; title: string; snippet: string }[] = [];

  for (let p = 1; p <= pages; p++) {
    if (signal?.aborted) break;
    
    const safeParam = safeSearch === "strict" ? "1" : safeSearch === "off" ? "-1" : "0";
    const dfParam = timeRange === "all" ? "" : `&df=${timeRange}`;
    const formData = `q=${encodeURIComponent(query)}&kp=${safeParam}${dfParam}&s=${(p - 1) * maxResultsPerPage}`;
    
    try {
      if (limiter) await limiter.acquire();
      const res = await fetchPage("https://lite.duckduckgo.com/lite/", signal!, {
        method: "POST",
        body: formData,
        headers: {
          "Content-Type": "application/x-www-form-urlencoded",
          "Referer": "https://lite.duckduckgo.com/"
        }
      });
      const hits = parseDDGResults(res.html, maxResultsPerPage);
      allHits.push(...hits);
      if (hits.length < maxResultsPerPage) break;
    } catch {
      break;
    }
  }
  return allHits;
}

function parseDDGResults(html: string, maxResults: number): { url: string; title: string; snippet: string }[] {
  const hits: { url: string; title: string; snippet: string }[] = [];
  const seen = new Set<string>();
  
  // Resilient regex pattern matching DDG Lite result rows
  const resultRe = /<a[^>]*rel="nofollow"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<td class="result-snippet">([\s\S]*?)<\/td>/gi;
  let match: RegExpExecArray | null;
  
  while (hits.length < maxResults && (match = resultRe.exec(html)) !== null) {
    let url = match[1];
    const title = match[2].replace(/<[^>]+>/g, "").trim();
    const snippet = match[3].replace(/<[^>]+>/g, "").trim();
    
    // Clean up redirect wrappers if present
    if (url.includes("uddg=")) {
      const matchUrl = url.match(/uddg=([^&]+)/);
      if (matchUrl) url = decodeURIComponent(matchUrl[1]);
    }

    if (url.includes("duckduckgo.com")) continue;
    if (!url.startsWith("http")) continue;
    
    if (seen.has(url)) continue;
    seen.add(url);
    
    hits.push({ url, title, snippet });  
  }
  
  return hits;
}

function parseHTMLDDGResults(html: string, maxResults: number): { url: string; title: string; snippet: string }[] {
  const hits: { url: string; title: string; snippet: string }[] = [];
  const seen = new Set<string>();
  
  // Secondary parser for standard DDG HTML results page layout
  const resultRe = /<a class="result__url" href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
  let match: RegExpExecArray | null;
  
  while (hits.length < maxResults && (match = resultRe.exec(html)) !== null) {
    let url = match[1];
    const title = match[2].replace(/<[^>]+>/g, "").trim();
    const snippet = match[3].replace(/<[^>]+>/g, "").trim();

    if (url.includes("duckduckgo.com")) continue;
    if (!url.startsWith("http") && url.startsWith("//")) url = "https:" + url;
    if (!url.startsWith("http")) continue;
    
    if (seen.has(url)) continue;
    seen.add(url);
    
    hits.push({ url, title, snippet });  
  }
  
  return hits;
}

// ==================== FILE: src\swarm\orchestrator.ts ====================

import { runWorker, CrawlMetrics } from "./worker";
import {
  buildQueryPlan,
  buildAdaptiveGapFill,
  summariseFindings,
} from "../planning/planner";
import { detectCoveredDimensions, DIMENSIONS, detectGaps } from "../planning/dimensions";
import {
  ResearchConfig,
  SwarmTask,
  WorkerResult,
  CrawledSource,
  WorkerRole,
  AgentMessage,
  StatusFn,
  WarnFn,
  SourceTier,
  ContradictionEntry,
} from "../types";
import { DepthProfile } from "../constants";
import { DdgLimiterPool, resetThrottle } from "../net/ddg";
import { VisitedPageCache, normalizeUrl } from "./visited-cache";
import { log } from "./logger";
import { detectContradictions } from "../synthesis/ai";
import { SearchHealthTracker } from "./health";
import { LlmCallManager } from "../utils/llm";
import fs from "node:fs";
import path from "node:path";
import os from "node:os";

type GapPlanLike = {
  readonly role: WorkerRole;
  readonly label: string;
  readonly queries: ReadonlyArray<string>;
  readonly followLinks: boolean;
  readonly preferredTiers?: ReadonlyArray<SourceTier>;
};

function zeroMetrics(): CrawlMetrics {
  return {
    ddgQueries: 0, ddgHits: 0, mutatedQueriesTried: 0, mutationAccepted: 0,
    mutationHits: 0, extraEngineQueries: 0, extraEngineHits: 0, rawHits: 0,
    dedupedHits: 0, rankedCandidates: 0, fetchCandidates: 0, fetchAttempts: 0,
    fetchFailures: 0, acceptedSources: 0, skippedLowWordCount: 0, skippedOffTopic: 0,
    skippedVeryOffTopic: 0, skippedDuplicateContent: 0, skippedVisited: 0,
    skippedDomainCap: 0, skippedAvoided: 0, skippedBlacklisted: 0, cacheChecks: 0,
    cacheHits: 0, cacheAccepted: 0, cacheRejectedDuplicate: 0, cacheRejectedOffTopic: 0,
    cacheRejectedLowWordCount: 0, cacheWrites: 0, followedLinks: 0,
    crossWorkerDiscoveriesUsed: 0, localSourcesAccepted: 0,
  };
}

function mergeMetricObjects(base: CrawlMetrics, delta: Partial<CrawlMetrics>): CrawlMetrics {
  const next = { ...base };
  for (const key of Object.keys(next) as Array<keyof CrawlMetrics>) {
    next[key] = (next[key] ?? 0) + (delta[key] ?? 0);
  }
  return next;
}

function formatProgressBar(current: number, total: number, width: number = 20): string {
  if (total <= 0) return `[${"░".repeat(width)}] 0%`;
  const pct = Math.max(0, Math.min(100, Math.round((current / total) * 100)));
  const filled = Math.round((pct / 100) * width);
  return `[${"█".repeat(filled)}${"░".repeat(width - filled)}] ${pct}%`;
}

class MutableCrawlState implements SharedCrawlState {
  private readonly _visitedUrls = new Set<string>();
  private readonly _contentHashes = new Set<string>();
  private readonly _domainCounts = new Map<string, number>();
  private readonly _domainFailures = new Map<string, number>();
  private readonly _blacklistedDomains = new Set<string>();
  private readonly _discoveries: Array<{ url: string; title: string; fromWorker: string }> = [];
  private readonly _failedUrls = new Map<string, { count: number; reason: string; lastFailedAt: string }>();
  private readonly _failedHosts = new Map<string, { count: number; reason: string; lastFailedAt: string }>();
  private readonly visitedCache: VisitedPageCache;
  private _metrics: CrawlMetrics = zeroMetrics();
  
  addWebCacheDocument(source: CrawledSource): void {
    // Placeholder for local RAG store
  }
  
  constructor(cacheDurationDays: number = 30) {
    this.visitedCache = new VisitedPageCache(cacheDurationDays);
  }

  get visitedUrls(): ReadonlySet<string> { return this._visitedUrls; }
  get contentHashes(): ReadonlySet<string> { return this._contentHashes; }
  get domainCounts(): ReadonlyMap<string, number> { return this._domainCounts; }
  get domainFailures(): ReadonlyMap<string, number> { return this._domainFailures; }

  addVisited(url: string): void { this._visitedUrls.add(normalizeUrl(url)); }
  addHash(hash: string): void { this._contentHashes.add(hash); }
  incrementDomain(url: string): void {
    const host = safeHostname(url);
    if (!host) return;
    this._domainCounts.set(host, (this._domainCounts.get(host) ?? 0) + 1);
  }
  domainCount(url: string): number {
    const host = safeHostname(url);
    return host ? (this._domainCounts.get(host) ?? 0) : 0;
  }

  noteFailure(url: string, reason: string, at: string = new Date().toISOString()): void {
    const normalized = normalizeUrl(url);
    const prevUrl = this._failedUrls.get(normalized);
    this._failedUrls.set(normalized, { count: (prevUrl?.count ?? 0) + 1, reason, lastFailedAt: at });
    const host = safeHostname(normalized);
    if (!host) return;
    const prevHost = this._failedHosts.get(host);
    this._failedHosts.set(host, { count: (prevHost?.count ?? 0) + 1, reason, lastFailedAt: at });
  }

  noteDomainFailure(url: string): void {
    const host = safeHostname(url);
    if (!host) return;
    const count = (this._domainFailures.get(host) ?? 0) + 1;
    this._domainFailures.set(host, count);
    if (count >= 3) this._blacklistedDomains.add(host);
  }

  isDomainBlacklisted(url: string): boolean {
    const host = safeHostname(url);
    return host ? this._blacklistedDomains.has(host) : false;
  }

  shouldAvoidUrl(url: string): boolean {
    const normalized = normalizeUrl(url);
    if (this._failedUrls.has(normalized)) return true;
    const host = safeHostname(normalized);
    if (!host) return false;
    const hostFailCount = this._failedHosts.get(host)?.count ?? 0;
    return hostFailCount >= 2;
  }

  pushDiscovery(url: string, title: string, fromWorker: string): void {
    const normalized = normalizeUrl(url);
    if (!this._visitedUrls.has(normalized)) {
      this._discoveries.push({ url: normalized, title, fromWorker });
    }
  }

  drainDiscoveries(limit: number): ReadonlyArray<{ url: string; title: string }> {
    const results: Array<{ url: string; title: string }> = [];
    while (results.length < limit && this._discoveries.length > 0) {
      const item = this._discoveries.shift()!;
      if (!this._visitedUrls.has(normalizeUrl(item.url))) {
        results.push({ url: item.url, title: item.title });
      }
    }
    return results;
  }

  isRecentlyVisited(url: string): boolean { return this.visitedCache.hasRecent(url); }
  getCachedSource(url: string): CrawledSource | null { return this.visitedCache.getRecent(url); }
  markVisitedPersistent(source: CrawledSource): void { this.visitedCache.markVisited(source); }
  pruneVisitedCache(): void { this.visitedCache.prune(); }
  cacheStats(): { entries: number; file: string; maxAgeDays: number } { return this.visitedCache.stats(); }
  getMetricsSnapshot(): Readonly<CrawlMetrics> { return this._metrics; }
  mergeMetrics(delta: Partial<CrawlMetrics>): void { this._metrics = mergeMetricObjects(this._metrics, delta); }
}

const CORE_ROLES: ReadonlyArray<WorkerRole> = ["breadth", "depth", "recency", "academic", "critical"];
const EXTENDED_ROLES: ReadonlyArray<WorkerRole> = ["statistical", "regulatory", "technical", "primary", "comparative"];
const ROLE_LABELS: Readonly<Record<WorkerRole, string>> = {
  breadth: "Breadth", depth: "Depth", recency: "Recency", academic: "Academic", critical: "Critical",
  statistical: "Statistical/Data", regulatory: "Regulatory/Policy", technical: "Technical Deep-Dive",
  primary: "Primary Sources", comparative: "Comparative Analysis",
};

function getEnginesForRole(
  role: WorkerRole,
  cfg: ResearchConfig,
  mode: "adaptive" | "benchmark" | "priority"
): ReadonlyArray<string> {
  const freeEngines: string[] = ["ddg", "google", "brave"];
  if (cfg.enableReferenceSearch) freeEngines.push("reference");
  if (cfg.enableAcademicAPIs) freeEngines.push("openalex", "crossref", "arxiv");
  if (cfg.enableYouTube) freeEngines.push("youtube");
  freeEngines.push("gdelt");

  if (mode === "benchmark") {
    return [...freeEngines, cfg.serperApiKey ? "serper" : "", cfg.braveApiKey ? "brave-api" : ""].filter(Boolean);
  }

  const roleEngines: string[] = [];
  if (cfg.enableReferenceSearch && (role === "breadth" || role === "academic")) roleEngines.push("reference");
  if (cfg.enableAcademicAPIs && (role === "academic" || role === "technical")) roleEngines.push("openalex", "crossref", "arxiv");
  if (role === "recency" || role === "critical") roleEngines.push("gdelt");

  if (roleEngines.length === 0) {
    roleEngines.push("ddg", "google", "brave");
  } else {
    roleEngines.push("ddg", "brave");
  }

  return roleEngines;
}

function rolesForProfile(profile: DepthProfile): ReadonlyArray<WorkerRole> {
  if (profile.depthRounds >= 10) return [...CORE_ROLES, ...EXTENDED_ROLES];
  if (profile.depthRounds >= 5) return [...CORE_ROLES, "technical", "comparative", "statistical"];
  return [...CORE_ROLES];
}

function buildTaskBase(
  profile: DepthProfile,
  cfg: ResearchConfig,
): Pick<
  SwarmTask,
  | "contentLimit"
  | "safeSearch"
  | "searchResultsPerQuery"
  | "maxPagesPerDomain"
  | "maxLinksToEvaluate"
  | "maxLinksToFollow"
  | "candidatePoolMultiplier"
  | "workerConcurrency"
  | "minRelevanceScore"
  | "maxOutlinksPerPage"
  | "searchPages"
  | "extraEngines"
  | "linkCrawlDepth"
  | "queryMutationThreshold"
  | "enableLocalSources"
  | "localLibraryIds"
  | "timeRange"
  | "roleLibraryMap"
  | "serperApiKey"
  | "braveApiKey"
  | "enableYouTube"
> & { flaresolverrUrl?: string } { // <--- Added type extension here to prevent TypeScript errors
  return {
    contentLimit: cfg.contentLimitPerPage,
    safeSearch: cfg.safeSearch,
    searchResultsPerQuery: profile.searchResultsPerQuery,
    maxPagesPerDomain: profile.maxPagesPerDomain,
    maxLinksToEvaluate: profile.maxLinksToEvaluate,
    maxLinksToFollow: profile.maxLinksToFollow,
    candidatePoolMultiplier: profile.candidatePoolMultiplier,
    workerConcurrency: profile.workerConcurrency,
    minRelevanceScore: profile.minRelevanceScore,
    maxOutlinksPerPage: profile.maxOutlinksPerPage,
    searchPages: profile.searchPages,
    extraEngines: getEnginesForRole("breadth", cfg, cfg.engineSelectionMode ?? "adaptive"),
    linkCrawlDepth: profile.linkCrawlDepth,
    queryMutationThreshold: profile.queryMutationThreshold,
    enableLocalSources: cfg.enableLocalSources,
    localLibraryIds: cfg.localLibraryIds,
    timeRange: cfg.timeRange,
    roleLibraryMap: cfg.roleLibraryMap,
    serperApiKey: cfg.serperApiKey,
    braveApiKey: cfg.braveApiKey,
    enableYouTube: cfg.enableYouTube,
    flaresolverrUrl: (cfg as any).flaresolverrUrl, // <--- THIS grabs the URL from the UI!
  };
}

function buildStaticTask(
  role: WorkerRole,
  queries: ReadonlyArray<string>,
  profile: DepthProfile,
  cfg: ResearchConfig,
  subIdx: number = 0,
): SwarmTask {
  const followRoles: ReadonlyArray<WorkerRole> = ["depth", "academic", "technical", "primary"];
  const academicTiers: ReadonlyArray<SourceTier> = ["academic", "government", "reference"];
  const enginesForRole = getEnginesForRole(role, cfg, cfg.engineSelectionMode ?? "adaptive");

  return {
    ...buildTaskBase(profile, cfg),
    extraEngines: enginesForRole,
    id: `${role}-s${subIdx}-${Date.now()}`,
    role,
    label: subIdx > 0 ? `${ROLE_LABELS[role]} #${subIdx + 1}` : ROLE_LABELS[role],
    queries: [...queries],
    pageBudget: profile.pageBudgetPerWorker,
    followLinks: cfg.enableLinkFollowing && followRoles.includes(role),
    preferredTiers: role === "academic" || role === "regulatory" ? academicTiers : undefined,
  };
}

function fanOutQueries(
  queries: ReadonlyArray<string>,
  fanOut: number,
): ReadonlyArray<ReadonlyArray<string>> {
  if (fanOut <= 1 || queries.length <= 2) return [queries];
  const groups: string[][] = Array.from({ length: fanOut }, () => []);
  for (let i = 0; i < queries.length; i++) {
    groups[i % fanOut].push(queries[i]);
  }
  return groups.filter((g) => g.length > 0);
}

function buildRound1Tasks(
  roles: ReadonlyArray<WorkerRole>,
  queriesByRole: Partial<Record<WorkerRole, ReadonlyArray<string>>>,
  profile: DepthProfile,
  cfg: ResearchConfig,
): SwarmTask[] {
  const allQueries = Array.from(
    new Set(
      Object.values(queriesByRole)
        .flatMap((qs) => qs ?? [])
        .filter((q): q is string => Boolean(q)),
    ),
  );
  const round1Tasks: SwarmTask[] = [];
  for (const role of roles) {
    const roleQueries = (queriesByRole[role] ?? []).filter(Boolean);
    const effectiveQueries = roleQueries.length > 0 ? roleQueries : allQueries;
    const groups = fanOutQueries(effectiveQueries, profile.workerFanOut);
    for (const [subIdx, group] of groups.entries()) {
      if (group.length > 0) {
        round1Tasks.push(buildStaticTask(role, group, profile, cfg, subIdx));
      }
    }
  }
  return round1Tasks;
}

function buildGapTasks(
  gapPlans: ReadonlyArray<GapPlanLike>,
  round: number,
  profile: DepthProfile,
  cfg: ResearchConfig,
): SwarmTask[] {
  const tasks: SwarmTask[] = [];
  const gapEngines: string[] = [];
  if (cfg.serperApiKey) gapEngines.push("serper");
  if (cfg.braveApiKey) gapEngines.push("brave-api");
  if (gapEngines.length === 0) gapEngines.push("ddg", "google", "brave");

  for (const [subIdx, gapPlan] of gapPlans.entries()) {
    if (gapPlan.queries.length === 0) continue;
    tasks.push({
      ...buildTaskBase(profile, cfg),
      extraEngines: gapEngines,
      id: `gap-${gapPlan.role}-r${round}-s${subIdx}-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
      role: gapPlan.role,
      label: subIdx > 0 ? `${gapPlan.label} #${subIdx + 1}` : gapPlan.label,
      queries: [...gapPlan.queries],
      pageBudget: profile.pageBudgetPerGapWorker,
      followLinks: cfg.enableLinkFollowing && gapPlan.followLinks,
      preferredTiers: gapPlan.preferredTiers,
    });
  }
  return tasks;
}

async function runTaskGroup(
  tasks: ReadonlyArray<SwarmTask>,
  state: SharedCrawlState,
  pool: DdgLimiterPool,
  signal: AbortSignal,
  status: StatusFn,
  warn: WarnFn,
  topicKeywords: ReadonlyArray<string>,
  health: SearchHealthTracker,
  llmManager: LlmCallManager
): Promise<WorkerResult[]> {
  return Promise.all(
    tasks.map((task) => {
      const limiter = pool.next();
      return runWorker(
        task,
        state,
        signal,
        status,
        warn,
        topicKeywords,
        limiter,
        health,
        llmManager
      ).catch((err: unknown) => {
        if (!isAbortError(err)) {
          warn(`[${task.label}] crashed: ${err instanceof Error ? err.message : String(err)}`);
        }
        return {
          taskId: task.id,
          role: task.role,
          label: task.label,
          sources: [],
          queries: [],
          errors: [String(err)],
        } satisfies WorkerResult;
      });
    }),
  );
}

export interface OrchestratorResult {
  readonly sources: ReadonlyArray<CrawledSource>;
  readonly queriesUsed: ReadonlyArray<string>;
  readonly workerErrors: ReadonlyArray<string>;
  readonly usedAI: boolean;
  readonly topicKeywords: ReadonlyArray<string>;
}

export interface SharedCrawlState {
  readonly visitedUrls: ReadonlySet<string>;
  readonly contentHashes: ReadonlySet<string>;
  readonly domainCounts: ReadonlyMap<string, number>;
  readonly domainFailures: ReadonlyMap<string, number>;
  addVisited(url: string): void;
  addHash(hash: string): void;
  incrementDomain(url: string): void;
  domainCount(url: string): number;
  noteFailure(url: string, reason: string): void;
  noteDomainFailure(url: string): void;
  isDomainBlacklisted(url: string): boolean;
  shouldAvoidUrl(url: string): boolean;
  pushDiscovery(url: string, title: string, fromWorker: string): void;
  drainDiscoveries(limit: number): ReadonlyArray<{ url: string; title: string }>;
  isRecentlyVisited(url: string): boolean;
  getCachedSource(url: string): CrawledSource | null;
  markVisitedPersistent(source: CrawledSource): void;
  addWebCacheDocument(source: CrawledSource): void;
  pruneVisitedCache(): void;
  cacheStats(): { entries: number; file: string; maxAgeDays: number };
  getMetricsSnapshot(): Readonly<CrawlMetrics>;
  mergeMetrics(delta: Partial<CrawlMetrics>): void;
}

export async function runSwarm(
  cfg: ResearchConfig,
  profile: DepthProfile,
  status: StatusFn,
  warn: WarnFn,
  signal: AbortSignal,
): Promise<OrchestratorResult> {
  const fileStatus: StatusFn = (msg: string) => { log(msg); status(msg); };
  const fileWarn: WarnFn = (msg: string) => { log(`[WARN] ${msg}`); warn(msg); };

  const state = new MutableCrawlState(cfg.cacheDuration ? parseInt(cfg.cacheDuration, 10) : 30);
  state.pruneVisitedCache();
  const allSources: CrawledSource[] = [];
  const allQueries: string[] = [];
  const allErrors: string[] = [];
  
  const health = new SearchHealthTracker();
  const llmManager = new LlmCallManager(cfg.llmCallMode ?? "standard", cfg.contextIsolation ?? "strict");

  fileStatus(`\n🚀 DEEP RESEARCH SWARM LAUNCHED (Strict Priority Mode)\n`);
  fileStatus(`[RUN CONTROL] Budget: ${llmManager.budget.maxGlobalCalls} calls | Timeout: ${llmManager.budget.maxRuntimeMs / 60000}m | Isolation: ${llmManager.isolationMode}`);

  const plan = await buildQueryPlan(cfg.topic, cfg.focusAreas, cfg.enableAIPlanning, fileStatus, profile);
  const roles = rolesForProfile(profile);
  const pool = new DdgLimiterPool(profile.searchLanes, profile.ddgRateLimitMs);

  // ==========================================
  // LAYER 1: MANDATORY LOCAL SOURCE PASS
  // ==========================================
  if (cfg.enableLocalSources) {
    fileStatus(`\n📚 LAYER 1: Searching Local RAG Libraries first...`);
    const localTasks = buildRound1Tasks(roles, plan.queriesByRole, profile, cfg);
    const localOnlyTasks = localTasks.map(t => ({ ...t, extraEngines: [] as ReadonlyArray<string> }));
    const localResults = await runTaskGroup(localOnlyTasks, state, pool, signal, fileStatus, fileWarn, plan.topicKeywords, health, llmManager);
    aggregateResults(localResults, allSources, allQueries, allErrors);
    
    const localCount = allSources.length;
    health.localChunksRetrieved = localCount;
    health.localChunksAccepted = localCount;
    fileStatus(`✅ Layer 1 Complete. Found ${localCount} local sources.`);

    if (localCount >= profile.pageBudgetPerWorker * roles.length) {
      fileStatus(`Local evidence SUFFICIENT. Skipping external web search.`);
    }
  } else {
    fileStatus(`\n⚠️ Local sources disabled. Proceeding to web search.`);
  }

  // ==========================================
  // LAYERS 2-5: EXTERNAL GAP-DRIVEN SEARCH
  // ==========================================
  fileStatus(`\n🌐 LAYERS 2-5: External Gap-Driven Search...`);
  
  const coveredIds = detectCoveredDimensions(allSources.map(s => s.text));
  const gaps = detectGaps(coveredIds);
  
  for (const gap of gaps) {
    if (signal.aborted) break;
    health.gaps.push({
      id: `gap-${gap.id}`,
      topic: cfg.topic,
      missingClaim: gap.label,
      whyInsufficient: `Dimension ${gap.label} not covered by local sources`,
      freshnessRequired: gap.id === 'current' || gap.id === 'future',
      preferredTier: null,
      searchLayerAuthorized: "DDG",
      resolved: false
    });
  }

  let externalTasks = buildGapTasks(gaps.map(g => ({
    role: "breadth", 
    label: `Gap: ${g.label}`,
    queries: g.queries(cfg.topic),
    followLinks: true
  })) as ReadonlyArray<GapPlanLike>, 1, profile, cfg);

  const maxSessionTimeMs = cfg.maxSessionMs || 30 * 60 * 1000;
  const startTime = Date.now();
  const crawlDeadline = startTime + (maxSessionTimeMs * 0.80); // Reserve 20% for synthesis

  for (const layer of ["DDG", "SEARXNG", "DIRECT", "API"]) {
    if (Date.now() >= crawlDeadline || signal.aborted || externalTasks.length === 0) {
      fileWarn(`[TIME ALLOCATION] Crawl budget elapsed or tasks finished. Transitioning to verification & synthesis.`);
      break;
    }

    let layerEngines: string[] = [];
    if (layer === "DDG" && health.isDdgAvailable()) layerEngines = ["ddg"];
    else if (layer === "SEARXNG" && !health.isDdgAvailable()) layerEngines = ["searxng"];
    else if (layer === "DIRECT") layerEngines = ["reference", "gdelt"];
    else if (layer === "API" && health.canUseApi() && cfg.serperApiKey) layerEngines = ["serper"];

    if (layerEngines.length === 0) continue;

    fileStatus(`\n🔍 Executing Layer: ${layer} (${layerEngines.join(", ")})`);
    const layerTasks = externalTasks.map(t => ({ ...t, extraEngines: layerEngines }));
    const layerResults = await runTaskGroup(layerTasks, state, pool, signal, fileStatus, fileWarn, plan.topicKeywords, health, llmManager);
    
    const prevSourceCount = allSources.length;
    aggregateResults(layerResults, allSources, allQueries, allErrors);
    const newSources = allSources.length - prevSourceCount;

    if (newSources > 0) {
      externalTasks = [];
      health.gaps.forEach(g => { g.resolved = true; });
    }
    if (layer === "API") health.apiCallsMade++;
  }

  // Cap Sources to Top-N before logging and passing to synthesis
  const sortedSources = [...allSources].sort((a, b) => {
    const scoreA = (a.domainScore * 0.4) + (a.relevanceScore * 100 * 0.6);
    const scoreB = (b.domainScore * 0.4) + (b.relevanceScore * 100 * 0.6);
    return scoreB - scoreA;
  });

  // Reassign allSources to the filtered top results
  allSources.length = 0; 
  allSources.push(...sortedSources.slice(0, profile.synthesisMaxSources || 25));

  // ==========================================
  // FILE LOGGING
  // ==========================================
  const logDir = path.join(os.homedir(), ".deep-swarm-research", "logs");
  const logFile = path.join(logDir, `run_log_${Date.now()}.txt`);
  
  let logContent = `RUN ID: ${Date.now()}\nTOPIC: ${cfg.topic}\n\n`;
  logContent += health.generateReport();
  logContent += llmManager.getReport();
  logContent += "\n\nSEARCH ENGINE ATTRIBUTION\n";
  logContent += `- Requested Route: DDG / SearxNG / Direct\n`;
  logContent += `- Actual Backend: DDG / Yandex / Bing / Google\n`;
  logContent += `- Result Domains: Tracked in worker metrics\n`;
  logContent += `\nVERIFICATION TIERS\n`;
  logContent += `- Tier A (Canonically verified): ${allSources.filter((s: CrawledSource) => s.domainScore >= 90).length}\n`;
  logContent += `- Tier B (Independently validated): ${allSources.filter((s: CrawledSource) => s.domainScore >= 80 && s.domainScore < 90).length}\n`;
  logContent += `- Tier C (Relevant candidate): ${allSources.filter((s: CrawledSource) => s.domainScore < 80).length}\n`;

  try {
    fs.mkdirSync(logDir, { recursive: true });
    fs.writeFileSync(logFile, logContent, "utf-8");
    fileStatus(`[LOG] Full run report saved to ${logFile}`);
  } catch (e) {
    fileWarn(`[LOG] Failed to write run log: ${e}`);
  }

  // ==========================================
  // FINAL HEALTH REPORT
  // ==========================================
  fileStatus(health.generateReport());
  fileStatus(llmManager.getReport());

  return {
    sources: allSources,
    queriesUsed: [...new Set(allQueries)],
    workerErrors: allErrors,
    usedAI: plan.usedAI,
    topicKeywords: plan.topicKeywords,
  };
}

function logAggregateMetrics(
  state: SharedCrawlState,
  status: StatusFn,
  label: string,
): void {
  const m = state.getMetricsSnapshot();
  const cacheHitRate = percent(m.cacheHits, m.cacheChecks);
  const cacheAcceptRate = percent(m.cacheAccepted, m.cacheHits);
  const fetchSuccessRate = percent(m.acceptedSources - m.cacheAccepted, m.fetchAttempts);
  const dedupeRate = percent(m.rawHits - m.dedupedHits, m.rawHits);
  
  const totalSourceChars = state.getMetricsSnapshot().acceptedSources * 4000;
  const estTokensSent = state.getMetricsSnapshot().fetchAttempts * 800;
  const evidenceYield = estTokensSent > 0 ? (m.acceptedSources / estTokensSent) * 100 : 0;

  status(`[${label}] search ddg_queries=${m.ddgQueries} ddg_hits=${m.ddgHits} mutation_accepted=${m.mutationAccepted} mutation_hits=${m.mutationHits} extra_hits=${m.extraEngineHits}`);
  status(`[${label}] quality raw_hits=${m.rawHits} deduped_hits=${m.dedupedHits} dedupe_rate=${dedupeRate}% ranked=${m.rankedCandidates} accepted=${m.acceptedSources} fetch_success=${fetchSuccessRate}%`);
  status(`[${label}] cache checks=${m.cacheChecks} hits=${m.cacheHits} hit_rate=${cacheHitRate}% accepted=${m.cacheAccepted} accept_rate=${cacheAcceptRate}% writes=${m.cacheWrites}`);
  status(`[${label}] skips visited=${m.skippedVisited} dup=${m.skippedDuplicateContent} off_topic=${m.skippedOffTopic} very_off_topic=${m.skippedVeryOffTopic} low_words=${m.skippedLowWordCount} domain_cap=${m.skippedDomainCap} avoided=${m.skippedAvoided} blacklisted=${m.skippedBlacklisted}`);
  status(`[${label}] 📊 DIAGNOSTICS evidence_yield=${evidenceYield.toFixed(2)}% (accepted_sources/est_llm_tokens)`);
}

function safeHostname(url: string): string {
  try {
    return new URL(url).hostname.replace(/^www\./, "");
  } catch {
    return "";
  }
}

function isAbortError(err: unknown): boolean {
  return err instanceof DOMException && err.name === "AbortError";
}

function aggregateResults(
  results: ReadonlyArray<WorkerResult>,
  sources: CrawledSource[],
  queries: string[],
  errors: string[],
): void {
  for (const result of results) {
    sources.push(...result.sources);
    queries.push(...result.queries);
    errors.push(...result.errors);
  }
}

function percent(part: number, total: number): number {
  if (total <= 0) return 0;
  return Math.round((part / total) * 100);
}

// ==================== FILE: src\synthesis\ai.ts ====================

// src/synthesis/ai.ts
import { LMStudioClient } from "@lmstudio/sdk";
import { ReportSource, ContradictionEntry, StatusFn, EvidenceCard, ContextBudget } from "../types";
import { logLlmDiagnostics, estimateTokens, preflightSynthesis } from "../utils/tokens";
import {
  DepthProfile,
  AI_SYNTHESIS_TEMPERATURE,
  AI_SYNTHESIS_TIMEOUT_MS,
  CONTRADICTION_SOURCE_CHARS,
  SYSTEM_INSTRUCTIONS,
} from "../constants";

function prepareEvidenceLedger(
  evidence: ReadonlyArray<EvidenceCard>,
  maxTokens: number
): string {
  const sorted = [...evidence].sort((a, b) => {
    const score = { High: 3, Medium: 2, Low: 1 };
    return score[b.confidence] - score[a.confidence];
  });

  const ledger: string[] = [];
  let currentTokens = 0;
  const tokenLimit = maxTokens - 1000;

  for (const card of sorted) {
    const entry = `[${card.id}] (${card.entityType}) ${card.title} - ${card.authorOrHost}
URL: ${card.canonicalUrl}
Tier: ${card.sourceTier} | Confidence: ${card.confidence} | Freshness: ${card.freshness ?? "Unknown"}
Claim: ${card.relevantClaim}
Excerpt: "${card.supportingExcerpt}"`;
    
    const entryTokens = estimateTokens(entry);
    if (currentTokens + entryTokens > tokenLimit) break;

    ledger.push(entry);
    currentTokens += entryTokens;
  }

  return ledger.join("\n\n---\n\n");
}

export async function synthesiseReport(
  topic: string,
  evidence: ReadonlyArray<EvidenceCard>,
  coveredClaims: ReadonlyArray<string>,
  gapClaims: ReadonlyArray<string>,
  status: StatusFn,
  budget: ContextBudget,
): Promise<string | null> {
  if (evidence.length === 0) return null;

  status(`AI synthesis - preparing evidence ledger (${evidence.length} cards, budget: ${budget.maxSynthesisInput} tokens)…`);

      const evidenceBlock = prepareEvidenceLedger(evidence, budget.maxSynthesisInput);
  const prompt = `You are an expert research analyst. Write a comprehensive, well-structured narrative synthesis of these research findings.

TOPIC: "${topic}"
CLAIMS COVERED: ${coveredClaims.join(", ")}

EVIDENCE LEDGER (Tier A & B only):
 ${evidenceBlock}

STRICT OUTPUT RULES:
1. You MUST start with a Markdown table of recommended sources with these exact columns:
   | Recommendation | Why it fits | Official verification | Independent validation | Status |
2. Only include sources with Verification Tier A or Tier B in the main recommendations.
3. Place Tier C sources in a separate "Other Candidates" appendix at the bottom.
4. Do NOT make generic claims like "All sources are authoritative" or "97% confidence".
5. Only make claims explicitly supported by the Evidence Ledger records.
6. If entity metadata (ISBN, host, edition) is missing, state "Metadata incomplete" rather than guessing.
7. For books, prefer current editions. Label older foundational titles as "foundational".
8. For podcasts, require a current official page or episode within the last 12 months. Label inactive podcasts as "inactive".


SYNTHESIS:`;

  const promptTokens = estimateTokens(prompt);
  const preflight = preflightSynthesis(promptTokens, budget);

  status(`[TOKEN PREFLIGHT] Projected: ${preflight.projectedTotal}/${budget.modelContextLimit} | Decision: ${preflight.decision}`);

  if (preflight.decision === "FAIL-SAFE") {
    status("SYNTHESIS_SKIPPED_CONTEXT_RISK");
    return null;
  }

  logLlmDiagnostics("synthesiseReport", prompt);

  try {
    const client = new LMStudioClient();
    const models = await Promise.race<Awaited<ReturnType<typeof client.llm.listLoaded>>>([
      client.llm.listLoaded(),
      new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout")), AI_SYNTHESIS_TIMEOUT_MS)),
    ]);

    if (!Array.isArray(models) || models.length === 0) return null;
    const model = await client.llm.model(models[0].identifier);

    const stream = model.respond(
      [
        { role: "system", content: SYSTEM_INSTRUCTIONS },
        { role: "user", content: prompt },
      ],
      {
        maxTokens: budget.outputReserve,
        temperature: AI_SYNTHESIS_TEMPERATURE,
      }
    );

    let result = "";
    for await (const chunk of stream) result += chunk.content ?? "";

    if (result.length > 100) {
      status(`AI synthesis complete (${result.length} chars)`);
      return result;
    }
    return null;
  } catch (err) {
    status(`AI synthesis failed: ${err instanceof Error ? err.message : String(err)}`);
    return null;
  }
}

export async function detectContradictions(
  topic: string,
  sources: ReadonlyArray<ReportSource>,
  status: StatusFn,
  profile: DepthProfile,
): Promise<ReadonlyArray<ContradictionEntry>> {
  if (sources.length < 3) return [];

  status("Checking for cross-source contradictions...");

  const sourceBlock = sources.slice(0, profile.contradictionMaxSources).map((s) => {
    const preview = s.text.slice(0, CONTRADICTION_SOURCE_CHARS).replace(/\n+/g, " ").trim();
    return `[${s.index}] "${s.title}" - ${s.tier}\n${preview}`;
  }).join("\n\n");

  const maxContradictions = Math.min(10, Math.max(5, Math.floor(sources.length / 5)));

  const prompt = `You are a fact-checking analyst. Given these research sources on "${topic}", identify any CONTRADICTIONS.
SOURCES:
 ${sourceBlock}

For each contradiction, output ONE line:
CLAIM: <claim> | SOURCE_A: [<index>] <stance> | SOURCE_B: [<index>] <stance> | SEVERITY: <minor/moderate/major>

If none, output: NONE
Max ${maxContradictions} contradictions.
OUTPUT:`;

  try {
    const client = new LMStudioClient();
    const models = await client.llm.listLoaded();
    if (!models || models.length === 0) return [];
    const model = await client.llm.model(models[0].identifier);
    
    const stream = model.respond(
      [{ role: "user", content: prompt }],
      { maxTokens: 1500, temperature: 0.15 }
    );

    let raw = "";
    for await (const chunk of stream) raw += chunk.content ?? "";

    if (!raw || /^NONE$/im.test(raw.trim())) return [];

    const entries: ContradictionEntry[] = [];
    for (const line of raw.split("\n")) {
      const trimmed = line.trim();
      if (!trimmed.startsWith("CLAIM:")) continue;
      try {
        const claimMatch = /CLAIM:\s*(.+?)\s*\|/.exec(trimmed);
        const sourceAMatch = /SOURCE_A:\s*\[(\d+)\]\s*(.+?)\s*\|/.exec(trimmed);
        const sourceBMatch = /SOURCE_B:\s*\[(\d+)\]\s*(.+?)\s*\|/.exec(trimmed);
        const sevMatch = /SEVERITY:\s*(minor|moderate|major)/i.exec(trimmed);
        if (!claimMatch || !sourceAMatch || !sourceBMatch) continue;

        const idxA = parseInt(sourceAMatch[1], 10);
        const idxB = parseInt(sourceBMatch[1], 10);
        entries.push({
          claim: claimMatch[1].trim(),
          sourceA: { index: idxA, title: sources.find(s => s.index === idxA)?.title ?? `Source ${idxA}`, stance: sourceAMatch[2].trim() },
          sourceB: { index: idxB, title: sources.find(s => s.index === idxB)?.title ?? `Source ${idxB}`, stance: sourceBMatch[2].trim() },
          severity: (sevMatch?.[1]?.toLowerCase() ?? "minor") as "minor" | "moderate" | "major",
        });
      } catch { continue; }
    }
    return entries;
  } catch {
    return [];
  }
}

// ==================== FILE: src\config.ts ====================

// src/config.ts
import { createConfigSchematics } from "@lmstudio/sdk";

export const configSchematics = createConfigSchematics()
  .field(
    "timeRange",
    "select",
    {
      displayName: "Search Time Range",
      subtitle: "Filter search results by publication date to ensure information freshness.",
      options: [
        { value: "all", displayName: "All time (default)" },
        { value: "year", displayName: "Past year" },
        { value: "month", displayName: "Past month" },
        { value: "week", displayName: "Past week" },
        { value: "day", displayName: "Past 24 hours" },
      ],
    },
    "all",
  )
  .field(
    "researchDepth",
    "select",
    {
      displayName: "Research Depth",
      subtitle:
        "Controls rounds, per-worker budgets, queries, and link-following aggressiveness. " +
        "Sources are collected adaptively with no hard cap — deeper = more sources.",
      options: [
        { value: "shallow", displayName: "Shallow — 1 round, ~10-25 sources, fast" },
        { value: "standard", displayName: "Standard — 3 rounds, ~30-60 sources (recommended)" },
        { value: "deep", displayName: "Deep — 5 rounds, ~60-120 sources, thorough" },
        { value: "deeper", displayName: "Deeper — 10 rounds, ~100-200+ sources, very thorough" },
        { value: "exhaustive", displayName: "Exhaustive — 15 rounds, 200+ sources, maximum depth" },
      ],
    },
    "standard",
  )
  .field(
    "engineSelectionMode",
    "select",
    {
      displayName: "Engine Selection Mode",
      subtitle:
        "Controls how search engines are chosen per worker. " +
        "Adaptive: DDG first, fallbacks if weak. " +
        "Benchmark: Run all engines to test performance. " +
        "Priority: Use best engines from historical data.",
      options: [
        { value: "adaptive", displayName: "Adaptive (Default)" },
        { value: "benchmark", displayName: "Benchmark (Run all)" },
        { value: "priority", displayName: "Priority (Historical)" },
      ],
    },
    "adaptive",
  )
  .field(
    "cacheDuration",
    "select",
    {
      displayName: "Persistent Cache Duration",
      subtitle: "How long visited pages and extracted content are kept in the local cache.",
      options: [
        { value: "30", displayName: "30 Days" },
        { value: "90", displayName: "90 Days" },
        { value: "180", displayName: "6 Months" },
        { value: "365", displayName: "12 Months" },
        { value: "730", displayName: "24 Months" },
      ],
    },
    "30",
  )
  .field(
    "contentLimitPerPage",
    "numeric",
    {
      displayName: "Content Per Page (chars)",
      subtitle:
        "Characters extracted per page. Higher = richer but slower. " +
        "Leave at default to auto-scale with depth preset (1000-20000)",
      min: 1000,
      max: 20000,
      int: true,
      slider: { step: 1000, min: 1000, max: 20000 },
    },
    4000,
  )
  .field(
    "enableLinkFollowing",
    "select",
    {
      displayName: "Link Following",
      subtitle: "Workers follow relevant in-page links (like citations and references)",
      options: [
        { value: "on", displayName: "On — follow top links (recommended)" },
        { value: "off", displayName: "Off — search results only" },
      ],
    },
    "on",
  )
  .field(
    "enableAIPlanning",
    "select",
    {
      displayName: "AI Query Planning",
      subtitle: "Use the loaded model for smarter queries, dynamic decomposition, and synthesis",
      options: [
        { value: "on", displayName: "On — AI-powered (best quality)" },
        { value: "off", displayName: "Off — dimension-based fallback (faster start)" },
      ],
    },
    "on",
  )
  .field(
    "safeSearch",
    "select",
    {
      displayName: "Safe Search",
      options: [
        { value: "strict", displayName: "Strict" },
        { value: "moderate", displayName: "Moderate" },
        { value: "off", displayName: "Off" },
      ],
    },
    "moderate",
  )
  .field(
    "enableLocalSources",
    "select",
    {
      displayName: "Data Sources",
      subtitle: "Choose where the research swarm should pull information from.",
      options: [
        { value: "off", displayName: "Web only" },
        { value: "local", displayName: "Local documents only" },
        { value: "web_local", displayName: "Local documents and web" },
      ],
    },
    "off",
  )
  .field(
    "maxSessionMinutes",
    "numeric",
    {
      displayName: "Max Session Time (minutes)",
      subtitle:
        "Hard cap on wall-clock time for Deep Research runs. " +
        "Set to 0 for Unlimited (runs until exhausted or stagnates).",
      min: 0,
      max: 240,
      int: true,
      slider: { step: 5, min: 0, max: 240 },
    },
    30,
  )
  .field(
    "enableAcademicAPIs",
    "select",
    {
      displayName: "Academic APIs (OpenAlex, Crossref, arXiv)",
      subtitle: "Query academic databases directly for papers and research.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "off",
  )
  .field(
    "enableYouTube",
    "select",
    {
      displayName: "YouTube Transcript Search",
      subtitle: "Search YouTube and extract video transcripts as text sources.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "off",
  )
  .field(
    "enableReferenceSearch",
    "select",
    {
      displayName: "Encyclopedia Search",
      subtitle: "Prioritize Grokipedia, Encyclopedia.com, and Britannica over Wikipedia.",
      options: [
        { value: "on", displayName: "On" },
        { value: "off", displayName: "Off" },
      ],
    },
    "on",
  )
    .field(
    "contextBudgetMode",
    "select",
    {
      displayName: "Context Budget Mode",
      subtitle: "Controls how the plugin manages token limits for synthesis. Auto is recommended.",
      options: [
        { value: "auto", displayName: "Auto (Recommended)" },
        { value: "conservative", displayName: "Conservative" },
        { value: "manual", displayName: "Manual Override" },
      ],
    },
    "auto",
  )
  .field(
    "manualContextLimit",
    "numeric",
    {
      displayName: "Manual Context Limit (Tokens)",
      subtitle: "Only used if Mode is Manual. E.g., 8192, 16384, 32768.",
      min: 2048,
      max: 131072,
      int: true,
    },
    8192,
  )
  .field(
    "maxSynthesisInputTokens",
    "numeric",
    {
      displayName: "Max Synthesis Input Tokens",
      subtitle: "Hard cap on tokens sent to the model for final report generation.",
      min: 2000,
      max: 32000,
      int: true,
    },
    18000,
  )
    .field(
    "contextIsolation",
    "select",
    {
      displayName: "LLM Context Isolation",
      subtitle: "Controls how model context is managed. Strict isolation prevents context overflow and history bleed.",
      options: [
        { value: "strict", displayName: "Strict isolation (Recommended)" },
        { value: "worker_reuse", displayName: "Reuse within one worker only" },
        { value: "advanced_reuse", displayName: "Advanced reuse (Not recommended)" },
      ],
    },
    "strict",
  )
  .field(
    "llmCallMode",
    "select",
    {
      displayName: "LLM Call Budget",
      subtitle: "Hard limit on model calls to prevent infinite loops. Standard is highly recommended.",
      options: [
        { value: "compact", displayName: "Compact (Max 20 calls)" },
        { value: "standard", displayName: "Standard (Max 45 calls)" },
        { value: "deep", displayName: "Deep (Max 80 calls)" },
        { value: "extended", displayName: "Extended (Max 120 calls)" },
      ],
    },
    "standard",
  )
  // Add this right BEFORE the final .build()
  .field(
    "flaresolverrUrl",
    "string",
    {
      displayName: "FlareSolverr URL (Advanced)",
      subtitle: "Optional: Local endpoint to bypass strict Cloudflare blocks (e.g., http://127.0.0.1:8191/v1). Leave blank to disable.",
    },
    ""
  )  
  .build();

// ==================== FILE: src\net\ddg.ts ====================

import { fetchPage } from "./http";

export class DdgRateLimiter {
  private lastRequest = 0;
  private minDelay: number;

  constructor(minDelayMs: number = 2500) {
    this.minDelay = minDelayMs;
  }

  async acquire(): Promise<void> {
    const now = Date.now();
    const elapsed = now - this.lastRequest;
    if (elapsed < this.minDelay) {
      await new Promise((resolve) => setTimeout(resolve, this.minDelay - elapsed));
    }
    this.lastRequest = Date.now();
  }
}

export const sharedDdgLimiter = new DdgRateLimiter(2500);

export class DdgLimiterPool {
  private limiters: DdgRateLimiter[] = [];
  private currentIndex = 0;

  constructor(numLanes: number, minDelayMs: number = 2500) {
    for (let i = 0; i < numLanes; i++) {
      this.limiters.push(new DdgRateLimiter(minDelayMs));
    }
  }

  next(): DdgRateLimiter {
    const limiter = this.limiters[this.currentIndex];
    this.currentIndex = (this.currentIndex + 1) % this.limiters.length;
    return limiter;
  }
}

export function resetThrottle(): void {}

/**
 * Bulletproof multi-tier DDG Search Waterfall:
 * Tier A: DDG Lite POST endpoint (fastest, mimics ddgr)
 * Tier B: DDG HTML fallback endpoint (handles strict blocks)
 * Tier C: Graceful recovery (returns empty array instead of throwing crash errors)
 */
export async function searchDDG(
  query: string,
  maxResults: number,
  safeSearch: "strict" | "moderate" | "off" = "moderate",
  signal?: AbortSignal,
  limiter?: DdgRateLimiter,
  timeRange: string = "all"
): Promise<ReadonlyArray<{ url: string; title: string; snippet: string }>> {
  if (limiter) await limiter.acquire();
  if (signal?.aborted) throw new DOMException("Aborted", "AbortError");

  const safeParam = safeSearch === "strict" ? "1" : safeSearch === "off" ? "-1" : "0";
  const dfParam = timeRange === "all" ? "" : `&df=${timeRange}`;
  const formData = `q=${encodeURIComponent(query)}&kp=${safeParam}${dfParam}`;

  // TIER A: DDG Lite Endpoint
  try {
    const res = await fetchPage("https://lite.duckduckgo.com/lite/", signal!, {
      method: "POST",
      body: formData,
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
        "Referer": "https://lite.duckduckgo.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
      }
    });
    const hits = parseDDGResults(res.html, maxResults);
    if (hits.length > 0) return hits;
  } catch (err) {
    if (signal?.aborted) throw err;
    console.warn(`[DDG Tier A] Failed for query "${query}": ${err instanceof Error ? err.message : String(err)}. Trying Tier B...`);
  }

  // TIER B: DDG HTML Endpoint Fallback
  try {
    if (limiter) await limiter.acquire();
    const htmlFallbackUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
    const resB = await fetchPage(htmlFallbackUrl, signal!, {
      method: "GET",
      headers: {
        "Referer": "https://html.duckduckgo.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
      }
    });
    const hitsB = parseHTMLDDGResults(resB.html, maxResults);
    if (hitsB.length > 0) return hitsB;
  } catch (errB) {
    if (signal?.aborted) throw errB;
    console.warn(`[DDG Tier B] Fallback also failed for query "${query}": ${errB instanceof Error ? errB.message : String(errB)}`);
  }

  // TIER C: Graceful exit (Returns empty array so the worker's outer health system handles it cleanly without crashing)
  return [];
}

export async function searchDDGPaginated(
  query: string,
  maxResultsPerPage: number,
  pages: number,
  safeSearch: "strict" | "moderate" | "off" = "moderate",
  signal?: AbortSignal,
  limiter?: DdgRateLimiter,
  timeRange: string = "all"
): Promise<ReadonlyArray<{ url: string; title: string; snippet: string }>> {
  const allHits: { url: string; title: string; snippet: string }[] = [];

  for (let p = 1; p <= pages; p++) {
    if (signal?.aborted) break;
    
    const safeParam = safeSearch === "strict" ? "1" : safeSearch === "off" ? "-1" : "0";
    const dfParam = timeRange === "all" ? "" : `&df=${timeRange}`;
    const formData = `q=${encodeURIComponent(query)}&kp=${safeParam}${dfParam}&s=${(p - 1) * maxResultsPerPage}`;
    
    try {
      if (limiter) await limiter.acquire();
      const res = await fetchPage("https://lite.duckduckgo.com/lite/", signal!, {
        method: "POST",
        body: formData,
        headers: {
          "Content-Type": "application/x-www-form-urlencoded",
          "Referer": "https://lite.duckduckgo.com/"
        }
      });
      const hits = parseDDGResults(res.html, maxResultsPerPage);
      allHits.push(...hits);
      if (hits.length < maxResultsPerPage) break;
    } catch {
      break;
    }
  }
  return allHits;
}

function parseDDGResults(html: string, maxResults: number): { url: string; title: string; snippet: string }[] {
  const hits: { url: string; title: string; snippet: string }[] = [];
  const seen = new Set<string>();
  
  // Resilient regex pattern matching DDG Lite result rows
  const resultRe = /<a[^>]*rel="nofollow"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<td class="result-snippet">([\s\S]*?)<\/td>/gi;
  let match: RegExpExecArray | null;
  
  while (hits.length < maxResults && (match = resultRe.exec(html)) !== null) {
    let url = match[1];
    const title = match[2].replace(/<[^>]+>/g, "").trim();
    const snippet = match[3].replace(/<[^>]+>/g, "").trim();
    
    // Clean up redirect wrappers if present
    if (url.includes("uddg=")) {
      const matchUrl = url.match(/uddg=([^&]+)/);
      if (matchUrl) url = decodeURIComponent(matchUrl[1]);
    }

    if (url.includes("duckduckgo.com")) continue;
    if (!url.startsWith("http")) continue;
    
    if (seen.has(url)) continue;
    seen.add(url);
    
    hits.push({ url, title, snippet });  
  }
  
  return hits;
}

function parseHTMLDDGResults(html: string, maxResults: number): { url: string; title: string; snippet: string }[] {
  const hits: { url: string; title: string; snippet: string }[] = [];
  const seen = new Set<string>();
  
  // Secondary parser for standard DDG HTML results page layout
  const resultRe = /<a class="result__url" href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
  let match: RegExpExecArray | null;
  
  while (hits.length < maxResults && (match = resultRe.exec(html)) !== null) {
    let url = match[1];
    const title = match[2].replace(/<[^>]+>/g, "").trim();
    const snippet = match[3].replace(/<[^>]+>/g, "").trim();

    if (url.includes("duckduckgo.com")) continue;
    if (!url.startsWith("http") && url.startsWith("//")) url = "https:" + url;
    if (!url.startsWith("http")) continue;
    
    if (seen.has(url)) continue;
    seen.add(url);
    
    hits.push({ url, title, snippet });  
  }
  
  return hits;
}

// ==================== FILE: src\swarm\orchestrator.ts ====================

import { runWorker, CrawlMetrics } from "./worker";
import {
  buildQueryPlan,
  buildAdaptiveGapFill,
  summariseFindings,
} from "../planning/planner";
import { detectCoveredDimensions, DIMENSIONS, detectGaps } from "../planning/dimensions";
import {
  ResearchConfig,
  SwarmTask,
  WorkerResult,
  CrawledSource,
  WorkerRole,
  AgentMessage,
  StatusFn,
  WarnFn,
  SourceTier,
  ContradictionEntry,
} from "../types";
import { DepthProfile } from "../constants";
import { DdgLimiterPool, resetThrottle } from "../net/ddg";
import { VisitedPageCache, normalizeUrl } from "./visited-cache";
import { log } from "./logger";
import { detectContradictions } from "../synthesis/ai";
import { SearchHealthTracker } from "./health";
import { LlmCallManager } from "../utils/llm";
import fs from "node:fs";
import path from "node:path";
import os from "node:os";

type GapPlanLike = {
  readonly role: WorkerRole;
  readonly label: string;
  readonly queries: ReadonlyArray<string>;
  readonly followLinks: boolean;
  readonly preferredTiers?: ReadonlyArray<SourceTier>;
};

function zeroMetrics(): CrawlMetrics {
  return {
    ddgQueries: 0, ddgHits: 0, mutatedQueriesTried: 0, mutationAccepted: 0,
    mutationHits: 0, extraEngineQueries: 0, extraEngineHits: 0, rawHits: 0,
    dedupedHits: 0, rankedCandidates: 0, fetchCandidates: 0, fetchAttempts: 0,
    fetchFailures: 0, acceptedSources: 0, skippedLowWordCount: 0, skippedOffTopic: 0,
    skippedVeryOffTopic: 0, skippedDuplicateContent: 0, skippedVisited: 0,
    skippedDomainCap: 0, skippedAvoided: 0, skippedBlacklisted: 0, cacheChecks: 0,
    cacheHits: 0, cacheAccepted: 0, cacheRejectedDuplicate: 0, cacheRejectedOffTopic: 0,
    cacheRejectedLowWordCount: 0, cacheWrites: 0, followedLinks: 0,
    crossWorkerDiscoveriesUsed: 0, localSourcesAccepted: 0,
  };
}

function mergeMetricObjects(base: CrawlMetrics, delta: Partial<CrawlMetrics>): CrawlMetrics {
  const next = { ...base };
  for (const key of Object.keys(next) as Array<keyof CrawlMetrics>) {
    next[key] = (next[key] ?? 0) + (delta[key] ?? 0);
  }
  return next;
}

function formatProgressBar(current: number, total: number, width: number = 20): string {
  if (total <= 0) return `[${"░".repeat(width)}] 0%`;
  const pct = Math.max(0, Math.min(100, Math.round((current / total) * 100)));
  const filled = Math.round((pct / 100) * width);
  return `[${"█".repeat(filled)}${"░".repeat(width - filled)}] ${pct}%`;
}

class MutableCrawlState implements SharedCrawlState {
  private readonly _visitedUrls = new Set<string>();
  private readonly _contentHashes = new Set<string>();
  private readonly _domainCounts = new Map<string, number>();
  private readonly _domainFailures = new Map<string, number>();
  private readonly _blacklistedDomains = new Set<string>();
  private readonly _discoveries: Array<{ url: string; title: string; fromWorker: string }> = [];
  private readonly _failedUrls = new Map<string, { count: number; reason: string; lastFailedAt: string }>();
  private readonly _failedHosts = new Map<string, { count: number; reason: string; lastFailedAt: string }>();
  private readonly visitedCache: VisitedPageCache;
  private _metrics: CrawlMetrics = zeroMetrics();
  
  addWebCacheDocument(source: CrawledSource): void {
    // Placeholder for local RAG store
  }
  
  constructor(cacheDurationDays: number = 30) {
    this.visitedCache = new VisitedPageCache(cacheDurationDays);
  }

  get visitedUrls(): ReadonlySet<string> { return this._visitedUrls; }
  get contentHashes(): ReadonlySet<string> { return this._contentHashes; }
  get domainCounts(): ReadonlyMap<string, number> { return this._domainCounts; }
  get domainFailures(): ReadonlyMap<string, number> { return this._domainFailures; }

  addVisited(url: string): void { this._visitedUrls.add(normalizeUrl(url)); }
  addHash(hash: string): void { this._contentHashes.add(hash); }
  incrementDomain(url: string): void {
    const host = safeHostname(url);
    if (!host) return;
    this._domainCounts.set(host, (this._domainCounts.get(host) ?? 0) + 1);
  }
  domainCount(url: string): number {
    const host = safeHostname(url);
    return host ? (this._domainCounts.get(host) ?? 0) : 0;
  }

  noteFailure(url: string, reason: string, at: string = new Date().toISOString()): void {
    const normalized = normalizeUrl(url);
    const prevUrl = this._failedUrls.get(normalized);
    this._failedUrls.set(normalized, { count: (prevUrl?.count ?? 0) + 1, reason, lastFailedAt: at });
    const host = safeHostname(normalized);
    if (!host) return;
    const prevHost = this._failedHosts.get(host);
    this._failedHosts.set(host, { count: (prevHost?.count ?? 0) + 1, reason, lastFailedAt: at });
  }

  noteDomainFailure(url: string): void {
    const host = safeHostname(url);
    if (!host) return;
    const count = (this._domainFailures.get(host) ?? 0) + 1;
    this._domainFailures.set(host, count);
    if (count >= 3) this._blacklistedDomains.add(host);
  }

  isDomainBlacklisted(url: string): boolean {
    const host = safeHostname(url);
    return host ? this._blacklistedDomains.has(host) : false;
  }

  shouldAvoidUrl(url: string): boolean {
    const normalized = normalizeUrl(url);
    if (this._failedUrls.has(normalized)) return true;
    const host = safeHostname(normalized);
    if (!host) return false;
    const hostFailCount = this._failedHosts.get(host)?.count ?? 0;
    return hostFailCount >= 2;
  }

  pushDiscovery(url: string, title: string, fromWorker: string): void {
    const normalized = normalizeUrl(url);
    if (!this._visitedUrls.has(normalized)) {
      this._discoveries.push({ url: normalized, title, fromWorker });
    }
  }

  drainDiscoveries(limit: number): ReadonlyArray<{ url: string; title: string }> {
    const results: Array<{ url: string; title: string }> = [];
    while (results.length < limit && this._discoveries.length > 0) {
      const item = this._discoveries.shift()!;
      if (!this._visitedUrls.has(normalizeUrl(item.url))) {
        results.push({ url: item.url, title: item.title });
      }
    }
    return results;
  }

  isRecentlyVisited(url: string): boolean { return this.visitedCache.hasRecent(url); }
  getCachedSource(url: string): CrawledSource | null { return this.visitedCache.getRecent(url); }
  markVisitedPersistent(source: CrawledSource): void { this.visitedCache.markVisited(source); }
  pruneVisitedCache(): void { this.visitedCache.prune(); }
  cacheStats(): { entries: number; file: string; maxAgeDays: number } { return this.visitedCache.stats(); }
  getMetricsSnapshot(): Readonly<CrawlMetrics> { return this._metrics; }
  mergeMetrics(delta: Partial<CrawlMetrics>): void { this._metrics = mergeMetricObjects(this._metrics, delta); }
}

const CORE_ROLES: ReadonlyArray<WorkerRole> = ["breadth", "depth", "recency", "academic", "critical"];
const EXTENDED_ROLES: ReadonlyArray<WorkerRole> = ["statistical", "regulatory", "technical", "primary", "comparative"];
const ROLE_LABELS: Readonly<Record<WorkerRole, string>> = {
  breadth: "Breadth", depth: "Depth", recency: "Recency", academic: "Academic", critical: "Critical",
  statistical: "Statistical/Data", regulatory: "Regulatory/Policy", technical: "Technical Deep-Dive",
  primary: "Primary Sources", comparative: "Comparative Analysis",
};

function getEnginesForRole(
  role: WorkerRole,
  cfg: ResearchConfig,
  mode: "adaptive" | "benchmark" | "priority"
): ReadonlyArray<string> {
  const freeEngines: string[] = ["ddg", "google", "brave"];
  if (cfg.enableReferenceSearch) freeEngines.push("reference");
  if (cfg.enableAcademicAPIs) freeEngines.push("openalex", "crossref", "arxiv");
  if (cfg.enableYouTube) freeEngines.push("youtube");
  freeEngines.push("gdelt");

  if (mode === "benchmark") {
    return [...freeEngines, cfg.serperApiKey ? "serper" : "", cfg.braveApiKey ? "brave-api" : ""].filter(Boolean);
  }

  const roleEngines: string[] = [];
  if (cfg.enableReferenceSearch && (role === "breadth" || role === "academic")) roleEngines.push("reference");
  if (cfg.enableAcademicAPIs && (role === "academic" || role === "technical")) roleEngines.push("openalex", "crossref", "arxiv");
  if (role === "recency" || role === "critical") roleEngines.push("gdelt");

  if (roleEngines.length === 0) {
    roleEngines.push("ddg", "google", "brave");
  } else {
    roleEngines.push("ddg", "brave");
  }

  return roleEngines;
}

function rolesForProfile(profile: DepthProfile): ReadonlyArray<WorkerRole> {
  if (profile.depthRounds >= 10) return [...CORE_ROLES, ...EXTENDED_ROLES];
  if (profile.depthRounds >= 5) return [...CORE_ROLES, "technical", "comparative", "statistical"];
  return [...CORE_ROLES];
}

function buildTaskBase(
  profile: DepthProfile,
  cfg: ResearchConfig,
): Pick<
  SwarmTask,
  | "contentLimit"
  | "safeSearch"
  | "searchResultsPerQuery"
  | "maxPagesPerDomain"
  | "maxLinksToEvaluate"
  | "maxLinksToFollow"
  | "candidatePoolMultiplier"
  | "workerConcurrency"
  | "minRelevanceScore"
  | "maxOutlinksPerPage"
  | "searchPages"
  | "extraEngines"
  | "linkCrawlDepth"
  | "queryMutationThreshold"
  | "enableLocalSources"
  | "localLibraryIds"
  | "timeRange"
  | "roleLibraryMap"
  | "serperApiKey"
  | "braveApiKey"
  | "enableYouTube"
> & { flaresolverrUrl?: string } { // <--- Added type extension here to prevent TypeScript errors
  return {
    contentLimit: cfg.contentLimitPerPage,
    safeSearch: cfg.safeSearch,
    searchResultsPerQuery: profile.searchResultsPerQuery,
    maxPagesPerDomain: profile.maxPagesPerDomain,
    maxLinksToEvaluate: profile.maxLinksToEvaluate,
    maxLinksToFollow: profile.maxLinksToFollow,
    candidatePoolMultiplier: profile.candidatePoolMultiplier,
    workerConcurrency: profile.workerConcurrency,
    minRelevanceScore: profile.minRelevanceScore,
    maxOutlinksPerPage: profile.maxOutlinksPerPage,
    searchPages: profile.searchPages,
    extraEngines: getEnginesForRole("breadth", cfg, cfg.engineSelectionMode ?? "adaptive"),
    linkCrawlDepth: profile.linkCrawlDepth,
    queryMutationThreshold: profile.queryMutationThreshold,
    enableLocalSources: cfg.enableLocalSources,
    localLibraryIds: cfg.localLibraryIds,
    timeRange: cfg.timeRange,
    roleLibraryMap: cfg.roleLibraryMap,
    serperApiKey: cfg.serperApiKey,
    braveApiKey: cfg.braveApiKey,
    enableYouTube: cfg.enableYouTube,
    flaresolverrUrl: (cfg as any).flaresolverrUrl, // <--- THIS grabs the URL from the UI!
  };
}

function buildStaticTask(
  role: WorkerRole,
  queries: ReadonlyArray<string>,
  profile: DepthProfile,
  cfg: ResearchConfig,
  subIdx: number = 0,
): SwarmTask {
  const followRoles: ReadonlyArray<WorkerRole> = ["depth", "academic", "technical", "primary"];
  const academicTiers: ReadonlyArray<SourceTier> = ["academic", "government", "reference"];
  const enginesForRole = getEnginesForRole(role, cfg, cfg.engineSelectionMode ?? "adaptive");

  return {
    ...buildTaskBase(profile, cfg),
    extraEngines: enginesForRole,
    id: `${role}-s${subIdx}-${Date.now()}`,
    role,
    label: subIdx > 0 ? `${ROLE_LABELS[role]} #${subIdx + 1}` : ROLE_LABELS[role],
    queries: [...queries],
    pageBudget: profile.pageBudgetPerWorker,
    followLinks: cfg.enableLinkFollowing && followRoles.includes(role),
    preferredTiers: role === "academic" || role === "regulatory" ? academicTiers : undefined,
  };
}

function fanOutQueries(
  queries: ReadonlyArray<string>,
  fanOut: number,
): ReadonlyArray<ReadonlyArray<string>> {
  if (fanOut <= 1 || queries.length <= 2) return [queries];
  const groups: string[][] = Array.from({ length: fanOut }, () => []);
  for (let i = 0; i < queries.length; i++) {
    groups[i % fanOut].push(queries[i]);
  }
  return groups.filter((g) => g.length > 0);
}

function buildRound1Tasks(
  roles: ReadonlyArray<WorkerRole>,
  queriesByRole: Partial<Record<WorkerRole, ReadonlyArray<string>>>,
  profile: DepthProfile,
  cfg: ResearchConfig,
): SwarmTask[] {
  const allQueries = Array.from(
    new Set(
      Object.values(queriesByRole)
        .flatMap((qs) => qs ?? [])
        .filter((q): q is string => Boolean(q)),
    ),
  );
  const round1Tasks: SwarmTask[] = [];
  for (const role of roles) {
    const roleQueries = (queriesByRole[role] ?? []).filter(Boolean);
    const effectiveQueries = roleQueries.length > 0 ? roleQueries : allQueries;
    const groups = fanOutQueries(effectiveQueries, profile.workerFanOut);
    for (const [subIdx, group] of groups.entries()) {
      if (group.length > 0) {
        round1Tasks.push(buildStaticTask(role, group, profile, cfg, subIdx));
      }
    }
  }
  return round1Tasks;
}

function buildGapTasks(
  gapPlans: ReadonlyArray<GapPlanLike>,
  round: number,
  profile: DepthProfile,
  cfg: ResearchConfig,
): SwarmTask[] {
  const tasks: SwarmTask[] = [];
  const gapEngines: string[] = [];
  if (cfg.serperApiKey) gapEngines.push("serper");
  if (cfg.braveApiKey) gapEngines.push("brave-api");
  if (gapEngines.length === 0) gapEngines.push("ddg", "google", "brave");

  for (const [subIdx, gapPlan] of gapPlans.entries()) {
    if (gapPlan.queries.length === 0) continue;
    tasks.push({
      ...buildTaskBase(profile, cfg),
      extraEngines: gapEngines,
      id: `gap-${gapPlan.role}-r${round}-s${subIdx}-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
      role: gapPlan.role,
      label: subIdx > 0 ? `${gapPlan.label} #${subIdx + 1}` : gapPlan.label,
      queries: [...gapPlan.queries],
      pageBudget: profile.pageBudgetPerGapWorker,
      followLinks: cfg.enableLinkFollowing && gapPlan.followLinks,
      preferredTiers: gapPlan.preferredTiers,
    });
  }
  return tasks;
}

async function runTaskGroup(
  tasks: ReadonlyArray<SwarmTask>,
  state: SharedCrawlState,
  pool: DdgLimiterPool,
  signal: AbortSignal,
  status: StatusFn,
  warn: WarnFn,
  topicKeywords: ReadonlyArray<string>,
  health: SearchHealthTracker,
  llmManager: LlmCallManager
): Promise<WorkerResult[]> {
  return Promise.all(
    tasks.map((task) => {
      const limiter = pool.next();
      return runWorker(
        task,
        state,
        signal,
        status,
        warn,
        topicKeywords,
        limiter,
        health,
        llmManager
      ).catch((err: unknown) => {
        if (!isAbortError(err)) {
          warn(`[${task.label}] crashed: ${err instanceof Error ? err.message : String(err)}`);
        }
        return {
          taskId: task.id,
          role: task.role,
          label: task.label,
          sources: [],
          queries: [],
          errors: [String(err)],
        } satisfies WorkerResult;
      });
    }),
  );
}

export interface OrchestratorResult {
  readonly sources: ReadonlyArray<CrawledSource>;
  readonly queriesUsed: ReadonlyArray<string>;
  readonly workerErrors: ReadonlyArray<string>;
  readonly usedAI: boolean;
  readonly topicKeywords: ReadonlyArray<string>;
}

export interface SharedCrawlState {
  readonly visitedUrls: ReadonlySet<string>;
  readonly contentHashes: ReadonlySet<string>;
  readonly domainCounts: ReadonlyMap<string, number>;
  readonly domainFailures: ReadonlyMap<string, number>;
  addVisited(url: string): void;
  addHash(hash: string): void;
  incrementDomain(url: string): void;
  domainCount(url: string): number;
  noteFailure(url: string, reason: string): void;
  noteDomainFailure(url: string): void;
  isDomainBlacklisted(url: string): boolean;
  shouldAvoidUrl(url: string): boolean;
  pushDiscovery(url: string, title: string, fromWorker: string): void;
  drainDiscoveries(limit: number): ReadonlyArray<{ url: string; title: string }>;
  isRecentlyVisited(url: string): boolean;
  getCachedSource(url: string): CrawledSource | null;
  markVisitedPersistent(source: CrawledSource): void;
  addWebCacheDocument(source: CrawledSource): void;
  pruneVisitedCache(): void;
  cacheStats(): { entries: number; file: string; maxAgeDays: number };
  getMetricsSnapshot(): Readonly<CrawlMetrics>;
  mergeMetrics(delta: Partial<CrawlMetrics>): void;
}

export async function runSwarm(
  cfg: ResearchConfig,
  profile: DepthProfile,
  status: StatusFn,
  warn: WarnFn,
  signal: AbortSignal,
): Promise<OrchestratorResult> {
  const fileStatus: StatusFn = (msg: string) => { log(msg); status(msg); };
  const fileWarn: WarnFn = (msg: string) => { log(`[WARN] ${msg}`); warn(msg); };

  const state = new MutableCrawlState(cfg.cacheDuration ? parseInt(cfg.cacheDuration, 10) : 30);
  state.pruneVisitedCache();
  const allSources: CrawledSource[] = [];
  const allQueries: string[] = [];
  const allErrors: string[] = [];
  
  const health = new SearchHealthTracker();
  const llmManager = new LlmCallManager(cfg.llmCallMode ?? "standard", cfg.contextIsolation ?? "strict");

  fileStatus(`\n🚀 DEEP RESEARCH SWARM LAUNCHED (Strict Priority Mode)\n`);
  fileStatus(`[RUN CONTROL] Budget: ${llmManager.budget.maxGlobalCalls} calls | Timeout: ${llmManager.budget.maxRuntimeMs / 60000}m | Isolation: ${llmManager.isolationMode}`);

  const plan = await buildQueryPlan(cfg.topic, cfg.focusAreas, cfg.enableAIPlanning, fileStatus, profile);
  const roles = rolesForProfile(profile);
  const pool = new DdgLimiterPool(profile.searchLanes, profile.ddgRateLimitMs);

  // ==========================================
  // LAYER 1: MANDATORY LOCAL SOURCE PASS
  // ==========================================
  if (cfg.enableLocalSources) {
    fileStatus(`\n📚 LAYER 1: Searching Local RAG Libraries first...`);
    const localTasks = buildRound1Tasks(roles, plan.queriesByRole, profile, cfg);
    const localOnlyTasks = localTasks.map(t => ({ ...t, extraEngines: [] as ReadonlyArray<string> }));
    const localResults = await runTaskGroup(localOnlyTasks, state, pool, signal, fileStatus, fileWarn, plan.topicKeywords, health, llmManager);
    aggregateResults(localResults, allSources, allQueries, allErrors);
    
    const localCount = allSources.length;
    health.localChunksRetrieved = localCount;
    health.localChunksAccepted = localCount;
    fileStatus(`✅ Layer 1 Complete. Found ${localCount} local sources.`);

    if (localCount >= profile.pageBudgetPerWorker * roles.length) {
      fileStatus(`Local evidence SUFFICIENT. Skipping external web search.`);
    }
  } else {
    fileStatus(`\n⚠️ Local sources disabled. Proceeding to web search.`);
  }

  // ==========================================
  // LAYERS 2-5: EXTERNAL GAP-DRIVEN SEARCH
  // ==========================================
  fileStatus(`\n🌐 LAYERS 2-5: External Gap-Driven Search...`);
  
  const coveredIds = detectCoveredDimensions(allSources.map(s => s.text));
  const gaps = detectGaps(coveredIds);
  
  for (const gap of gaps) {
    if (signal.aborted) break;
    health.gaps.push({
      id: `gap-${gap.id}`,
      topic: cfg.topic,
      missingClaim: gap.label,
      whyInsufficient: `Dimension ${gap.label} not covered by local sources`,
      freshnessRequired: gap.id === 'current' || gap.id === 'future',
      preferredTier: null,
      searchLayerAuthorized: "DDG",
      resolved: false
    });
  }

  let externalTasks = buildGapTasks(gaps.map(g => ({
    role: "breadth", 
    label: `Gap: ${g.label}`,
    queries: g.queries(cfg.topic),
    followLinks: true
  })) as ReadonlyArray<GapPlanLike>, 1, profile, cfg);

  const maxSessionTimeMs = cfg.maxSessionMs || 30 * 60 * 1000;
  const startTime = Date.now();
  const crawlDeadline = startTime + (maxSessionTimeMs * 0.80); // Reserve 20% for synthesis

  for (const layer of ["DDG", "SEARXNG", "DIRECT", "API"]) {
    if (Date.now() >= crawlDeadline || signal.aborted || externalTasks.length === 0) {
      fileWarn(`[TIME ALLOCATION] Crawl budget elapsed or tasks finished. Transitioning to verification & synthesis.`);
      break;
    }

    let layerEngines: string[] = [];
    if (layer === "DDG" && health.isDdgAvailable()) layerEngines = ["ddg"];
    else if (layer === "SEARXNG" && !health.isDdgAvailable()) layerEngines = ["searxng"];
    else if (layer === "DIRECT") layerEngines = ["reference", "gdelt"];
    else if (layer === "API" && health.canUseApi() && cfg.serperApiKey) layerEngines = ["serper"];

    if (layerEngines.length === 0) continue;

    fileStatus(`\n🔍 Executing Layer: ${layer} (${layerEngines.join(", ")})`);
    const layerTasks = externalTasks.map(t => ({ ...t, extraEngines: layerEngines }));
    const layerResults = await runTaskGroup(layerTasks, state, pool, signal, fileStatus, fileWarn, plan.topicKeywords, health, llmManager);
    
    const prevSourceCount = allSources.length;
    aggregateResults(layerResults, allSources, allQueries, allErrors);
    const newSources = allSources.length - prevSourceCount;

    if (newSources > 0) {
      externalTasks = [];
      health.gaps.forEach(g => { g.resolved = true; });
    }
    if (layer === "API") health.apiCallsMade++;
  }

  // Cap Sources to Top-N before logging and passing to synthesis
  const sortedSources = [...allSources].sort((a, b) => {
    const scoreA = (a.domainScore * 0.4) + (a.relevanceScore * 100 * 0.6);
    const scoreB = (b.domainScore * 0.4) + (b.relevanceScore * 100 * 0.6);
    return scoreB - scoreA;
  });

  // Reassign allSources to the filtered top results
  allSources.length = 0; 
  allSources.push(...sortedSources.slice(0, profile.synthesisMaxSources || 25));

  // ==========================================
  // FILE LOGGING
  // ==========================================
  const logDir = path.join(os.homedir(), ".deep-swarm-research", "logs");
  const logFile = path.join(logDir, `run_log_${Date.now()}.txt`);
  
  let logContent = `RUN ID: ${Date.now()}\nTOPIC: ${cfg.topic}\n\n`;
  logContent += health.generateReport();
  logContent += llmManager.getReport();
  logContent += "\n\nSEARCH ENGINE ATTRIBUTION\n";
  logContent += `- Requested Route: DDG / SearxNG / Direct\n`;
  logContent += `- Actual Backend: DDG / Yandex / Bing / Google\n`;
  logContent += `- Result Domains: Tracked in worker metrics\n`;
  logContent += `\nVERIFICATION TIERS\n`;
  logContent += `- Tier A (Canonically verified): ${allSources.filter((s: CrawledSource) => s.domainScore >= 90).length}\n`;
  logContent += `- Tier B (Independently validated): ${allSources.filter((s: CrawledSource) => s.domainScore >= 80 && s.domainScore < 90).length}\n`;
  logContent += `- Tier C (Relevant candidate): ${allSources.filter((s: CrawledSource) => s.domainScore < 80).length}\n`;

  try {
    fs.mkdirSync(logDir, { recursive: true });
    fs.writeFileSync(logFile, logContent, "utf-8");
    fileStatus(`[LOG] Full run report saved to ${logFile}`);
  } catch (e) {
    fileWarn(`[LOG] Failed to write run log: ${e}`);
  }

  // ==========================================
  // FINAL HEALTH REPORT
  // ==========================================
  fileStatus(health.generateReport());
  fileStatus(llmManager.getReport());

  return {
    sources: allSources,
    queriesUsed: [...new Set(allQueries)],
    workerErrors: allErrors,
    usedAI: plan.usedAI,
    topicKeywords: plan.topicKeywords,
  };
}

function logAggregateMetrics(
  state: SharedCrawlState,
  status: StatusFn,
  label: string,
): void {
  const m = state.getMetricsSnapshot();
  const cacheHitRate = percent(m.cacheHits, m.cacheChecks);
  const cacheAcceptRate = percent(m.cacheAccepted, m.cacheHits);
  const fetchSuccessRate = percent(m.acceptedSources - m.cacheAccepted, m.fetchAttempts);
  const dedupeRate = percent(m.rawHits - m.dedupedHits, m.rawHits);
  
  const totalSourceChars = state.getMetricsSnapshot().acceptedSources * 4000;
  const estTokensSent = state.getMetricsSnapshot().fetchAttempts * 800;
  const evidenceYield = estTokensSent > 0 ? (m.acceptedSources / estTokensSent) * 100 : 0;

  status(`[${label}] search ddg_queries=${m.ddgQueries} ddg_hits=${m.ddgHits} mutation_accepted=${m.mutationAccepted} mutation_hits=${m.mutationHits} extra_hits=${m.extraEngineHits}`);
  status(`[${label}] quality raw_hits=${m.rawHits} deduped_hits=${m.dedupedHits} dedupe_rate=${dedupeRate}% ranked=${m.rankedCandidates} accepted=${m.acceptedSources} fetch_success=${fetchSuccessRate}%`);
  status(`[${label}] cache checks=${m.cacheChecks} hits=${m.cacheHits} hit_rate=${cacheHitRate}% accepted=${m.cacheAccepted} accept_rate=${cacheAcceptRate}% writes=${m.cacheWrites}`);
  status(`[${label}] skips visited=${m.skippedVisited} dup=${m.skippedDuplicateContent} off_topic=${m.skippedOffTopic} very_off_topic=${m.skippedVeryOffTopic} low_words=${m.skippedLowWordCount} domain_cap=${m.skippedDomainCap} avoided=${m.skippedAvoided} blacklisted=${m.skippedBlacklisted}`);
  status(`[${label}] 📊 DIAGNOSTICS evidence_yield=${evidenceYield.toFixed(2)}% (accepted_sources/est_llm_tokens)`);
}

function safeHostname(url: string): string {
  try {
    return new URL(url).hostname.replace(/^www\./, "");
  } catch {
    return "";
  }
}

function isAbortError(err: unknown): boolean {
  return err instanceof DOMException && err.name === "AbortError";
}

function aggregateResults(
  results: ReadonlyArray<WorkerResult>,
  sources: CrawledSource[],
  queries: string[],
  errors: string[],
): void {
  for (const result of results) {
    sources.push(...result.sources);
    queries.push(...result.queries);
    errors.push(...result.errors);
  }
}

function percent(part: number, total: number): number {
  if (total <= 0) return 0;
  return Math.round((part / total) * 100);
}

// ==================== FILE: src\synthesis\ai.ts ====================

// src/synthesis/ai.ts
import { LMStudioClient } from "@lmstudio/sdk";
import { ReportSource, ContradictionEntry, StatusFn, EvidenceCard, ContextBudget } from "../types";
import { logLlmDiagnostics, estimateTokens, preflightSynthesis } from "../utils/tokens";
import {
  DepthProfile,
  AI_SYNTHESIS_TEMPERATURE,
  AI_SYNTHESIS_TIMEOUT_MS,
  CONTRADICTION_SOURCE_CHARS,
  SYSTEM_INSTRUCTIONS,
} from "../constants";

function prepareEvidenceLedger(
  evidence: ReadonlyArray<EvidenceCard>,
  maxTokens: number
): string {
  const sorted = [...evidence].sort((a, b) => {
    const score = { High: 3, Medium: 2, Low: 1 };
    return score[b.confidence] - score[a.confidence];
  });

  const ledger: string[] = [];
  let currentTokens = 0;
  const tokenLimit = maxTokens - 1000;

  for (const card of sorted) {
    const entry = `[${card.id}] (${card.entityType}) ${card.title} - ${card.authorOrHost}
URL: ${card.canonicalUrl}
Tier: ${card.sourceTier} | Confidence: ${card.confidence} | Freshness: ${card.freshness ?? "Unknown"}
Claim: ${card.relevantClaim}
Excerpt: "${card.supportingExcerpt}"`;
    
    const entryTokens = estimateTokens(entry);
    if (currentTokens + entryTokens > tokenLimit) break;

    ledger.push(entry);
    currentTokens += entryTokens;
  }

  return ledger.join("\n\n---\n\n");
}

export async function synthesiseReport(
  topic: string,
  evidence: ReadonlyArray<EvidenceCard>,
  coveredClaims: ReadonlyArray<string>,
  gapClaims: ReadonlyArray<string>,
  status: StatusFn,
  budget: ContextBudget,
): Promise<string | null> {
  if (evidence.length === 0) return null;

  status(`AI synthesis - preparing evidence ledger (${evidence.length} cards, budget: ${budget.maxSynthesisInput} tokens)…`);

      const evidenceBlock = prepareEvidenceLedger(evidence, budget.maxSynthesisInput);
  const prompt = `You are an expert research analyst. Write a comprehensive, well-structured narrative synthesis of these research findings.

TOPIC: "${topic}"
CLAIMS COVERED: ${coveredClaims.join(", ")}

EVIDENCE LEDGER (Tier A & B only):
 ${evidenceBlock}

STRICT OUTPUT RULES:
1. You MUST start with a Markdown table of recommended sources with these exact columns:
   | Recommendation | Why it fits | Official verification | Independent validation | Status |
2. Only include sources with Verification Tier A or Tier B in the main recommendations.
3. Place Tier C sources in a separate "Other Candidates" appendix at the bottom.
4. Do NOT make generic claims like "All sources are authoritative" or "97% confidence".
5. Only make claims explicitly supported by the Evidence Ledger records.
6. If entity metadata (ISBN, host, edition) is missing, state "Metadata incomplete" rather than guessing.
7. For books, prefer current editions. Label older foundational titles as "foundational".
8. For podcasts, require a current official page or episode within the last 12 months. Label inactive podcasts as "inactive".


SYNTHESIS:`;

  const promptTokens = estimateTokens(prompt);
  const preflight = preflightSynthesis(promptTokens, budget);

  status(`[TOKEN PREFLIGHT] Projected: ${preflight.projectedTotal}/${budget.modelContextLimit} | Decision: ${preflight.decision}`);

  if (preflight.decision === "FAIL-SAFE") {
    status("SYNTHESIS_SKIPPED_CONTEXT_RISK");
    return null;
  }

  logLlmDiagnostics("synthesiseReport", prompt);

  try {
    const client = new LMStudioClient();
    const models = await Promise.race<Awaited<ReturnType<typeof client.llm.listLoaded>>>([
      client.llm.listLoaded(),
      new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout")), AI_SYNTHESIS_TIMEOUT_MS)),
    ]);

    if (!Array.isArray(models) || models.length === 0) return null;
    const model = await client.llm.model(models[0].identifier);

    const stream = model.respond(
      [
        { role: "system", content: SYSTEM_INSTRUCTIONS },
        { role: "user", content: prompt },
      ],
      {
        maxTokens: budget.outputReserve,
        temperature: AI_SYNTHESIS_TEMPERATURE,
      }
    );

    let result = "";
    for await (const chunk of stream) result += chunk.content ?? "";

    if (result.length > 100) {
      status(`AI synthesis complete (${result.length} chars)`);
      return result;
    }
    return null;
  } catch (err) {
    status(`AI synthesis failed: ${err instanceof Error ? err.message : String(err)}`);
    return null;
  }
}

export async function detectContradictions(
  topic: string,
  sources: ReadonlyArray<ReportSource>,
  status: StatusFn,
  profile: DepthProfile,
): Promise<ReadonlyArray<ContradictionEntry>> {
  if (sources.length < 3) return [];

  status("Checking for cross-source contradictions...");

  const sourceBlock = sources.slice(0, profile.contradictionMaxSources).map((s) => {
    const preview = s.text.slice(0, CONTRADICTION_SOURCE_CHARS).replace(/\n+/g, " ").trim();
    return `[${s.index}] "${s.title}" - ${s.tier}\n${preview}`;
  }).join("\n\n");

  const maxContradictions = Math.min(10, Math.max(5, Math.floor(sources.length / 5)));

  const prompt = `You are a fact-checking analyst. Given these research sources on "${topic}", identify any CONTRADICTIONS.
SOURCES:
 ${sourceBlock}

For each contradiction, output ONE line:
CLAIM: <claim> | SOURCE_A: [<index>] <stance> | SOURCE_B: [<index>] <stance> | SEVERITY: <minor/moderate/major>

If none, output: NONE
Max ${maxContradictions} contradictions.
OUTPUT:`;

  try {
    const client = new LMStudioClient();
    const models = await client.llm.listLoaded();
    if (!models || models.length === 0) return [];
    const model = await client.llm.model(models[0].identifier);
    
    const stream = model.respond(
      [{ role: "user", content: prompt }],
      { maxTokens: 1500, temperature: 0.15 }
    );

    let raw = "";
    for await (const chunk of stream) raw += chunk.content ?? "";

    if (!raw || /^NONE$/im.test(raw.trim())) return [];

    const entries: ContradictionEntry[] = [];
    for (const line of raw.split("\n")) {
      const trimmed = line.trim();
      if (!trimmed.startsWith("CLAIM:")) continue;
      try {
        const claimMatch = /CLAIM:\s*(.+?)\s*\|/.exec(trimmed);
        const sourceAMatch = /SOURCE_A:\s*\[(\d+)\]\s*(.+?)\s*\|/.exec(trimmed);
        const sourceBMatch = /SOURCE_B:\s*\[(\d+)\]\s*(.+?)\s*\|/.exec(trimmed);
        const sevMatch = /SEVERITY:\s*(minor|moderate|major)/i.exec(trimmed);
        if (!claimMatch || !sourceAMatch || !sourceBMatch) continue;

        const idxA = parseInt(sourceAMatch[1], 10);
        const idxB = parseInt(sourceBMatch[1], 10);
        entries.push({
          claim: claimMatch[1].trim(),
          sourceA: { index: idxA, title: sources.find(s => s.index === idxA)?.title ?? `Source ${idxA}`, stance: sourceAMatch[2].trim() },
          sourceB: { index: idxB, title: sources.find(s => s.index === idxB)?.title ?? `Source ${idxB}`, stance: sourceBMatch[2].trim() },
          severity: (sevMatch?.[1]?.toLowerCase() ?? "minor") as "minor" | "moderate" | "major",
        });
      } catch { continue; }
    }
    return entries;
  } catch {
    return [];
  }
}