src / tools / evalTools.ts
src / tools / evalTools.ts
import fs from 'fs/promises';
import path from 'path';
import crypto from 'crypto';
import { getState } from '../stateRegistry';
const EVAL_FILE = '.timber_evaluations.json';
interface EvalEntry {
id: string;
agent_id: string;
task: string;
success: boolean;
latency_ms: number;
tools_used: string[];
timestamp: number;
}
let evaluations: EvalEntry[] = [];
async function loadEvals(): Promise<void> {
try {
const content = await fs.readFile(path.join(process.cwd(), EVAL_FILE), 'utf-8');
evaluations = JSON.parse(content);
} catch {}
}
export async function eval_record(agentId: string, task: string, success: boolean, latencyMs: number, toolsUsed: string[]): Promise<{ success: boolean }> {
await loadEvals();
const id = `eval_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
evaluations.push({ id, agent_id: agentId, task, success, latency_ms: latencyMs, tools_used: toolsUsed, timestamp: Date.now() });
await fs.writeFile(path.join(process.cwd(), EVAL_FILE), JSON.stringify(evaluations, null, 2));
getState().lastConfigChange = new Date();
return { success: true };
}
export async function eval_get_stats(): Promise<{ total_tasks: number; success_rate: number; avg_latency_ms: number }> {
await loadEvals();
const total = evaluations.length;
if (total === 0) return { total_tasks: 0, success_rate: 0, avg_latency_ms: 0 };
const successes = evaluations.filter(e => e.success).length;
const avgLatency = evaluations.reduce((sum, e) => sum + e.latency_ms, 0) / total;
return {
total_tasks: total,
success_rate: Math.round((successes / total) * 100),
avg_latency_ms: Math.round(avgLatency)
};
}
export async function eval_get_history(limit?: number): Promise<{ entries: EvalEntry[] }> {
await loadEvals();
const sorted = evaluations.sort((a, b) => b.timestamp - a.timestamp);
return { entries: limit ? sorted.slice(0, limit) : sorted };
}
export async function eval_clear(): Promise<{ success: boolean }> {
try {
await fs.unlink(path.join(process.cwd(), EVAL_FILE));
evaluations = [];
return { success: true };
} catch {
return { success: false };
}
}
import fs from 'fs/promises';
import path from 'path';
import crypto from 'crypto';
import { getState } from '../stateRegistry';
const EVAL_FILE = '.timber_evaluations.json';
interface EvalEntry {
id: string;
agent_id: string;
task: string;
success: boolean;
latency_ms: number;
tools_used: string[];
timestamp: number;
}
let evaluations: EvalEntry[] = [];
async function loadEvals(): Promise<void> {
try {
const content = await fs.readFile(path.join(process.cwd(), EVAL_FILE), 'utf-8');
evaluations = JSON.parse(content);
} catch {}
}
export async function eval_record(agentId: string, task: string, success: boolean, latencyMs: number, toolsUsed: string[]): Promise<{ success: boolean }> {
await loadEvals();
const id = `eval_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
evaluations.push({ id, agent_id: agentId, task, success, latency_ms: latencyMs, tools_used: toolsUsed, timestamp: Date.now() });
await fs.writeFile(path.join(process.cwd(), EVAL_FILE), JSON.stringify(evaluations, null, 2));
getState().lastConfigChange = new Date();
return { success: true };
}
export async function eval_get_stats(): Promise<{ total_tasks: number; success_rate: number; avg_latency_ms: number }> {
await loadEvals();
const total = evaluations.length;
if (total === 0) return { total_tasks: 0, success_rate: 0, avg_latency_ms: 0 };
const successes = evaluations.filter(e => e.success).length;
const avgLatency = evaluations.reduce((sum, e) => sum + e.latency_ms, 0) / total;
return {
total_tasks: total,
success_rate: Math.round((successes / total) * 100),
avg_latency_ms: Math.round(avgLatency)
};
}
export async function eval_get_history(limit?: number): Promise<{ entries: EvalEntry[] }> {
await loadEvals();
const sorted = evaluations.sort((a, b) => b.timestamp - a.timestamp);
return { entries: limit ? sorted.slice(0, limit) : sorted };
}
export async function eval_clear(): Promise<{ success: boolean }> {
try {
await fs.unlink(path.join(process.cwd(), EVAL_FILE));
evaluations = [];
return { success: true };
} catch {
return { success: false };
}
}