ai-lab

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDataDir = getDataDir;
exports.getDb = getDb;
exports.savePrompt = savePrompt;
exports.getPrompt = getPrompt;
exports.listPromptNames = listPromptNames;
exports.listPromptVersions = listPromptVersions;
exports.createDataset = createDataset;
exports.getDataset = getDataset;
exports.listDatasets = listDatasets;
exports.addEvalCase = addEvalCase;
exports.getDatasetCases = getDatasetCases;
exports.logModelResult = logModelResult;
exports.getResultsForDataset = getResultsForDataset;
exports.getModelStats = getModelStats;
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
const os_1 = require("os");
const fs_1 = require("fs");
const path_1 = require("path");
function getDataDir(configPath) {
    const p = configPath.trim() || "~/ai-lab-data";
    return p.startsWith("~/") ? (0, path_1.join)((0, os_1.homedir)(), p.slice(2)) : p;
}
const _dbCache = new Map();
function getDb(dataDir) {
    if (_dbCache.has(dataDir))
        return _dbCache.get(dataDir);
    (0, fs_1.mkdirSync)(dataDir, { recursive: true });
    const db = new better_sqlite3_1.default((0, path_1.join)(dataDir, "ai-lab.db"));
    db.pragma("journal_mode = WAL");
    db.exec(`
    CREATE TABLE IF NOT EXISTS prompts (
      id          INTEGER PRIMARY KEY AUTOINCREMENT,
      name        TEXT    NOT NULL,
      version     INTEGER NOT NULL DEFAULT 1,
      content     TEXT    NOT NULL,
      description TEXT    NOT NULL DEFAULT '',
      tags        TEXT    NOT NULL DEFAULT '[]',
      variables   TEXT    NOT NULL DEFAULT '[]',
      createdAt   TEXT    NOT NULL,
      UNIQUE(name, version)
    );

    CREATE TABLE IF NOT EXISTS eval_datasets (
      id          INTEGER PRIMARY KEY AUTOINCREMENT,
      name        TEXT    NOT NULL UNIQUE,
      description TEXT    NOT NULL DEFAULT '',
      tags        TEXT    NOT NULL DEFAULT '[]',
      createdAt   TEXT    NOT NULL,
      updatedAt   TEXT    NOT NULL
    );

    CREATE TABLE IF NOT EXISTS eval_cases (
      id              INTEGER PRIMARY KEY AUTOINCREMENT,
      datasetId       INTEGER NOT NULL,
      input           TEXT    NOT NULL,
      expectedOutput  TEXT    NOT NULL DEFAULT '',
      tags            TEXT    NOT NULL DEFAULT '[]',
      notes           TEXT    NOT NULL DEFAULT '',
      createdAt       TEXT    NOT NULL,
      FOREIGN KEY (datasetId) REFERENCES eval_datasets(id) ON DELETE CASCADE
    );

    CREATE TABLE IF NOT EXISTS model_results (
      id          INTEGER PRIMARY KEY AUTOINCREMENT,
      caseId      INTEGER NOT NULL,
      model       TEXT    NOT NULL,
      promptName  TEXT    NOT NULL DEFAULT '',
      promptVer   INTEGER NOT NULL DEFAULT 0,
      output      TEXT    NOT NULL,
      score       REAL,
      scoreLabel  TEXT    NOT NULL DEFAULT '',
      notes       TEXT    NOT NULL DEFAULT '',
      runAt       TEXT    NOT NULL,
      FOREIGN KEY (caseId) REFERENCES eval_cases(id) ON DELETE CASCADE
    );

    CREATE INDEX IF NOT EXISTS idx_prompts_name      ON prompts(name);
    CREATE INDEX IF NOT EXISTS idx_prompts_version   ON prompts(name, version);
    CREATE INDEX IF NOT EXISTS idx_cases_dataset     ON eval_cases(datasetId);
    CREATE INDEX IF NOT EXISTS idx_results_caseId    ON model_results(caseId);
    CREATE INDEX IF NOT EXISTS idx_results_model     ON model_results(model);
  `);
    _dbCache.set(dataDir, db);
    return db;
}
function savePrompt(db, name, content, description, tags, variables) {
    const now = new Date().toISOString();
    const latest = db.prepare("SELECT MAX(version) as v FROM prompts WHERE name = ?").get(name);
    const version = (latest.v ?? 0) + 1;
    db.prepare(`INSERT INTO prompts (name, version, content, description, tags, variables, createdAt)
     VALUES (?, ?, ?, ?, ?, ?, ?)`).run(name, version, content, description, JSON.stringify(tags), JSON.stringify(variables), now);
    return db.prepare("SELECT * FROM prompts WHERE name = ? AND version = ?").get(name, version);
}
function getPrompt(db, name, version) {
    if (version !== undefined) {
        const row = db.prepare("SELECT * FROM prompts WHERE name = ? AND version = ?").get(name, version);
        if (!row)
            throw new Error(`Prompt '${name}' version ${version} not found.`);
        return row;
    }
    const row = db.prepare("SELECT * FROM prompts WHERE name = ? ORDER BY version DESC LIMIT 1").get(name);
    if (!row)
        throw new Error(`Prompt '${name}' not found.`);
    return row;
}
function listPromptNames(db) {
    return db.prepare(`SELECT name, MAX(version) as latestVersion, description, tags
     FROM prompts GROUP BY name ORDER BY name`).all();
}
function listPromptVersions(db, name) {
    return db.prepare("SELECT * FROM prompts WHERE name = ? ORDER BY version DESC").all(name);
}
function createDataset(db, name, description, tags) {
    const now = new Date().toISOString();
    db.prepare(`INSERT INTO eval_datasets (name, description, tags, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`).run(name, description, JSON.stringify(tags), now, now);
    return db.prepare("SELECT * FROM eval_datasets WHERE name = ?").get(name);
}
function getDataset(db, name) {
    const row = db.prepare("SELECT * FROM eval_datasets WHERE name = ?").get(name);
    if (!row)
        throw new Error(`Dataset '${name}' not found.`);
    return row;
}
function listDatasets(db) {
    return db.prepare(`SELECT d.*, COUNT(c.id) as caseCount
     FROM eval_datasets d
     LEFT JOIN eval_cases c ON c.datasetId = d.id
     GROUP BY d.id ORDER BY d.name`).all();
}
function addEvalCase(db, datasetId, input, expectedOutput, tags, notes) {
    const now = new Date().toISOString();
    const r = db.prepare(`INSERT INTO eval_cases (datasetId, input, expectedOutput, tags, notes, createdAt)
     VALUES (?, ?, ?, ?, ?, ?)`).run(datasetId, input, expectedOutput, JSON.stringify(tags), notes, now);
    return db.prepare("SELECT * FROM eval_cases WHERE id = ?").get(r.lastInsertRowid);
}
function getDatasetCases(db, datasetId) {
    return db.prepare("SELECT * FROM eval_cases WHERE datasetId = ? ORDER BY createdAt").all(datasetId);
}
function logModelResult(db, caseId, model, promptName, promptVer, output, score, scoreLabel, notes) {
    const now = new Date().toISOString();
    const r = db.prepare(`INSERT INTO model_results (caseId, model, promptName, promptVer, output, score, scoreLabel, notes, runAt)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(caseId, model, promptName, promptVer, output, score, scoreLabel, notes, now);
    return db.prepare("SELECT * FROM model_results WHERE id = ?").get(r.lastInsertRowid);
}
function getResultsForDataset(db, datasetId) {
    const cases = getDatasetCases(db, datasetId);
    return cases.map((c) => ({
        ...c,
        results: db.prepare("SELECT * FROM model_results WHERE caseId = ? ORDER BY runAt").all(c.id),
    }));
}
function getModelStats(db, datasetId) {
    return db.prepare(`SELECT mr.model, mr.promptName, mr.promptVer,
            COUNT(mr.id) as caseCount,
            AVG(mr.score) as avgScore,
            COUNT(mr.score) as scoredCount
     FROM model_results mr
     JOIN eval_cases ec ON ec.id = mr.caseId
     WHERE ec.datasetId = ?
     GROUP BY mr.model, mr.promptName, mr.promptVer
     ORDER BY avgScore DESC NULLS LAST`).all(datasetId);
}