src / research / store.ts
src / research / store.ts
import { AgenticError } from "../core/errors";
import { createId } from "../core/id";
import type { InternalStorage } from "../core/internalStorage";
import type { FetchedWebPage, WebSearchResponse } from "./web";
export type ResearchStatus = "active" | "completed" | "failed" | "archived";
export interface ResearchQueryRecord {
id: string;
query: string;
provider: string;
resultCount: number;
createdAt: string;
resultPath: string;
}
export interface ResearchSourceRecord {
id: string;
url: string;
title?: string;
contentType: string;
fetchedAt: string;
sha256: string;
bytes: number;
truncated: boolean;
contentPath: string;
excerpt: string;
}
export interface ResearchNote {
id: string;
createdAt: string;
kind: "finding" | "claim" | "question" | "warning" | "method";
text: string;
sourceIds: string[];
}
export interface ResearchProject {
version: 1;
id: string;
title: string;
objective: string;
status: ResearchStatus;
createdAt: string;
updatedAt: string;
completedAt?: string;
archivedAt?: string;
agentRunId?: string;
queries: ResearchQueryRecord[];
sources: ResearchSourceRecord[];
notes: ResearchNote[];
reportPath?: string;
finalSummary?: string;
error?: string;
}
const researchProjectLocks = new Map<string, Promise<void>>();
function validateProjectId(id: string): void {
if (!/^research_[a-z0-9_]+$/i.test(id)) {
throw new AgenticError("INVALID_INPUT", `Invalid research project id: ${id}`);
}
}
function validateSourceId(id: string): void {
if (!/^source_[a-z0-9_]+$/i.test(id)) {
throw new AgenticError("INVALID_INPUT", `Invalid research source id: ${id}`);
}
}
function clean(value: string, field: string, max: number): string {
const result = value.trim();
if (!result) throw new AgenticError("INVALID_INPUT", `${field} may not be empty.`);
if (result.length > max) {
throw new AgenticError("INVALID_INPUT", `${field} is limited to ${max} characters.`);
}
return result;
}
export class ResearchStore {
private readonly rootRelative: string;
public constructor(private readonly storage: InternalStorage) {
this.rootRelative = storage.relative("research");
}
public async initialize(): Promise<void> {
await this.storage.ensureDirectory(this.rootRelative);
}
public async create(input: { title?: string; objective: string }): Promise<ResearchProject> {
await this.initialize();
const now = new Date().toISOString();
const objective = clean(input.objective, "Research objective", 20_000);
const project: ResearchProject = {
version: 1,
id: createId("research"),
title: input.title?.trim()
? clean(input.title, "Research title", 300)
: objective.slice(0, 180),
objective,
status: "active",
createdAt: now,
updatedAt: now,
queries: [],
sources: [],
notes: [],
};
await this.storage.ensureDirectory(this.projectRelative(project.id));
await this.storage.ensureDirectory(this.sourcesRelative(project.id));
await this.storage.ensureDirectory(this.queriesRelative(project.id));
await this.save(project);
return project;
}
public async load(id: string): Promise<ResearchProject> {
validateProjectId(id);
try {
const project = await this.storage.readJson<ResearchProject>(this.stateRelative(id));
if (project.version !== 1 || project.id !== id) {
throw new AgenticError("INTERNAL", `Research project state is invalid: ${id}`);
}
return project;
} catch (error) {
if (error instanceof AgenticError && error.code === "NOT_FOUND") {
throw new AgenticError("NOT_FOUND", `Research project not found: ${id}`);
}
throw error;
}
}
public async save(project: ResearchProject): Promise<void> {
validateProjectId(project.id);
await this.withLock(project.id, async () => {
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(project.id), project);
});
}
public async list(limit = 20, includeArchived = false): Promise<ResearchProject[]> {
await this.initialize();
const entries = await this.storage.readDirectory(this.rootRelative, { withFileTypes: true });
const projects: ResearchProject[] = [];
for (const entry of entries) {
if (!entry.isDirectory() || !entry.name.startsWith("research_")) continue;
try {
const project = await this.load(entry.name);
if (includeArchived || project.status !== "archived") projects.push(project);
} catch {
// Ignore incomplete directories while keeping the rest available.
}
}
return projects
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
.slice(0, Math.max(1, Math.min(limit, 100)));
}
public async recordSearch(id: string, search: WebSearchResponse): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const queryId = createId("query");
const resultPath = `${this.queriesRelative(id)}/${queryId}.json`;
await this.storage.writeJson(resultPath, search);
project.queries = [
...project.queries,
{
id: queryId,
query: search.query,
provider: search.provider,
resultCount: search.results.length,
createdAt: search.fetchedAt,
resultPath,
},
].slice(-200);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async recordSource(id: string, page: FetchedWebPage): Promise<{
project: ResearchProject;
source: ResearchSourceRecord;
}> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const existing = project.sources.find(
(source) => source.url === page.url && source.sha256 === page.sha256,
);
if (existing) return { project, source: existing };
const sourceId = createId("source");
const contentPath = `${this.sourcesRelative(id)}/${sourceId}.txt`;
const header = [
`URL: ${page.url}`,
`Fetched: ${page.fetchedAt}`,
`Content-Type: ${page.contentType}`,
`SHA-256: ${page.sha256}`,
page.title ? `Title: ${page.title}` : "",
"",
]
.filter((line) => line !== "")
.join("\n");
await this.storage.writeText(contentPath, `${header}\n${page.text}\n`);
const source: ResearchSourceRecord = {
id: sourceId,
url: page.url,
...(page.title ? { title: page.title } : {}),
contentType: page.contentType,
fetchedAt: page.fetchedAt,
sha256: page.sha256,
bytes: page.bytes,
truncated: page.truncated,
contentPath,
excerpt: page.text.slice(0, 1200),
};
project.sources = [...project.sources, source].slice(-500);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return { project, source };
});
}
public async addNote(
id: string,
input: {
kind?: ResearchNote["kind"];
text: string;
sourceIds?: string[];
},
): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const sourceIds = [...new Set(input.sourceIds ?? [])];
for (const sourceId of sourceIds) {
validateSourceId(sourceId);
if (!project.sources.some((source) => source.id === sourceId)) {
throw new AgenticError("NOT_FOUND", `Research source not found: ${sourceId}`);
}
}
project.notes = [
...project.notes,
{
id: createId("note"),
createdAt: new Date().toISOString(),
kind: input.kind ?? "finding",
text: clean(input.text, "Research note", 8000),
sourceIds,
},
].slice(-1000);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async attachAgent(id: string, runId: string): Promise<ResearchProject> {
return await this.mutate(id, (project) => {
project.agentRunId = runId;
});
}
public async complete(
id: string,
input: { summary: string; report?: string; error?: string },
): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
project.finalSummary = clean(input.summary, "Research summary", 20_000);
if (input.report?.trim()) {
const reportPath = `${this.projectRelative(id)}/report.md`;
await this.storage.writeText(reportPath, `${input.report.trim()}\n`);
project.reportPath = reportPath;
}
project.status = input.error ? "failed" : "completed";
project.completedAt = new Date().toISOString();
if (input.error) project.error = input.error.slice(0, 4000);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async archive(id: string): Promise<ResearchProject> {
return await this.mutate(id, (project) => {
project.status = "archived";
project.archivedAt = new Date().toISOString();
});
}
public stateRelativePath(id: string): string {
validateProjectId(id);
return this.stateRelative(id);
}
private async mutate(
id: string,
action: (project: ResearchProject) => void,
): Promise<ResearchProject> {
validateProjectId(id);
return await this.withLock(id, async () => {
const project = await this.load(id);
action(project);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
private async withLock<T>(id: string, action: () => Promise<T>): Promise<T> {
validateProjectId(id);
const key = `${this.storage.boundary.realRoot}\0${id}`;
const previous = researchProjectLocks.get(key) ?? Promise.resolve();
let release: () => void = () => undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const queued = previous.then(() => current);
researchProjectLocks.set(key, queued);
await previous;
try {
return await action();
} finally {
release();
if (researchProjectLocks.get(key) === queued) researchProjectLocks.delete(key);
}
}
private projectRelative(id: string): string {
validateProjectId(id);
return `${this.rootRelative}/${id}`;
}
private stateRelative(id: string): string {
return `${this.projectRelative(id)}/state.json`;
}
private sourcesRelative(id: string): string {
return `${this.projectRelative(id)}/sources`;
}
private queriesRelative(id: string): string {
return `${this.projectRelative(id)}/queries`;
}
}
import { AgenticError } from "../core/errors";
import { createId } from "../core/id";
import type { InternalStorage } from "../core/internalStorage";
import type { FetchedWebPage, WebSearchResponse } from "./web";
export type ResearchStatus = "active" | "completed" | "failed" | "archived";
export interface ResearchQueryRecord {
id: string;
query: string;
provider: string;
resultCount: number;
createdAt: string;
resultPath: string;
}
export interface ResearchSourceRecord {
id: string;
url: string;
title?: string;
contentType: string;
fetchedAt: string;
sha256: string;
bytes: number;
truncated: boolean;
contentPath: string;
excerpt: string;
}
export interface ResearchNote {
id: string;
createdAt: string;
kind: "finding" | "claim" | "question" | "warning" | "method";
text: string;
sourceIds: string[];
}
export interface ResearchProject {
version: 1;
id: string;
title: string;
objective: string;
status: ResearchStatus;
createdAt: string;
updatedAt: string;
completedAt?: string;
archivedAt?: string;
agentRunId?: string;
queries: ResearchQueryRecord[];
sources: ResearchSourceRecord[];
notes: ResearchNote[];
reportPath?: string;
finalSummary?: string;
error?: string;
}
const researchProjectLocks = new Map<string, Promise<void>>();
function validateProjectId(id: string): void {
if (!/^research_[a-z0-9_]+$/i.test(id)) {
throw new AgenticError("INVALID_INPUT", `Invalid research project id: ${id}`);
}
}
function validateSourceId(id: string): void {
if (!/^source_[a-z0-9_]+$/i.test(id)) {
throw new AgenticError("INVALID_INPUT", `Invalid research source id: ${id}`);
}
}
function clean(value: string, field: string, max: number): string {
const result = value.trim();
if (!result) throw new AgenticError("INVALID_INPUT", `${field} may not be empty.`);
if (result.length > max) {
throw new AgenticError("INVALID_INPUT", `${field} is limited to ${max} characters.`);
}
return result;
}
export class ResearchStore {
private readonly rootRelative: string;
public constructor(private readonly storage: InternalStorage) {
this.rootRelative = storage.relative("research");
}
public async initialize(): Promise<void> {
await this.storage.ensureDirectory(this.rootRelative);
}
public async create(input: { title?: string; objective: string }): Promise<ResearchProject> {
await this.initialize();
const now = new Date().toISOString();
const objective = clean(input.objective, "Research objective", 20_000);
const project: ResearchProject = {
version: 1,
id: createId("research"),
title: input.title?.trim()
? clean(input.title, "Research title", 300)
: objective.slice(0, 180),
objective,
status: "active",
createdAt: now,
updatedAt: now,
queries: [],
sources: [],
notes: [],
};
await this.storage.ensureDirectory(this.projectRelative(project.id));
await this.storage.ensureDirectory(this.sourcesRelative(project.id));
await this.storage.ensureDirectory(this.queriesRelative(project.id));
await this.save(project);
return project;
}
public async load(id: string): Promise<ResearchProject> {
validateProjectId(id);
try {
const project = await this.storage.readJson<ResearchProject>(this.stateRelative(id));
if (project.version !== 1 || project.id !== id) {
throw new AgenticError("INTERNAL", `Research project state is invalid: ${id}`);
}
return project;
} catch (error) {
if (error instanceof AgenticError && error.code === "NOT_FOUND") {
throw new AgenticError("NOT_FOUND", `Research project not found: ${id}`);
}
throw error;
}
}
public async save(project: ResearchProject): Promise<void> {
validateProjectId(project.id);
await this.withLock(project.id, async () => {
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(project.id), project);
});
}
public async list(limit = 20, includeArchived = false): Promise<ResearchProject[]> {
await this.initialize();
const entries = await this.storage.readDirectory(this.rootRelative, { withFileTypes: true });
const projects: ResearchProject[] = [];
for (const entry of entries) {
if (!entry.isDirectory() || !entry.name.startsWith("research_")) continue;
try {
const project = await this.load(entry.name);
if (includeArchived || project.status !== "archived") projects.push(project);
} catch {
// Ignore incomplete directories while keeping the rest available.
}
}
return projects
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
.slice(0, Math.max(1, Math.min(limit, 100)));
}
public async recordSearch(id: string, search: WebSearchResponse): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const queryId = createId("query");
const resultPath = `${this.queriesRelative(id)}/${queryId}.json`;
await this.storage.writeJson(resultPath, search);
project.queries = [
...project.queries,
{
id: queryId,
query: search.query,
provider: search.provider,
resultCount: search.results.length,
createdAt: search.fetchedAt,
resultPath,
},
].slice(-200);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async recordSource(id: string, page: FetchedWebPage): Promise<{
project: ResearchProject;
source: ResearchSourceRecord;
}> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const existing = project.sources.find(
(source) => source.url === page.url && source.sha256 === page.sha256,
);
if (existing) return { project, source: existing };
const sourceId = createId("source");
const contentPath = `${this.sourcesRelative(id)}/${sourceId}.txt`;
const header = [
`URL: ${page.url}`,
`Fetched: ${page.fetchedAt}`,
`Content-Type: ${page.contentType}`,
`SHA-256: ${page.sha256}`,
page.title ? `Title: ${page.title}` : "",
"",
]
.filter((line) => line !== "")
.join("\n");
await this.storage.writeText(contentPath, `${header}\n${page.text}\n`);
const source: ResearchSourceRecord = {
id: sourceId,
url: page.url,
...(page.title ? { title: page.title } : {}),
contentType: page.contentType,
fetchedAt: page.fetchedAt,
sha256: page.sha256,
bytes: page.bytes,
truncated: page.truncated,
contentPath,
excerpt: page.text.slice(0, 1200),
};
project.sources = [...project.sources, source].slice(-500);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return { project, source };
});
}
public async addNote(
id: string,
input: {
kind?: ResearchNote["kind"];
text: string;
sourceIds?: string[];
},
): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
const sourceIds = [...new Set(input.sourceIds ?? [])];
for (const sourceId of sourceIds) {
validateSourceId(sourceId);
if (!project.sources.some((source) => source.id === sourceId)) {
throw new AgenticError("NOT_FOUND", `Research source not found: ${sourceId}`);
}
}
project.notes = [
...project.notes,
{
id: createId("note"),
createdAt: new Date().toISOString(),
kind: input.kind ?? "finding",
text: clean(input.text, "Research note", 8000),
sourceIds,
},
].slice(-1000);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async attachAgent(id: string, runId: string): Promise<ResearchProject> {
return await this.mutate(id, (project) => {
project.agentRunId = runId;
});
}
public async complete(
id: string,
input: { summary: string; report?: string; error?: string },
): Promise<ResearchProject> {
return await this.withLock(id, async () => {
const project = await this.load(id);
project.finalSummary = clean(input.summary, "Research summary", 20_000);
if (input.report?.trim()) {
const reportPath = `${this.projectRelative(id)}/report.md`;
await this.storage.writeText(reportPath, `${input.report.trim()}\n`);
project.reportPath = reportPath;
}
project.status = input.error ? "failed" : "completed";
project.completedAt = new Date().toISOString();
if (input.error) project.error = input.error.slice(0, 4000);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
public async archive(id: string): Promise<ResearchProject> {
return await this.mutate(id, (project) => {
project.status = "archived";
project.archivedAt = new Date().toISOString();
});
}
public stateRelativePath(id: string): string {
validateProjectId(id);
return this.stateRelative(id);
}
private async mutate(
id: string,
action: (project: ResearchProject) => void,
): Promise<ResearchProject> {
validateProjectId(id);
return await this.withLock(id, async () => {
const project = await this.load(id);
action(project);
project.updatedAt = new Date().toISOString();
await this.storage.writeJson(this.stateRelative(id), project);
return project;
});
}
private async withLock<T>(id: string, action: () => Promise<T>): Promise<T> {
validateProjectId(id);
const key = `${this.storage.boundary.realRoot}\0${id}`;
const previous = researchProjectLocks.get(key) ?? Promise.resolve();
let release: () => void = () => undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const queued = previous.then(() => current);
researchProjectLocks.set(key, queued);
await previous;
try {
return await action();
} finally {
release();
if (researchProjectLocks.get(key) === queued) researchProjectLocks.delete(key);
}
}
private projectRelative(id: string): string {
validateProjectId(id);
return `${this.rootRelative}/${id}`;
}
private stateRelative(id: string): string {
return `${this.projectRelative(id)}/state.json`;
}
private sourcesRelative(id: string): string {
return `${this.projectRelative(id)}/sources`;
}
private queriesRelative(id: string): string {
return `${this.projectRelative(id)}/queries`;
}
}