src / tools / bulk.ts
src / tools / bulk.ts
import { tool, type Tool } from "@lmstudio/sdk";
import { existsSync } from "fs";
import { readdir, readFile, stat, writeFile } from "fs/promises";
import { extname, join } from "path";
import { z } from "zod";
import { snapshot } from "../backup";
import { validateSyntax } from "../validate";
import { clamp, formatBytes, IGNORED_DIRS, type Workspace } from "../workspace";
/**
* Tools that trade many round-trips for one. Every tool call costs a full
* prediction on a local model, so reading six files one at a time is six
* generations of latency; doing it in one call is one.
*/
const MAX_FILES_READ = 12;
const MAX_TOTAL_CHARS = 40000;
const MAX_FILES_CHANGED = 60;
const MAX_RECENT_LISTED = 40;
const SKIP_EXTENSIONS = new Set([
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".exe",
".dll", ".so", ".dylib", ".mp3", ".wav", ".mp4", ".bin", ".3dsx", ".cia", ".woff", ".woff2",
]);
export function bulkTools(ws: Workspace): Tool[] {
const tools: Tool[] = [
tool({
name: "read_many_files",
description:
"Read several files at once and get them back together. Use this instead of calling " +
"read_file repeatedly -- it is much faster. Good for reading a module and its tests, or " +
"every file you are about to change. Each file is truncated if long.",
parameters: {
paths: z.array(z.string()).min(1).describe("File paths relative to the workspace root."),
max_lines_each: z
.number()
.int()
.min(1)
.default(200)
.describe("Lines to show per file before truncating."),
},
implementation: async ({ paths, max_lines_each }, ctx) => {
const wanted = paths.slice(0, MAX_FILES_READ);
ctx.status(`Reading ${wanted.length} file(s)`);
const sections: string[] = [];
let budget = MAX_TOTAL_CHARS;
for (const candidate of wanted) {
let absPath: string;
try {
absPath = ws.resolveInRoot(candidate);
} catch (error) {
sections.push(`===== ${candidate} =====\n${(error as Error).message}`);
continue;
}
if (!existsSync(absPath)) {
sections.push(`===== ${candidate} =====\nDoes not exist.`);
continue;
}
const info = await stat(absPath);
if (info.isDirectory()) {
sections.push(`===== ${candidate} =====\nThis is a directory, not a file.`);
continue;
}
if (budget <= 0) {
sections.push(`===== ${candidate} =====\nSkipped: combined output limit reached.`);
continue;
}
try {
const lines = (await readFile(absPath, "utf-8")).split(/\r?\n/);
const shown = lines.slice(0, max_lines_each);
const width = String(shown.length).length;
const body = shown
.map((line, i) => `${String(i + 1).padStart(width, " ")}\t${line}`)
.join("\n");
const more =
lines.length > shown.length
? `\n[${lines.length - shown.length} more line(s); use read_file with start_line=${shown.length + 1}]`
: "";
const section = `===== ${ws.rel(absPath)} (${lines.length} lines, ${formatBytes(info.size)}) =====\n${body}${more}`;
budget -= section.length;
sections.push(section);
} catch {
sections.push(`===== ${candidate} =====\nCould not be read as text.`);
}
}
const skipped =
paths.length > wanted.length
? `\n\n[${paths.length - wanted.length} path(s) not read: at most ${MAX_FILES_READ} per call]`
: "";
return clamp(sections.join("\n\n") + skipped, MAX_TOTAL_CHARS + 2000, "combined files");
},
}),
];
tools.push(
tool({
name: "recent_changes",
description:
"List files in the workspace modified recently, whoever changed them. Use this when " +
"resuming work, or when the user says they edited something: changed_files only knows " +
"about edits YOU made, so this is the only way to notice theirs.",
parameters: {
minutes: z.number().int().min(1).default(30).describe("How far back to look, in minutes."),
},
implementation: async ({ minutes }, ctx) => {
ctx.status(`Looking for files changed in the last ${minutes} min`);
const now = Date.now();
const cutoff = now - minutes * 60_000;
const found: { display: string; ageMin: number; size: number }[] = [];
await walk(ws, ws.root, /^.*$/, async (absPath) => {
try {
const info = await stat(absPath);
if (info.mtimeMs < cutoff) return;
found.push({
display: ws.rel(absPath),
ageMin: Math.max(0, Math.round((now - info.mtimeMs) / 60_000)),
size: info.size,
});
} catch {
// Vanished between listing and stat; ignore.
}
});
if (found.length === 0) {
return `No files in the workspace were modified in the last ${minutes} minute(s).`;
}
found.sort((a, b) => a.ageMin - b.ageMin);
const shown = found.slice(0, MAX_RECENT_LISTED);
const lines = shown.map(
(item) =>
` ${item.display} -- ${item.ageMin === 0 ? "just now" : `${item.ageMin} min ago`} (${formatBytes(item.size)})`,
);
const more =
found.length > shown.length ? `\n ...and ${found.length - shown.length} more` : "";
return (
`${found.length} file(s) modified in the last ${minutes} minute(s), newest first:\n` +
`${lines.join("\n")}${more}\n\n` +
`Read any you did not write yourself before assuming you know what they contain.`
);
},
}),
);
if (ws.allowWrite) {
tools.push(
tool({
name: "replace_in_files",
description:
"Replace an exact piece of text everywhere it appears across many files. Use this for " +
"renames and other sweeping changes, so no call site is missed. ALWAYS run it once with " +
"dry_run true to see what would change, then again with dry_run false to apply it.",
parameters: {
find: z.string().default("").describe("Exact text to find. Not a regular expression."),
replace: z.string().default("").describe("Text to put in its place."),
file_glob: z
.string()
.default("**/*")
.describe("Which files to touch, e.g. '**/*.ts'. Defaults to every text file."),
dry_run: z
.boolean()
.default(true)
.describe("True reports what would change without writing. Run this way first."),
},
implementation: async ({ find, replace, file_glob, dry_run }, ctx) => {
if (find.trim() === "") return "Error: find is empty. Pass the exact text to replace.";
if (find === replace) {
return "Error: find and replace are identical, so nothing would change.";
}
ctx.status(dry_run ? `Previewing replacement of ${find}` : `Replacing ${find}`);
const matcher = globToRegExp(file_glob);
const hits: { absPath: string; display: string; count: number; content: string }[] = [];
await walk(ws, ws.root, matcher, async (absPath, content) => {
if (hits.length >= MAX_FILES_CHANGED) return;
const count = countOccurrences(content, find);
if (count > 0) hits.push({ absPath, display: ws.rel(absPath), count, content });
});
if (hits.length === 0) {
return `No file matching "${file_glob}" contains "${find}". Check the spelling, or widen file_glob.`;
}
const total = hits.reduce((sum, hit) => sum + hit.count, 0);
const listing = hits
.map((hit) => ` ${hit.display}: ${hit.count} occurrence${hit.count === 1 ? "" : "s"}`)
.join("\n");
if (dry_run) {
return (
`Dry run -- nothing was changed.\n\n${total} occurrence(s) of "${find}" in ` +
`${hits.length} file(s):\n${listing}\n\nRun again with dry_run false to apply.`
);
}
const complaints: string[] = [];
for (const hit of hits) {
await snapshot(ws, hit.absPath);
await writeFile(hit.absPath, hit.content.split(find).join(replace), "utf-8");
const complaint = await validateSyntax(ws, hit.absPath);
if (complaint !== "") complaints.push(complaint.trim());
}
return (
`Replaced ${total} occurrence(s) of "${find}" with "${replace}" across ${hits.length} file(s):\n` +
`${listing}\n\nEvery file was backed up first. Run verify now.` +
`${complaints.length > 0 ? `\n\n${complaints.join("\n")}` : ""}`
);
},
}),
);
}
return tools;
}
function countOccurrences(haystack: string, needle: string): number {
if (needle === "") return 0;
return haystack.split(needle).length - 1;
}
/** Same translation the search tools use: **, *, ? and {a,b}. */
function globToRegExp(pattern: string): RegExp {
let out = "";
for (let i = 0; i < pattern.length; i++) {
const char = pattern[i];
if (char === "*") {
if (pattern[i + 1] === "*") {
out += ".*";
i++;
if (pattern[i + 1] === "/") i++;
} else {
out += "[^/]*";
}
} else if (char === "?") {
out += "[^/]";
} else if (char === "{") {
out += "(";
} else if (char === "}") {
out += ")";
} else if (char === ",") {
out += "|";
} else if ("\\^$+.()|[]".includes(char)) {
out += `\\${char}`;
} else {
out += char;
}
}
return new RegExp(`^${out}$`, "i");
}
async function walk(
ws: Workspace,
dir: string,
matcher: RegExp,
onFile: (absPath: string, content: string) => Promise<void>,
depth = 0,
): Promise<void> {
if (depth > 12) return;
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
const full = join(dir, entry.name);
if (entry.isDirectory()) {
if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith(".")) continue;
await walk(ws, full, matcher, onFile, depth + 1);
continue;
}
if (SKIP_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
const rel = ws.rel(full).replace(/\\/g, "/");
if (!matcher.test(rel) && !matcher.test(entry.name)) continue;
let info;
try {
info = await stat(full);
} catch {
continue;
}
if (info.size > ws.maxBytes) continue;
try {
await onFile(full, await readFile(full, "utf-8"));
} catch {
// Binary or unreadable; skip.
}
}
}
import { tool, type Tool } from "@lmstudio/sdk";
import { existsSync } from "fs";
import { readdir, readFile, stat, writeFile } from "fs/promises";
import { extname, join } from "path";
import { z } from "zod";
import { snapshot } from "../backup";
import { validateSyntax } from "../validate";
import { clamp, formatBytes, IGNORED_DIRS, type Workspace } from "../workspace";
/**
* Tools that trade many round-trips for one. Every tool call costs a full
* prediction on a local model, so reading six files one at a time is six
* generations of latency; doing it in one call is one.
*/
const MAX_FILES_READ = 12;
const MAX_TOTAL_CHARS = 40000;
const MAX_FILES_CHANGED = 60;
const MAX_RECENT_LISTED = 40;
const SKIP_EXTENSIONS = new Set([
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".exe",
".dll", ".so", ".dylib", ".mp3", ".wav", ".mp4", ".bin", ".3dsx", ".cia", ".woff", ".woff2",
]);
export function bulkTools(ws: Workspace): Tool[] {
const tools: Tool[] = [
tool({
name: "read_many_files",
description:
"Read several files at once and get them back together. Use this instead of calling " +
"read_file repeatedly -- it is much faster. Good for reading a module and its tests, or " +
"every file you are about to change. Each file is truncated if long.",
parameters: {
paths: z.array(z.string()).min(1).describe("File paths relative to the workspace root."),
max_lines_each: z
.number()
.int()
.min(1)
.default(200)
.describe("Lines to show per file before truncating."),
},
implementation: async ({ paths, max_lines_each }, ctx) => {
const wanted = paths.slice(0, MAX_FILES_READ);
ctx.status(`Reading ${wanted.length} file(s)`);
const sections: string[] = [];
let budget = MAX_TOTAL_CHARS;
for (const candidate of wanted) {
let absPath: string;
try {
absPath = ws.resolveInRoot(candidate);
} catch (error) {
sections.push(`===== ${candidate} =====\n${(error as Error).message}`);
continue;
}
if (!existsSync(absPath)) {
sections.push(`===== ${candidate} =====\nDoes not exist.`);
continue;
}
const info = await stat(absPath);
if (info.isDirectory()) {
sections.push(`===== ${candidate} =====\nThis is a directory, not a file.`);
continue;
}
if (budget <= 0) {
sections.push(`===== ${candidate} =====\nSkipped: combined output limit reached.`);
continue;
}
try {
const lines = (await readFile(absPath, "utf-8")).split(/\r?\n/);
const shown = lines.slice(0, max_lines_each);
const width = String(shown.length).length;
const body = shown
.map((line, i) => `${String(i + 1).padStart(width, " ")}\t${line}`)
.join("\n");
const more =
lines.length > shown.length
? `\n[${lines.length - shown.length} more line(s); use read_file with start_line=${shown.length + 1}]`
: "";
const section = `===== ${ws.rel(absPath)} (${lines.length} lines, ${formatBytes(info.size)}) =====\n${body}${more}`;
budget -= section.length;
sections.push(section);
} catch {
sections.push(`===== ${candidate} =====\nCould not be read as text.`);
}
}
const skipped =
paths.length > wanted.length
? `\n\n[${paths.length - wanted.length} path(s) not read: at most ${MAX_FILES_READ} per call]`
: "";
return clamp(sections.join("\n\n") + skipped, MAX_TOTAL_CHARS + 2000, "combined files");
},
}),
];
tools.push(
tool({
name: "recent_changes",
description:
"List files in the workspace modified recently, whoever changed them. Use this when " +
"resuming work, or when the user says they edited something: changed_files only knows " +
"about edits YOU made, so this is the only way to notice theirs.",
parameters: {
minutes: z.number().int().min(1).default(30).describe("How far back to look, in minutes."),
},
implementation: async ({ minutes }, ctx) => {
ctx.status(`Looking for files changed in the last ${minutes} min`);
const now = Date.now();
const cutoff = now - minutes * 60_000;
const found: { display: string; ageMin: number; size: number }[] = [];
await walk(ws, ws.root, /^.*$/, async (absPath) => {
try {
const info = await stat(absPath);
if (info.mtimeMs < cutoff) return;
found.push({
display: ws.rel(absPath),
ageMin: Math.max(0, Math.round((now - info.mtimeMs) / 60_000)),
size: info.size,
});
} catch {
// Vanished between listing and stat; ignore.
}
});
if (found.length === 0) {
return `No files in the workspace were modified in the last ${minutes} minute(s).`;
}
found.sort((a, b) => a.ageMin - b.ageMin);
const shown = found.slice(0, MAX_RECENT_LISTED);
const lines = shown.map(
(item) =>
` ${item.display} -- ${item.ageMin === 0 ? "just now" : `${item.ageMin} min ago`} (${formatBytes(item.size)})`,
);
const more =
found.length > shown.length ? `\n ...and ${found.length - shown.length} more` : "";
return (
`${found.length} file(s) modified in the last ${minutes} minute(s), newest first:\n` +
`${lines.join("\n")}${more}\n\n` +
`Read any you did not write yourself before assuming you know what they contain.`
);
},
}),
);
if (ws.allowWrite) {
tools.push(
tool({
name: "replace_in_files",
description:
"Replace an exact piece of text everywhere it appears across many files. Use this for " +
"renames and other sweeping changes, so no call site is missed. ALWAYS run it once with " +
"dry_run true to see what would change, then again with dry_run false to apply it.",
parameters: {
find: z.string().default("").describe("Exact text to find. Not a regular expression."),
replace: z.string().default("").describe("Text to put in its place."),
file_glob: z
.string()
.default("**/*")
.describe("Which files to touch, e.g. '**/*.ts'. Defaults to every text file."),
dry_run: z
.boolean()
.default(true)
.describe("True reports what would change without writing. Run this way first."),
},
implementation: async ({ find, replace, file_glob, dry_run }, ctx) => {
if (find.trim() === "") return "Error: find is empty. Pass the exact text to replace.";
if (find === replace) {
return "Error: find and replace are identical, so nothing would change.";
}
ctx.status(dry_run ? `Previewing replacement of ${find}` : `Replacing ${find}`);
const matcher = globToRegExp(file_glob);
const hits: { absPath: string; display: string; count: number; content: string }[] = [];
await walk(ws, ws.root, matcher, async (absPath, content) => {
if (hits.length >= MAX_FILES_CHANGED) return;
const count = countOccurrences(content, find);
if (count > 0) hits.push({ absPath, display: ws.rel(absPath), count, content });
});
if (hits.length === 0) {
return `No file matching "${file_glob}" contains "${find}". Check the spelling, or widen file_glob.`;
}
const total = hits.reduce((sum, hit) => sum + hit.count, 0);
const listing = hits
.map((hit) => ` ${hit.display}: ${hit.count} occurrence${hit.count === 1 ? "" : "s"}`)
.join("\n");
if (dry_run) {
return (
`Dry run -- nothing was changed.\n\n${total} occurrence(s) of "${find}" in ` +
`${hits.length} file(s):\n${listing}\n\nRun again with dry_run false to apply.`
);
}
const complaints: string[] = [];
for (const hit of hits) {
await snapshot(ws, hit.absPath);
await writeFile(hit.absPath, hit.content.split(find).join(replace), "utf-8");
const complaint = await validateSyntax(ws, hit.absPath);
if (complaint !== "") complaints.push(complaint.trim());
}
return (
`Replaced ${total} occurrence(s) of "${find}" with "${replace}" across ${hits.length} file(s):\n` +
`${listing}\n\nEvery file was backed up first. Run verify now.` +
`${complaints.length > 0 ? `\n\n${complaints.join("\n")}` : ""}`
);
},
}),
);
}
return tools;
}
function countOccurrences(haystack: string, needle: string): number {
if (needle === "") return 0;
return haystack.split(needle).length - 1;
}
/** Same translation the search tools use: **, *, ? and {a,b}. */
function globToRegExp(pattern: string): RegExp {
let out = "";
for (let i = 0; i < pattern.length; i++) {
const char = pattern[i];
if (char === "*") {
if (pattern[i + 1] === "*") {
out += ".*";
i++;
if (pattern[i + 1] === "/") i++;
} else {
out += "[^/]*";
}
} else if (char === "?") {
out += "[^/]";
} else if (char === "{") {
out += "(";
} else if (char === "}") {
out += ")";
} else if (char === ",") {
out += "|";
} else if ("\\^$+.()|[]".includes(char)) {
out += `\\${char}`;
} else {
out += char;
}
}
return new RegExp(`^${out}$`, "i");
}
async function walk(
ws: Workspace,
dir: string,
matcher: RegExp,
onFile: (absPath: string, content: string) => Promise<void>,
depth = 0,
): Promise<void> {
if (depth > 12) return;
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
const full = join(dir, entry.name);
if (entry.isDirectory()) {
if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith(".")) continue;
await walk(ws, full, matcher, onFile, depth + 1);
continue;
}
if (SKIP_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
const rel = ws.rel(full).replace(/\\/g, "/");
if (!matcher.test(rel) && !matcher.test(entry.name)) continue;
let info;
try {
info = await stat(full);
} catch {
continue;
}
if (info.size > ws.maxBytes) continue;
try {
await onFile(full, await readFile(full, "utf-8"));
} catch {
// Binary or unreadable; skip.
}
}
}