src / copies.ts
src / copies.ts
// Catches a model retyping a file it read as a "new" file. Retyping costs many tokens, can be cut
// off mid-call, and can quietly drop or change lines; copy_files does the same job byte for byte.
// simpleFiles remembers which files were read recently (path and a hash, never contents), and a
// proposed new file that matches one of them, exactly or nearly, is turned away with a pointer
// to copy_files.
import * as fs from "node:fs";
import * as path from "node:path";
import { dataDir, readJson, sha256, writeJson } from "./store";
interface Read {
path: string;
hash: string;
lines: number;
}
const MAX_READS = 200;
// Files smaller than this are cheap to retype, and short files often share most of their lines.
const MIN_LINES = 40;
const MIN_CHARS = 2000;
const NEAR = 0.9;
const file = () => path.join(dataDir(), "recent-reads.json");
const normalize = (text: string) =>
text
.replace(/\r\n?/g, "\n")
.split("\n")
.map((l) => l.replace(/\s+$/, ""))
.join("\n")
.replace(/\n+$/, "");
export function recordRead(p: string, text: string): void {
try {
const norm = normalize(text);
const lines = norm ? norm.split("\n").length : 0;
if (lines < MIN_LINES && norm.length < MIN_CHARS) return;
const reads = readJson<Read[]>(file(), []).filter((r) => r.path !== p);
reads.push({ path: p, hash: sha256(norm), lines });
writeJson(file(), reads.slice(-MAX_READS));
} catch {
// best effort: a failure here only means a copy is not caught
}
}
// Share of a's non-blank lines found in b, counting repeats.
function overlap(a: string[], b: string[]): number {
const counts = new Map<string, number>();
for (const l of b) if (l.trim()) counts.set(l, (counts.get(l) ?? 0) + 1);
let total = 0;
let found = 0;
for (const l of a) {
if (!l.trim()) continue;
total++;
const n = counts.get(l) ?? 0;
if (n > 0) {
found++;
counts.set(l, n - 1);
}
}
return total ? found / total : 0;
}
// The recently read file a proposed new file copies, if any: exact, or with only a few lines
// different. The source is checked on disk, so a file that has changed since it was read is not
// mistaken for one.
export function copiedFrom(content: string, newPath: string): { source: string; exact: boolean } | null {
const norm = normalize(content ?? "");
const lines = norm.split("\n");
if (lines.length < MIN_LINES && norm.length < MIN_CHARS) return null;
const hash = sha256(norm);
const reads = readJson<Read[]>(file(), []).reverse();
const current = (p: string) => {
try {
return normalize(fs.readFileSync(p, "utf8"));
} catch {
return null;
}
};
for (const r of reads) {
if (r.hash !== hash || r.path === newPath) continue;
const now = current(r.path);
if (now !== null && sha256(now) === hash) return { source: r.path, exact: true };
}
const base = path.basename(newPath).toLowerCase();
const candidates = reads
.filter((r) => r.path !== newPath && Math.abs(r.lines - lines.length) <= Math.max(3, r.lines * 0.15))
.sort((a, b) => Number(path.basename(b.path).toLowerCase() === base) - Number(path.basename(a.path).toLowerCase() === base))
.slice(0, 20);
for (const r of candidates) {
const now = current(r.path);
if (now === null) continue;
const theirs = now.split("\n");
if (overlap(lines, theirs) >= NEAR && overlap(theirs, lines) >= NEAR) return { source: r.path, exact: false };
}
return null;
}
// Catches a model retyping a file it read as a "new" file. Retyping costs many tokens, can be cut
// off mid-call, and can quietly drop or change lines; copy_files does the same job byte for byte.
// simpleFiles remembers which files were read recently (path and a hash, never contents), and a
// proposed new file that matches one of them, exactly or nearly, is turned away with a pointer
// to copy_files.
import * as fs from "node:fs";
import * as path from "node:path";
import { dataDir, readJson, sha256, writeJson } from "./store";
interface Read {
path: string;
hash: string;
lines: number;
}
const MAX_READS = 200;
// Files smaller than this are cheap to retype, and short files often share most of their lines.
const MIN_LINES = 40;
const MIN_CHARS = 2000;
const NEAR = 0.9;
const file = () => path.join(dataDir(), "recent-reads.json");
const normalize = (text: string) =>
text
.replace(/\r\n?/g, "\n")
.split("\n")
.map((l) => l.replace(/\s+$/, ""))
.join("\n")
.replace(/\n+$/, "");
export function recordRead(p: string, text: string): void {
try {
const norm = normalize(text);
const lines = norm ? norm.split("\n").length : 0;
if (lines < MIN_LINES && norm.length < MIN_CHARS) return;
const reads = readJson<Read[]>(file(), []).filter((r) => r.path !== p);
reads.push({ path: p, hash: sha256(norm), lines });
writeJson(file(), reads.slice(-MAX_READS));
} catch {
// best effort: a failure here only means a copy is not caught
}
}
// Share of a's non-blank lines found in b, counting repeats.
function overlap(a: string[], b: string[]): number {
const counts = new Map<string, number>();
for (const l of b) if (l.trim()) counts.set(l, (counts.get(l) ?? 0) + 1);
let total = 0;
let found = 0;
for (const l of a) {
if (!l.trim()) continue;
total++;
const n = counts.get(l) ?? 0;
if (n > 0) {
found++;
counts.set(l, n - 1);
}
}
return total ? found / total : 0;
}
// The recently read file a proposed new file copies, if any: exact, or with only a few lines
// different. The source is checked on disk, so a file that has changed since it was read is not
// mistaken for one.
export function copiedFrom(content: string, newPath: string): { source: string; exact: boolean } | null {
const norm = normalize(content ?? "");
const lines = norm.split("\n");
if (lines.length < MIN_LINES && norm.length < MIN_CHARS) return null;
const hash = sha256(norm);
const reads = readJson<Read[]>(file(), []).reverse();
const current = (p: string) => {
try {
return normalize(fs.readFileSync(p, "utf8"));
} catch {
return null;
}
};
for (const r of reads) {
if (r.hash !== hash || r.path === newPath) continue;
const now = current(r.path);
if (now !== null && sha256(now) === hash) return { source: r.path, exact: true };
}
const base = path.basename(newPath).toLowerCase();
const candidates = reads
.filter((r) => r.path !== newPath && Math.abs(r.lines - lines.length) <= Math.max(3, r.lines * 0.15))
.sort((a, b) => Number(path.basename(b.path).toLowerCase() === base) - Number(path.basename(a.path).toLowerCase() === base))
.slice(0, 20);
for (const r of candidates) {
const now = current(r.path);
if (now === null) continue;
const theirs = now.split("\n");
if (overlap(lines, theirs) >= NEAR && overlap(theirs, lines) >= NEAR) return { source: r.path, exact: false };
}
return null;
}