src / rules.ts
src / rules.ts
import * as fs from "node:fs";
import * as path from "node:path";
import { load } from "cheerio/slim";
import { EngineName, KNOWN_ENGINES } from "./config";
// Selectors that pick results out of a search results page. These are the only parts
// a model may propose changes to. Search URLs and engine domains stay fixed in code,
// so a proposal can never redirect searches somewhere else.
export interface SelectorRules {
container: string;
title: string;
link: string;
snippet?: string;
exclude?: string;
urlParam?: string;
}
export const EDITABLE_FIELDS = ["container", "title", "link", "snippet", "exclude", "urlParam"] as const;
export interface EngineDef {
name: EngineName;
searchUrl: string;
domains: string[];
noResults?: string;
defaults: SelectorRules;
// Raise this when the built-in defaults change, so stored defaults get refreshed.
defaultsVersion: number;
}
export const ENGINES: Record<EngineName, EngineDef> = {
duckduckgo: {
name: "duckduckgo",
searchUrl: "https://html.duckduckgo.com/html/?q={q}",
domains: ["duckduckgo.com"],
noResults: ".no-results",
defaults: {
container: "div.result, div.web-result",
exclude: ".result--ad, .result--ad--small",
title: "a.result__a, h2 a",
link: "a.result__a, h2 a",
snippet: ".result__snippet",
urlParam: "uddg",
},
defaultsVersion: 1,
},
startpage: {
name: "startpage",
searchUrl: "https://www.startpage.com/sp/search?query={q}&cat=web",
domains: ["startpage.com"],
defaults: {
container: "div.result, div.w-gl__result",
exclude: ".result-ad, .w-gl__result--ad",
title: "a.result-title, a.w-gl__result-title, h2 a, h3",
link: "a.result-title, a.w-gl__result-title, a.result-link, h2 a",
snippet: "p.description, p.w-gl__description, .description",
},
defaultsVersion: 1,
},
mojeek: {
name: "mojeek",
searchUrl: "https://www.mojeek.com/search?q={q}",
domains: ["mojeek.com", "mojeek.co.uk"],
defaults: {
container: "ul.results-standard > li",
title: "a.title, h2 a",
link: "a.title, h2 a",
snippet: "p.s",
},
defaultsVersion: 1,
},
};
export interface StoredRules extends SelectorRules {
source: "default" | "repaired";
defaultsVersion: number;
updatedAt: string;
}
export interface PendingRules {
rules: SelectorRules;
proposedAt: string;
summary: string;
}
export function pickSelectors(input: Partial<Record<string, unknown>>): SelectorRules {
const out: Record<string, string> = {};
for (const field of EDITABLE_FIELDS) {
const v = input[field];
if (typeof v === "string" && v.trim()) out[field] = v.trim();
}
return out as unknown as SelectorRules;
}
// Returns a list of problems; an empty list means the selectors are usable.
export function checkSelectors(rules: SelectorRules): string[] {
const problems: string[] = [];
const $ = load("<div></div>");
for (const field of EDITABLE_FIELDS) {
const value = rules[field];
if (value === undefined || value.trim() === "") {
if (field === "container" || field === "title" || field === "link") problems.push(`${field} is required.`);
continue;
}
if (value.length > 300) {
problems.push(`${field} is longer than 300 characters.`);
continue;
}
if (field === "urlParam") {
if (!/^[A-Za-z0-9_-]{1,32}$/.test(value)) problems.push("url_param must be a plain query parameter name such as uddg.");
continue;
}
try {
$(value);
} catch {
problems.push(`${field} is not a valid CSS selector: ${value}`);
}
}
return problems;
}
function readJson<T>(file: string): T | null {
try {
return JSON.parse(fs.readFileSync(file, "utf8")) as T;
} catch {
return null;
}
}
function writeJson(file: string, value: unknown): void {
fs.mkdirSync(path.dirname(file), { recursive: true });
const tmp = `${file}.tmp`;
fs.writeFileSync(tmp, JSON.stringify(value, null, 2), "utf8");
fs.renameSync(tmp, file);
}
function stamp(): string {
return new Date().toISOString().replace(/[:.]/g, "-");
}
export class RuleStore {
constructor(private dir: string, private log: (msg: string) => void = () => {}) {}
private activeFile(engine: EngineName): string {
return path.join(this.dir, "rules", `${engine}.json`);
}
private pendingFile(engine: EngineName): string {
return path.join(this.dir, "rules", "pending", `${engine}.json`);
}
private historyDir(): string {
return path.join(this.dir, "rules", "history");
}
private changeLog(): string {
return path.join(this.dir, "changes.log");
}
private logChange(line: string): void {
try {
fs.mkdirSync(this.dir, { recursive: true });
fs.appendFileSync(this.changeLog(), `${new Date().toISOString()} ${line}\n`, "utf8");
} catch {
// logging is best effort
}
}
private writeDefaults(engine: EngineName): StoredRules {
const def = ENGINES[engine];
const stored: StoredRules = {
...def.defaults,
source: "default",
defaultsVersion: def.defaultsVersion,
updatedAt: new Date().toISOString(),
};
try {
writeJson(this.activeFile(engine), stored);
} catch (e) {
this.log(`could not write rules for ${engine}: ${(e as Error).message}`);
}
return stored;
}
get(engine: EngineName): StoredRules {
const def = ENGINES[engine];
const stored = readJson<StoredRules>(this.activeFile(engine));
if (!stored || checkSelectors(stored).length) return this.writeDefaults(engine);
if (stored.source === "default" && (stored.defaultsVersion ?? 0) < def.defaultsVersion) {
return this.writeDefaults(engine);
}
return stored;
}
apply(engine: EngineName, rules: SelectorRules, reason: string): void {
const current = readJson<StoredRules>(this.activeFile(engine));
if (current) writeJson(path.join(this.historyDir(), `${engine}-${stamp()}.json`), current);
const stored: StoredRules = {
...pickSelectors(rules as unknown as Record<string, unknown>),
source: "repaired",
defaultsVersion: ENGINES[engine].defaultsVersion,
updatedAt: new Date().toISOString(),
};
writeJson(this.activeFile(engine), stored);
this.logChange(`${engine}: rules applied (${reason})`);
}
getPending(engine: EngineName): PendingRules | null {
return readJson<PendingRules>(this.pendingFile(engine));
}
setPending(engine: EngineName, rules: SelectorRules, summary: string): void {
writeJson(this.pendingFile(engine), { rules, proposedAt: new Date().toISOString(), summary });
this.logChange(`${engine}: rules proposed, waiting for approval (${summary})`);
}
listPending(): EngineName[] {
return KNOWN_ENGINES.filter((e) => this.getPending(e) !== null);
}
approve(engine: EngineName): string {
const pending = this.getPending(engine);
if (!pending) return `No proposed rules are waiting for ${engine}.`;
this.apply(engine, pending.rules, "approved by user");
fs.rmSync(this.pendingFile(engine), { force: true });
return `Approved. ${engine} now uses the proposed rules. The previous rules were saved and can be restored with rollback.`;
}
reject(engine: EngineName): string {
if (!this.getPending(engine)) return `No proposed rules are waiting for ${engine}.`;
fs.rmSync(this.pendingFile(engine), { force: true });
this.logChange(`${engine}: proposed rules rejected by user`);
return `Rejected. The proposed rules for ${engine} were discarded.`;
}
rollback(engine: EngineName): string {
let files: string[] = [];
try {
files = fs.readdirSync(this.historyDir()).filter((f) => f.startsWith(`${engine}-`) && f.endsWith(".json")).sort();
} catch {
// no history yet
}
const latest = files[files.length - 1];
if (!latest) return `There are no earlier rules for ${engine} to roll back to. Use reset to restore the built-in defaults.`;
const file = path.join(this.historyDir(), latest);
const previous = readJson<StoredRules>(file);
if (!previous || checkSelectors(previous).length) return `The saved rules in ${latest} are unreadable.`;
writeJson(this.activeFile(engine), previous);
fs.rmSync(file, { force: true });
this.logChange(`${engine}: rolled back to rules from ${latest}`);
return `Rolled back. ${engine} now uses the rules it had before the last change.`;
}
reset(engine: EngineName): string {
const current = readJson<StoredRules>(this.activeFile(engine));
if (current) writeJson(path.join(this.historyDir(), `${engine}-${stamp()}.json`), current);
this.writeDefaults(engine);
this.logChange(`${engine}: reset to built-in defaults`);
return `Reset. ${engine} now uses the built-in default rules.`;
}
}
import * as fs from "node:fs";
import * as path from "node:path";
import { load } from "cheerio/slim";
import { EngineName, KNOWN_ENGINES } from "./config";
// Selectors that pick results out of a search results page. These are the only parts
// a model may propose changes to. Search URLs and engine domains stay fixed in code,
// so a proposal can never redirect searches somewhere else.
export interface SelectorRules {
container: string;
title: string;
link: string;
snippet?: string;
exclude?: string;
urlParam?: string;
}
export const EDITABLE_FIELDS = ["container", "title", "link", "snippet", "exclude", "urlParam"] as const;
export interface EngineDef {
name: EngineName;
searchUrl: string;
domains: string[];
noResults?: string;
defaults: SelectorRules;
// Raise this when the built-in defaults change, so stored defaults get refreshed.
defaultsVersion: number;
}
export const ENGINES: Record<EngineName, EngineDef> = {
duckduckgo: {
name: "duckduckgo",
searchUrl: "https://html.duckduckgo.com/html/?q={q}",
domains: ["duckduckgo.com"],
noResults: ".no-results",
defaults: {
container: "div.result, div.web-result",
exclude: ".result--ad, .result--ad--small",
title: "a.result__a, h2 a",
link: "a.result__a, h2 a",
snippet: ".result__snippet",
urlParam: "uddg",
},
defaultsVersion: 1,
},
startpage: {
name: "startpage",
searchUrl: "https://www.startpage.com/sp/search?query={q}&cat=web",
domains: ["startpage.com"],
defaults: {
container: "div.result, div.w-gl__result",
exclude: ".result-ad, .w-gl__result--ad",
title: "a.result-title, a.w-gl__result-title, h2 a, h3",
link: "a.result-title, a.w-gl__result-title, a.result-link, h2 a",
snippet: "p.description, p.w-gl__description, .description",
},
defaultsVersion: 1,
},
mojeek: {
name: "mojeek",
searchUrl: "https://www.mojeek.com/search?q={q}",
domains: ["mojeek.com", "mojeek.co.uk"],
defaults: {
container: "ul.results-standard > li",
title: "a.title, h2 a",
link: "a.title, h2 a",
snippet: "p.s",
},
defaultsVersion: 1,
},
};
export interface StoredRules extends SelectorRules {
source: "default" | "repaired";
defaultsVersion: number;
updatedAt: string;
}
export interface PendingRules {
rules: SelectorRules;
proposedAt: string;
summary: string;
}
export function pickSelectors(input: Partial<Record<string, unknown>>): SelectorRules {
const out: Record<string, string> = {};
for (const field of EDITABLE_FIELDS) {
const v = input[field];
if (typeof v === "string" && v.trim()) out[field] = v.trim();
}
return out as unknown as SelectorRules;
}
// Returns a list of problems; an empty list means the selectors are usable.
export function checkSelectors(rules: SelectorRules): string[] {
const problems: string[] = [];
const $ = load("<div></div>");
for (const field of EDITABLE_FIELDS) {
const value = rules[field];
if (value === undefined || value.trim() === "") {
if (field === "container" || field === "title" || field === "link") problems.push(`${field} is required.`);
continue;
}
if (value.length > 300) {
problems.push(`${field} is longer than 300 characters.`);
continue;
}
if (field === "urlParam") {
if (!/^[A-Za-z0-9_-]{1,32}$/.test(value)) problems.push("url_param must be a plain query parameter name such as uddg.");
continue;
}
try {
$(value);
} catch {
problems.push(`${field} is not a valid CSS selector: ${value}`);
}
}
return problems;
}
function readJson<T>(file: string): T | null {
try {
return JSON.parse(fs.readFileSync(file, "utf8")) as T;
} catch {
return null;
}
}
function writeJson(file: string, value: unknown): void {
fs.mkdirSync(path.dirname(file), { recursive: true });
const tmp = `${file}.tmp`;
fs.writeFileSync(tmp, JSON.stringify(value, null, 2), "utf8");
fs.renameSync(tmp, file);
}
function stamp(): string {
return new Date().toISOString().replace(/[:.]/g, "-");
}
export class RuleStore {
constructor(private dir: string, private log: (msg: string) => void = () => {}) {}
private activeFile(engine: EngineName): string {
return path.join(this.dir, "rules", `${engine}.json`);
}
private pendingFile(engine: EngineName): string {
return path.join(this.dir, "rules", "pending", `${engine}.json`);
}
private historyDir(): string {
return path.join(this.dir, "rules", "history");
}
private changeLog(): string {
return path.join(this.dir, "changes.log");
}
private logChange(line: string): void {
try {
fs.mkdirSync(this.dir, { recursive: true });
fs.appendFileSync(this.changeLog(), `${new Date().toISOString()} ${line}\n`, "utf8");
} catch {
// logging is best effort
}
}
private writeDefaults(engine: EngineName): StoredRules {
const def = ENGINES[engine];
const stored: StoredRules = {
...def.defaults,
source: "default",
defaultsVersion: def.defaultsVersion,
updatedAt: new Date().toISOString(),
};
try {
writeJson(this.activeFile(engine), stored);
} catch (e) {
this.log(`could not write rules for ${engine}: ${(e as Error).message}`);
}
return stored;
}
get(engine: EngineName): StoredRules {
const def = ENGINES[engine];
const stored = readJson<StoredRules>(this.activeFile(engine));
if (!stored || checkSelectors(stored).length) return this.writeDefaults(engine);
if (stored.source === "default" && (stored.defaultsVersion ?? 0) < def.defaultsVersion) {
return this.writeDefaults(engine);
}
return stored;
}
apply(engine: EngineName, rules: SelectorRules, reason: string): void {
const current = readJson<StoredRules>(this.activeFile(engine));
if (current) writeJson(path.join(this.historyDir(), `${engine}-${stamp()}.json`), current);
const stored: StoredRules = {
...pickSelectors(rules as unknown as Record<string, unknown>),
source: "repaired",
defaultsVersion: ENGINES[engine].defaultsVersion,
updatedAt: new Date().toISOString(),
};
writeJson(this.activeFile(engine), stored);
this.logChange(`${engine}: rules applied (${reason})`);
}
getPending(engine: EngineName): PendingRules | null {
return readJson<PendingRules>(this.pendingFile(engine));
}
setPending(engine: EngineName, rules: SelectorRules, summary: string): void {
writeJson(this.pendingFile(engine), { rules, proposedAt: new Date().toISOString(), summary });
this.logChange(`${engine}: rules proposed, waiting for approval (${summary})`);
}
listPending(): EngineName[] {
return KNOWN_ENGINES.filter((e) => this.getPending(e) !== null);
}
approve(engine: EngineName): string {
const pending = this.getPending(engine);
if (!pending) return `No proposed rules are waiting for ${engine}.`;
this.apply(engine, pending.rules, "approved by user");
fs.rmSync(this.pendingFile(engine), { force: true });
return `Approved. ${engine} now uses the proposed rules. The previous rules were saved and can be restored with rollback.`;
}
reject(engine: EngineName): string {
if (!this.getPending(engine)) return `No proposed rules are waiting for ${engine}.`;
fs.rmSync(this.pendingFile(engine), { force: true });
this.logChange(`${engine}: proposed rules rejected by user`);
return `Rejected. The proposed rules for ${engine} were discarded.`;
}
rollback(engine: EngineName): string {
let files: string[] = [];
try {
files = fs.readdirSync(this.historyDir()).filter((f) => f.startsWith(`${engine}-`) && f.endsWith(".json")).sort();
} catch {
// no history yet
}
const latest = files[files.length - 1];
if (!latest) return `There are no earlier rules for ${engine} to roll back to. Use reset to restore the built-in defaults.`;
const file = path.join(this.historyDir(), latest);
const previous = readJson<StoredRules>(file);
if (!previous || checkSelectors(previous).length) return `The saved rules in ${latest} are unreadable.`;
writeJson(this.activeFile(engine), previous);
fs.rmSync(file, { force: true });
this.logChange(`${engine}: rolled back to rules from ${latest}`);
return `Rolled back. ${engine} now uses the rules it had before the last change.`;
}
reset(engine: EngineName): string {
const current = readJson<StoredRules>(this.activeFile(engine));
if (current) writeJson(path.join(this.historyDir(), `${engine}-${stamp()}.json`), current);
this.writeDefaults(engine);
this.logChange(`${engine}: reset to built-in defaults`);
return `Reset. ${engine} now uses the built-in default rules.`;
}
}