anonymize

// Ad-hoc diagnostic: surface a sample of TEL false positives on AI4Privacy FR.
// Local-only output (stdout). Do not redirect to a versioned file.
//
// Run: npm run eval:inspect-tel

import { createReadStream, existsSync } from "node:fs";
import { createInterface } from "node:readline";
import { detectAll } from "../src/detectors.ts";

type GoldSpan = { start: number; end: number; label: string; value: string };
type Sample = { source_text: string; locale: string; privacy_mask: GoldSpan[] };

const FILES = [
  "datasets/ai4privacy-400k/data/validation/fr.jsonl",
  "datasets/ai4privacy-400k/data/train/fr.jsonl",
];

const MAX_SAMPLES = 25;
const CONTEXT = 30;

function overlap(a: { start: number; end: number }, b: { start: number; end: number }): boolean {
  return a.start < b.end && b.start < a.end;
}

async function main() {
  const examples: { pred_value: string; pred_span: string; nearby_gold_labels: string[]; context: string }[] = [];

  for (const path of FILES) {
    if (!existsSync(path)) continue;
    if (examples.length >= MAX_SAMPLES) break;
    const rl = createInterface({ input: createReadStream(path, { encoding: "utf-8" }), crlfDelay: Infinity });
    for await (const line of rl) {
      if (examples.length >= MAX_SAMPLES) break;
      if (!line) continue;
      const sample = JSON.parse(line) as Sample;
      if (sample.locale !== "FR") continue;
      const predicted = detectAll(sample.source_text).filter((s) => s.type === "TEL");
      if (predicted.length === 0) continue;

      const telGold = sample.privacy_mask.filter((g) => g.label === "TELEPHONENUM");
      for (const p of predicted) {
        if (examples.length >= MAX_SAMPLES) break;
        const isTP = telGold.some((g) => overlap(p, g));
        if (isTP) continue;
        const overlapping = sample.privacy_mask.filter((g) => overlap(p, g)).map((g) => g.label);
        const start = Math.max(0, p.start - CONTEXT);
        const end = Math.min(sample.source_text.length, p.end + CONTEXT);
        examples.push({
          pred_value: p.value,
          pred_span: `${p.start}-${p.end}`,
          nearby_gold_labels: overlapping,
          context: sample.source_text.slice(start, end).replace(/\s+/g, " "),
        });
      }
    }
  }

  console.log(`Sample of ${examples.length} TEL false positives:\n`);
  for (const [i, ex] of examples.entries()) {
    const labels = ex.nearby_gold_labels.length ? `[gold-overlap: ${ex.nearby_gold_labels.join(",")}]` : "[no gold overlap]";
    console.log(`${String(i + 1).padStart(2)}. value=${JSON.stringify(ex.pred_value)} ${labels}`);
    console.log(`    …${ex.context}…`);
    console.log("");
  }
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});