crawler

export function cleanText(value: string | undefined | null): string {
  return (value ?? "")
    .replace(/\u00a0/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

export function truncateText(value: string, maxChars: number): string {
  if (value.length <= maxChars) {
    return value;
  }

  return `${value.slice(0, maxChars).trimEnd()}\n\n[truncated]`;
}

export function uniqueStrings(values: string[]): string[] {
  const seen = new Set<string>();
  const result: string[] = [];

  for (const value of values) {
    const normalized = cleanText(value);
    if (!normalized) continue;

    const key = normalized.toLowerCase();
    if (seen.has(key)) continue;

    seen.add(key);
    result.push(normalized);
  }

  return result;
}

export function isLikelyBoilerplate(text: string): boolean {
  const lower = text.toLowerCase();
  return [
    "cookie",
    "privacy policy",
    "terms of service",
    "all rights reserved",
    "log in",
    "sign in",
    "subscribe",
    "newsletter",
    "accept cookies",
    "advertisement",
  ].some((fragment) => lower.includes(fragment));
}