crawler

import * as cheerio from "cheerio";

import { retryFetch }
from "../../services/retryFetch";

import type { SearchProvider }
from "./SearchProvider";

function decodeUrl(
    href: string
): string {
    try {
        const parsed = new URL(
            href,
            "https://duckduckgo.com"
        );

        const uddg =
            parsed.searchParams.get(
                "uddg"
            );

        return uddg
            ? decodeURIComponent(
                uddg
            )
            : href;
    } catch {
        return href;
    }
}

export class DuckDuckGoProvider
implements SearchProvider {

    async search(
        query: string
    ): Promise<string[]> {

        const url = new URL(
            "https://html.duckduckgo.com/html/"
        );

        url.searchParams.set(
            "q",
            query
        );

        const html =
            await retryFetch(
                url.toString()
            );

        const $ =
            cheerio.load(html);

        const results: string[] = [];

        $("a").each((_, el) => {
            const href =
                $(el).attr("href");

            if (!href) {
                return;
            }

            const decoded =
                decodeUrl(href);

            if (
                decoded.startsWith(
                    "http"
                ) &&
                !results.includes(
                    decoded
                )
            ) {
                results.push(
                    decoded
                );
            }
        });

        return results.slice(0, 5);
    }
}