crawler

import { tool } from "@lmstudio/sdk";
import { z } from "zod";
import pLimit from "p-limit";

import { DuckDuckGoProvider }
from "../search/providers/DuckDuckGoProvider";

import { crawlPage }
from "../crawler/crawlPage";

const provider =
    new DuckDuckGoProvider();

const limit = pLimit(2);

export const searchWebTool =
    tool({
        name: "searchWeb",

        description:
            "Advanced optimized web search.",

        parameters: {
            query: z.string()
        },

        implementation: async ({
            query
        }: {
            query: string;
        }) => {

            const urls =
                await provider.search(
                    query
                );

            const pages =
                await Promise.allSettled(
                    urls.map(url =>
                        limit(() =>
                            crawlPage(
                                url,
                                query
                            )
                        )
                    )
                );

            const successful =
                pages
                    .filter(
                        page =>
                            page.status ===
                            "fulfilled"
                    )
                    .map(
                        page =>
                            page.value
                    );

            return successful
                .map(
                    (
                        page,
                        index
                    ) =>
                        [
                            `Result ${index + 1}`,
                            `Title: ${page.title}`,
                            `URL: ${page.url}`,
                            "",
                            page.content
                        ].join("\n")
                )
                .join(
                    "\n\n====================\n\n"
                );
        }
    });