crawler

import { tool } from "@lmstudio/sdk";
import { z } from "zod";

import { crawlPage } from "../crawler/crawlPage";

export const crawlWebPageTool = tool({
    name: "crawlWebPage",

    description:
        "Extract readable webpage content.",

    parameters: {
        url: z
            .string()
            .url()
            .describe("Webpage URL")
    },

    implementation: async ({
        url
    }: {
        url: string;
    }) => {
        try {
            const page =
                await crawlPage(url);

            return [
                `TITLE: ${page.title}`,
                `URL: ${page.url}`,
                "",
                `CONTENT:`,
                page.content,
                "",
                `LINKS:`,
                page.links
                    .slice(0, 20)
                    .join("\n")
            ].join("\n");
        } catch (error) {
            return `Crawl failed: ${
                error instanceof Error
                    ? error.message
                    : "Unknown error"
            }`;
        }
    }
});