crawler

import { tool } from "@lmstudio/sdk";
import { z } from "zod";
import { crawlPage } from "../crawler/crawlPage";
import { extractStructuredElements } from "../services/structuredExtraction";
import { MAX_CONTENT_LENGTH } from "../utils/constants";

export const crawlSpecificPageTool =
    tool({
        name: "crawlSpecificPage",

        description:
            "Crawl one webpage directly using clean structural layout processing.",

        parameters: {
            url: z.string().url()
        },

        implementation: async ({
            url
        }: {
            url: string;
        }) => {
            const page = await crawlPage(url);

            // Run linear structural text block assembly
            const structuredContent = extractStructuredElements(page.rawHtml);

            // Use structured blocks if present; otherwise use the high-capacity fallback array
            let finalContent = structuredContent.length > 0 
                ? structuredContent.join("\n\n") 
                : page.content;

            if (finalContent.length > MAX_CONTENT_LENGTH) {
                finalContent = finalContent.slice(0, MAX_CONTENT_LENGTH);
            }

            return [
                `Title: ${page.title}`,
                `URL: ${page.url}`,
                "",
                finalContent
            ].join("\n");
        }
    });