web-fetcher-plugin

import { tool, Tool, ToolsProviderController } from "@lmstudio/sdk";
import { z } from "zod";
import * as fetcher from "./fetcher";
import * as parser from "./parser";
import { configSchematics, WebFetcherConfig } from "./config";

function getWebConfig(ctl: ToolsProviderController): WebFetcherConfig {
  const config = ctl.getPluginConfig(configSchematics);
  return {
    apiKey: config.get("apiKey"),
    requestDelaySeconds: config.get("requestDelaySeconds"),
    userAgent: config.get("userAgent"),
    maxRetries: config.get("maxRetries"),
  };
}

export async function toolsProvider(ctl: ToolsProviderController) {
  const config = getWebConfig(ctl);
  const tools: Tool[] = [];

  tools.push(
    tool({
      name: "fetch_webpage",
      description: "Fetch a webpage from a URL and return cleaned text content.",
      parameters: {
        url: z.string().url().describe("The full URL to fetch."),
      },
      implementation: async ({ url }) => {
        try {
          return await fetcher.fetchAndClean(url, config);
        } catch (error) {
          return {
            success: false,
            error: String(error),
          };
        }
      },
    }),
  );

  tools.push(
    tool({
      name: "parse_html",
      description:
        "Parse raw HTML and extract headings, paragraphs, links, and metadata.",
      parameters: {
        html: z.string().describe("HTML content to parse."),
      },
      implementation: async ({ html }) => {
        try {
          const parsed = await parser.parseHtml(html);
          return {
            success: true,
            data: parsed,
          };
        } catch (error) {
          return {
            success: false,
            error: String(error),
          };
        }
      },
    }),
  );

  tools.push(
    tool({
      name: "extract_elements",
      description: "Extract HTML elements using a CSS selector.",
      parameters: {
        html: z.string().describe("HTML content to parse."),
        selector: z.string().describe("CSS selector to match."),
      },
      implementation: async ({ html, selector }) => {
        try {
          const elements = parser.extractElementsBySelector(html, selector);
          return {
            success: true,
            count: elements.length,
            elements,
          };
        } catch (error) {
          return {
            success: false,
            error: String(error),
          };
        }
      },
    }),
  );

  tools.push(
    tool({
      name: "search_in_webpage",
      description:
        "Fetch a webpage and search for keywords, returning matches and context snippets.",
      parameters: {
        url: z.string().url().describe("The URL to fetch and search."),
        keywords: z.array(z.string()).min(1).describe("Keywords to search for."),
      },
      implementation: async ({ url, keywords }) => {
        try {
          const fetchResult = await fetcher.fetchWebpage(url, config);
          if (!fetchResult.success || !fetchResult.html) {
            return {
              success: false,
              error: fetchResult.error,
            };
          }

          const results = parser.searchInHtml(fetchResult.html, keywords);
          return {
            success: true,
            url,
            results,
          };
        } catch (error) {
          return {
            success: false,
            error: String(error),
          };
        }
      },
    }),
  );

  return tools;
}