src / providers / FirecrawlSearchProvider.ts
src / providers / FirecrawlSearchProvider.ts
import { HTTPClient, JsonObject } from "../http/HTTPClient";
import { ProviderError } from "../models/errors";
import { SearchOptions } from "../types/search";
import { KeyManager } from "../utils/KeyManager";
import { BaseProvider } from "./BaseProvider";
import { InternalProviderResult } from "./types/InternalProviderResult";
export const FIRECRAWL_SEARCH_URL = "https://api.firecrawl.dev/v1/search";
export interface FirecrawlSearchProviderOptions {
keys?: string[];
keyManager?: KeyManager;
httpClient?: HTTPClient;
cooldownHours?: number;
}
/**
* Firecrawl web search provider implementation.
* Queries Firecrawl v1 Search API endpoint and extracts candidate result items.
*/
export class FirecrawlSearchProvider extends BaseProvider {
public static readonly PROVIDER_NAME = "firecrawl";
public static readonly DISPLAY_NAME = "Firecrawl";
private httpClient: HTTPClient;
constructor(options: FirecrawlSearchProviderOptions = {}) {
super({
providerName: FirecrawlSearchProvider.PROVIDER_NAME,
displayName: FirecrawlSearchProvider.DISPLAY_NAME,
capabilities: ["web"],
keys: options.keys,
keyManager: options.keyManager,
cooldownHours: options.cooldownHours,
});
this._rotationStatusCodes = [401, 403, 429];
this.httpClient = options.httpClient ?? new HTTPClient();
}
/**
* Execute web search using Firecrawl Search API endpoint with key rotation.
*/
public async search(query: string, options: SearchOptions = {}): Promise<InternalProviderResult[]> {
if (!query || !query.trim()) {
return [];
}
const numResults = options.numResults ?? 10;
const { apiKey: _k, numResults: _n, provider: _p, ...extraKwargs } = options;
const doSearch = async (apiKey: string): Promise<InternalProviderResult[]> => {
const headers: Record<string, string> = {
Authorization: `Bearer ${apiKey.trim()}`,
"Content-Type": "application/json",
};
const payload: JsonObject = {
query: query.trim(),
limit: numResults,
...extraKwargs,
};
let response;
try {
response = await this.httpClient.post(FIRECRAWL_SEARCH_URL, {
json: payload,
headers,
});
} catch (exc: unknown) {
const err = exc instanceof Error ? exc : new Error(String(exc));
throw new ProviderError(`Firecrawl search HTTP request failed: ${err.message}`);
}
const data = this.parseJsonResponse<Record<string, unknown>>(response);
const rawResults = data.data ?? data.results;
if (rawResults === undefined || rawResults === null) {
return [];
}
if (!Array.isArray(rawResults)) {
throw new ProviderError(
"Invalid response payload from Firecrawl API: 'data' field must be a list"
);
}
this._lastRawResultsCount = rawResults.length;
const candidates = rawResults.map((item: unknown) =>
this.extractCandidate((item as Record<string, unknown>) || {})
);
return this.enforceResultLimit(candidates, numResults);
};
return this.executeWithKeyRotation(doSearch);
}
private extractCandidate(rawItem: Record<string, unknown>): InternalProviderResult {
const metadata = (rawItem.metadata as Record<string, unknown>) || {};
const title =
(rawItem.title as string) ||
(metadata.title as string) ||
"";
const url =
(rawItem.url as string) ||
(metadata.sourceURL as string) ||
(metadata.url as string) ||
"";
const snippetCandidate =
(rawItem.description as string) ||
(metadata.description as string) ||
(rawItem.snippet as string) ||
"";
const score = typeof rawItem.score === "number" ? rawItem.score : undefined;
return {
title: String(title),
url: String(url),
snippetCandidate,
source: this.providerName,
score,
raw: rawItem,
debug: { snippetSource: "description" },
};
}
}
import { HTTPClient, JsonObject } from "../http/HTTPClient";
import { ProviderError } from "../models/errors";
import { SearchOptions } from "../types/search";
import { KeyManager } from "../utils/KeyManager";
import { BaseProvider } from "./BaseProvider";
import { InternalProviderResult } from "./types/InternalProviderResult";
export const FIRECRAWL_SEARCH_URL = "https://api.firecrawl.dev/v1/search";
export interface FirecrawlSearchProviderOptions {
keys?: string[];
keyManager?: KeyManager;
httpClient?: HTTPClient;
cooldownHours?: number;
}
/**
* Firecrawl web search provider implementation.
* Queries Firecrawl v1 Search API endpoint and extracts candidate result items.
*/
export class FirecrawlSearchProvider extends BaseProvider {
public static readonly PROVIDER_NAME = "firecrawl";
public static readonly DISPLAY_NAME = "Firecrawl";
private httpClient: HTTPClient;
constructor(options: FirecrawlSearchProviderOptions = {}) {
super({
providerName: FirecrawlSearchProvider.PROVIDER_NAME,
displayName: FirecrawlSearchProvider.DISPLAY_NAME,
capabilities: ["web"],
keys: options.keys,
keyManager: options.keyManager,
cooldownHours: options.cooldownHours,
});
this._rotationStatusCodes = [401, 403, 429];
this.httpClient = options.httpClient ?? new HTTPClient();
}
/**
* Execute web search using Firecrawl Search API endpoint with key rotation.
*/
public async search(query: string, options: SearchOptions = {}): Promise<InternalProviderResult[]> {
if (!query || !query.trim()) {
return [];
}
const numResults = options.numResults ?? 10;
const { apiKey: _k, numResults: _n, provider: _p, ...extraKwargs } = options;
const doSearch = async (apiKey: string): Promise<InternalProviderResult[]> => {
const headers: Record<string, string> = {
Authorization: `Bearer ${apiKey.trim()}`,
"Content-Type": "application/json",
};
const payload: JsonObject = {
query: query.trim(),
limit: numResults,
...extraKwargs,
};
let response;
try {
response = await this.httpClient.post(FIRECRAWL_SEARCH_URL, {
json: payload,
headers,
});
} catch (exc: unknown) {
const err = exc instanceof Error ? exc : new Error(String(exc));
throw new ProviderError(`Firecrawl search HTTP request failed: ${err.message}`);
}
const data = this.parseJsonResponse<Record<string, unknown>>(response);
const rawResults = data.data ?? data.results;
if (rawResults === undefined || rawResults === null) {
return [];
}
if (!Array.isArray(rawResults)) {
throw new ProviderError(
"Invalid response payload from Firecrawl API: 'data' field must be a list"
);
}
this._lastRawResultsCount = rawResults.length;
const candidates = rawResults.map((item: unknown) =>
this.extractCandidate((item as Record<string, unknown>) || {})
);
return this.enforceResultLimit(candidates, numResults);
};
return this.executeWithKeyRotation(doSearch);
}
private extractCandidate(rawItem: Record<string, unknown>): InternalProviderResult {
const metadata = (rawItem.metadata as Record<string, unknown>) || {};
const title =
(rawItem.title as string) ||
(metadata.title as string) ||
"";
const url =
(rawItem.url as string) ||
(metadata.sourceURL as string) ||
(metadata.url as string) ||
"";
const snippetCandidate =
(rawItem.description as string) ||
(metadata.description as string) ||
(rawItem.snippet as string) ||
"";
const score = typeof rawItem.score === "number" ? rawItem.score : undefined;
return {
title: String(title),
url: String(url),
snippetCandidate,
source: this.providerName,
score,
raw: rawItem,
debug: { snippetSource: "description" },
};
}
}