src / providers / DuckDuckGoProvider.ts
src / providers / DuckDuckGoProvider.ts
import { HTTPClient } from "../http/HTTPClient";
import { ProviderError } from "../models/errors";
import { SearchOptions } from "../types/search";
import { KeyManager } from "../utils/KeyManager";
import { BaseProvider } from "./BaseProvider";
import { InternalProviderResult } from "./types/InternalProviderResult";
export const DDG_WEB_SEARCH_URL = "https://html.duckduckgo.com/html/";
export interface DuckDuckGoProviderOptions {
keys?: string[];
keyManager?: KeyManager;
httpClient?: HTTPClient;
cooldownHours?: number;
}
/**
* DuckDuckGo Web Search provider implementation.
* Queries DuckDuckGo HTML web search endpoint and extracts candidate result items.
* Requires no API keys.
* Mapped from Python `providers/duckduckgo.py` DuckDuckGoProvider.
*/
export class DuckDuckGoProvider extends BaseProvider {
public static readonly PROVIDER_NAME = "duckduckgo";
public static readonly DISPLAY_NAME = "DuckDuckGo";
private httpClient: HTTPClient;
constructor(options: DuckDuckGoProviderOptions = {}) {
super({
providerName: DuckDuckGoProvider.PROVIDER_NAME,
displayName: DuckDuckGoProvider.DISPLAY_NAME,
capabilities: ["web"],
keys: options.keys,
keyManager: options.keyManager,
cooldownHours: options.cooldownHours,
});
this.httpClient = options.httpClient ?? new HTTPClient();
}
/**
* Execute web search using DuckDuckGo HTML search endpoint.
*/
public async search(query: string, options: SearchOptions = {}): Promise<InternalProviderResult[]> {
if (!query || !query.trim()) {
return [];
}
const cleanQuery = query.trim();
const numResults = options.numResults ?? 10;
let response;
try {
response = await this.httpClient.post(DDG_WEB_SEARCH_URL, {
body: `q=${encodeURIComponent(cleanQuery)}`,
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
},
});
} catch (exc: unknown) {
const err = exc instanceof Error ? exc : new Error(String(exc));
throw new ProviderError(`DuckDuckGo web search HTTP request failed: ${err.message}`);
}
this._lastHttpResponseBytes = response.buffer?.length ?? Buffer.byteLength(response.text ?? "", "utf-8");
if (response.statusCode !== 200 || !response.text) {
throw new ProviderError(
`DuckDuckGo web search failed with status code ${response.statusCode}`
);
}
const results = this.parseHtmlResults(response.text);
this._lastRawResultsCount = results.length;
return this.enforceResultLimit(results, numResults);
}
private parseHtmlResults(html: string): InternalProviderResult[] {
const results: InternalProviderResult[] = [];
const resultRegex =
/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?(?:<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>|<div[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/div>)/gi;
let match: RegExpExecArray | null;
while ((match = resultRegex.exec(html)) !== null) {
let rawUrl = match[1] || "";
const rawTitle = match[2] || "";
const rawSnippet = match[3] || match[4] || "";
if (rawUrl.includes("uddg=")) {
try {
const u = new URL("https://duckduckgo.com" + rawUrl);
const uddg = u.searchParams.get("uddg");
if (uddg) rawUrl = uddg;
} catch {
// Keep rawUrl
}
}
results.push({
title: rawTitle,
url: rawUrl.trim(),
snippetCandidate: rawSnippet,
source: this.providerName,
debug: { snippetSource: "native_snippet" },
});
}
return results;
}
}
import { HTTPClient } from "../http/HTTPClient";
import { ProviderError } from "../models/errors";
import { SearchOptions } from "../types/search";
import { KeyManager } from "../utils/KeyManager";
import { BaseProvider } from "./BaseProvider";
import { InternalProviderResult } from "./types/InternalProviderResult";
export const DDG_WEB_SEARCH_URL = "https://html.duckduckgo.com/html/";
export interface DuckDuckGoProviderOptions {
keys?: string[];
keyManager?: KeyManager;
httpClient?: HTTPClient;
cooldownHours?: number;
}
/**
* DuckDuckGo Web Search provider implementation.
* Queries DuckDuckGo HTML web search endpoint and extracts candidate result items.
* Requires no API keys.
* Mapped from Python `providers/duckduckgo.py` DuckDuckGoProvider.
*/
export class DuckDuckGoProvider extends BaseProvider {
public static readonly PROVIDER_NAME = "duckduckgo";
public static readonly DISPLAY_NAME = "DuckDuckGo";
private httpClient: HTTPClient;
constructor(options: DuckDuckGoProviderOptions = {}) {
super({
providerName: DuckDuckGoProvider.PROVIDER_NAME,
displayName: DuckDuckGoProvider.DISPLAY_NAME,
capabilities: ["web"],
keys: options.keys,
keyManager: options.keyManager,
cooldownHours: options.cooldownHours,
});
this.httpClient = options.httpClient ?? new HTTPClient();
}
/**
* Execute web search using DuckDuckGo HTML search endpoint.
*/
public async search(query: string, options: SearchOptions = {}): Promise<InternalProviderResult[]> {
if (!query || !query.trim()) {
return [];
}
const cleanQuery = query.trim();
const numResults = options.numResults ?? 10;
let response;
try {
response = await this.httpClient.post(DDG_WEB_SEARCH_URL, {
body: `q=${encodeURIComponent(cleanQuery)}`,
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
},
});
} catch (exc: unknown) {
const err = exc instanceof Error ? exc : new Error(String(exc));
throw new ProviderError(`DuckDuckGo web search HTTP request failed: ${err.message}`);
}
this._lastHttpResponseBytes = response.buffer?.length ?? Buffer.byteLength(response.text ?? "", "utf-8");
if (response.statusCode !== 200 || !response.text) {
throw new ProviderError(
`DuckDuckGo web search failed with status code ${response.statusCode}`
);
}
const results = this.parseHtmlResults(response.text);
this._lastRawResultsCount = results.length;
return this.enforceResultLimit(results, numResults);
}
private parseHtmlResults(html: string): InternalProviderResult[] {
const results: InternalProviderResult[] = [];
const resultRegex =
/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?(?:<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>|<div[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/div>)/gi;
let match: RegExpExecArray | null;
while ((match = resultRegex.exec(html)) !== null) {
let rawUrl = match[1] || "";
const rawTitle = match[2] || "";
const rawSnippet = match[3] || match[4] || "";
if (rawUrl.includes("uddg=")) {
try {
const u = new URL("https://duckduckgo.com" + rawUrl);
const uddg = u.searchParams.get("uddg");
if (uddg) rawUrl = uddg;
} catch {
// Keep rawUrl
}
}
results.push({
title: rawTitle,
url: rawUrl.trim(),
snippetCandidate: rawSnippet,
source: this.providerName,
debug: { snippetSource: "native_snippet" },
});
}
return results;
}
}