src / providers / BaseProvider.ts
src / providers / BaseProvider.ts
import { Config } from "../config/Config";
import { HTTPResponse } from "../http/HTTPClient";
import { KeyAuthOrQuotaError, NoAPIKeysError, ProviderError } from "../models/errors";
import { ProviderCapability } from "../types/provider";
import { SearchOptions } from "../types/search";
import { InternalProviderResult } from "./types/InternalProviderResult";
import { KeyManager } from "../utils/KeyManager";
import { Logger } from "../utils/Logger";
import { ISyncableProviderInstance } from "./ISyncableProviderInstance";
import { defaultProviderInstanceRegistry } from "./ProviderInstanceRegistry";
export interface BaseProviderOptions {
providerName: string;
displayName?: string;
capabilities?: ProviderCapability[];
keys?: string[];
keyManager?: KeyManager;
cooldownHours?: number;
}
/**
* Abstract base class for all search providers.
* Manages provider identity, capabilities, enabled state, API key rotation via KeyManager, and response parsing.
* Mapped from Python `providers/base_provider.py` BaseProvider.
*/
export abstract class BaseProvider implements ISyncableProviderInstance {
public readonly category: string = "web";
protected readonly _providerName: string;
protected readonly _displayName: string;
protected readonly _capabilities: Set<ProviderCapability>;
protected readonly _keyManager: KeyManager;
protected _enabled: boolean = true;
protected _rotationStatusCodes: number[] = [401, 403];
constructor(
providerNameOrOptions: string | BaseProviderOptions,
displayName?: string,
capabilities: ProviderCapability[] = ["web"]
) {
if (typeof providerNameOrOptions === "string") {
const pName = providerNameOrOptions;
if (!pName || !pName.trim()) {
throw new Error("Provider name cannot be empty.");
}
this._providerName = pName.toLowerCase().trim();
this._displayName = displayName ?? pName;
this._capabilities = new Set(capabilities);
this._keyManager = new KeyManager({ providerName: this._providerName });
} else {
const options = providerNameOrOptions;
if (!options.providerName || !options.providerName.trim()) {
throw new Error("Provider name cannot be empty.");
}
this._providerName = options.providerName.toLowerCase().trim();
this._displayName = options.displayName ?? options.providerName;
this._capabilities = new Set(options.capabilities ?? ["web"]);
this._keyManager =
options.keyManager ??
new KeyManager({
providerName: this._providerName,
keys: options.keys,
defaultCooldownHours: options.cooldownHours ?? 24.0,
});
}
// Auto-register this provider instance with global ProviderInstanceRegistry
defaultProviderInstanceRegistry.register(this);
}
public get configIdentity(): string {
return `${this.category}.${this._providerName}`;
}
public get providerName(): string {
return this._providerName;
}
public get displayName(): string {
return this._displayName;
}
public get capabilities(): ProviderCapability[] {
return Array.from(this._capabilities);
}
public get rotationStatusCodes(): number[] {
return this._rotationStatusCodes;
}
public isEnabled(): boolean {
return this._enabled;
}
public enable(): void {
this._enabled = true;
}
public disable(): void {
this._enabled = false;
}
public hasCapability(capability: ProviderCapability): boolean {
return this._capabilities.has(capability);
}
public hasKeys(): boolean {
return this._keyManager.hasKeys();
}
public setKeys(keys: string[]): void {
this._keyManager.setKeys(keys);
}
public currentKey(): string {
return this._keyManager.currentKey();
}
public nextKey(): string {
return this._keyManager.nextKey();
}
public validate(): boolean {
return Boolean(
this._providerName &&
this._providerName.trim().length > 0
);
}
protected _lastHttpResponseBytes: number = 0;
protected _lastRawResultsCount: number = 0;
public get lastHttpResponseBytes(): number {
return this._lastHttpResponseBytes;
}
public get lastRawResultsCount(): number {
return this._lastRawResultsCount;
}
public setLastHttpResponseBytes(bytes: number): void {
this._lastHttpResponseBytes = bytes;
}
public setLastRawResultsCount(count: number): void {
this._lastRawResultsCount = count;
}
/**
* Validate HTTP response status and decode JSON payload.
* Mapped from Python `BaseProvider._parse_json_response()`.
*/
protected parseJsonResponse<T = Record<string, unknown>>(response: HTTPResponse): T {
this._lastHttpResponseBytes = response.buffer?.length ?? Buffer.byteLength(response.text ?? "", "utf-8");
const providerLabel =
this._providerName.charAt(0).toUpperCase() + this._providerName.slice(1);
if (response.statusCode !== 200) {
if (this._rotationStatusCodes.includes(response.statusCode)) {
throw new KeyAuthOrQuotaError(
`${providerLabel} search failed with status code ${response.statusCode}: ${response.text}`,
response.statusCode
);
}
throw new ProviderError(
`${providerLabel} search failed with status code ${response.statusCode}: ${response.text}`
);
}
const data = response.data;
if (!data || typeof data !== "object") {
throw new ProviderError(
`Invalid response payload from ${providerLabel} API: expected JSON object`
);
}
return data as T;
}
/**
* Defensively slice result array to guarantee returning at most maxResults items.
*/
protected enforceResultLimit<T>(results: T[], maxResults?: number): T[] {
if (typeof maxResults === "number" && maxResults >= 0) {
return results.slice(0, maxResults);
}
return results;
}
/**
* Execute request function with automatic API key rotation on auth/quota errors.
* Mapped from Python `BaseProvider._execute_with_key_rotation()`.
*/
protected async executeWithKeyRotation<T>(
requestFn: (apiKey: string) => Promise<T>
): Promise<T> {
const totalKeys = this._keyManager.keyCount();
if (totalKeys === 0) {
throw new NoAPIKeysError(
`No API keys configured for provider '${this._providerName}'`
);
}
let lastKeyError: KeyAuthOrQuotaError | null = null;
let lastGeneralError: Error | null = null;
for (let attempt = 0; attempt < totalKeys; attempt++) {
const activeKey = this.currentKey();
try {
const result = await requestFn(activeKey);
this._keyManager.markKeySuccess(activeKey);
return result;
} catch (err: unknown) {
if (err instanceof KeyAuthOrQuotaError) {
lastKeyError = err;
if (err.statusCode === 401) {
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} received status 401 (Unauthorized). Disabling key permanently...`
);
this._keyManager.markKeyInvalid(activeKey);
} else {
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} received status ${err.statusCode} (Quota/Rate Limit). Disabling key (cooldown active)...`
);
this._keyManager.markKeyExhausted(activeKey, err.retryAfter);
}
if (attempt < totalKeys - 1) {
this.nextKey();
}
} else {
lastGeneralError = err instanceof Error ? err : new Error(String(err));
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} request failed (${lastGeneralError.message}). Trying next available key...`
);
if (attempt < totalKeys - 1) {
this.nextKey();
} else {
throw lastGeneralError;
}
}
}
}
if (lastGeneralError) {
throw lastGeneralError;
}
const lastStatus = lastKeyError ? lastKeyError.statusCode : "Unknown";
const msg = `All ${totalKeys} configured API keys for provider '${this._providerName}' failed authentication or quota limits. Last status: ${lastStatus}`;
throw new ProviderError(msg);
}
public abstract search(query: string, options?: SearchOptions): Promise<InternalProviderResult[]>;
}
import { Config } from "../config/Config";
import { HTTPResponse } from "../http/HTTPClient";
import { KeyAuthOrQuotaError, NoAPIKeysError, ProviderError } from "../models/errors";
import { ProviderCapability } from "../types/provider";
import { SearchOptions } from "../types/search";
import { InternalProviderResult } from "./types/InternalProviderResult";
import { KeyManager } from "../utils/KeyManager";
import { Logger } from "../utils/Logger";
import { ISyncableProviderInstance } from "./ISyncableProviderInstance";
import { defaultProviderInstanceRegistry } from "./ProviderInstanceRegistry";
export interface BaseProviderOptions {
providerName: string;
displayName?: string;
capabilities?: ProviderCapability[];
keys?: string[];
keyManager?: KeyManager;
cooldownHours?: number;
}
/**
* Abstract base class for all search providers.
* Manages provider identity, capabilities, enabled state, API key rotation via KeyManager, and response parsing.
* Mapped from Python `providers/base_provider.py` BaseProvider.
*/
export abstract class BaseProvider implements ISyncableProviderInstance {
public readonly category: string = "web";
protected readonly _providerName: string;
protected readonly _displayName: string;
protected readonly _capabilities: Set<ProviderCapability>;
protected readonly _keyManager: KeyManager;
protected _enabled: boolean = true;
protected _rotationStatusCodes: number[] = [401, 403];
constructor(
providerNameOrOptions: string | BaseProviderOptions,
displayName?: string,
capabilities: ProviderCapability[] = ["web"]
) {
if (typeof providerNameOrOptions === "string") {
const pName = providerNameOrOptions;
if (!pName || !pName.trim()) {
throw new Error("Provider name cannot be empty.");
}
this._providerName = pName.toLowerCase().trim();
this._displayName = displayName ?? pName;
this._capabilities = new Set(capabilities);
this._keyManager = new KeyManager({ providerName: this._providerName });
} else {
const options = providerNameOrOptions;
if (!options.providerName || !options.providerName.trim()) {
throw new Error("Provider name cannot be empty.");
}
this._providerName = options.providerName.toLowerCase().trim();
this._displayName = options.displayName ?? options.providerName;
this._capabilities = new Set(options.capabilities ?? ["web"]);
this._keyManager =
options.keyManager ??
new KeyManager({
providerName: this._providerName,
keys: options.keys,
defaultCooldownHours: options.cooldownHours ?? 24.0,
});
}
// Auto-register this provider instance with global ProviderInstanceRegistry
defaultProviderInstanceRegistry.register(this);
}
public get configIdentity(): string {
return `${this.category}.${this._providerName}`;
}
public get providerName(): string {
return this._providerName;
}
public get displayName(): string {
return this._displayName;
}
public get capabilities(): ProviderCapability[] {
return Array.from(this._capabilities);
}
public get rotationStatusCodes(): number[] {
return this._rotationStatusCodes;
}
public isEnabled(): boolean {
return this._enabled;
}
public enable(): void {
this._enabled = true;
}
public disable(): void {
this._enabled = false;
}
public hasCapability(capability: ProviderCapability): boolean {
return this._capabilities.has(capability);
}
public hasKeys(): boolean {
return this._keyManager.hasKeys();
}
public setKeys(keys: string[]): void {
this._keyManager.setKeys(keys);
}
public currentKey(): string {
return this._keyManager.currentKey();
}
public nextKey(): string {
return this._keyManager.nextKey();
}
public validate(): boolean {
return Boolean(
this._providerName &&
this._providerName.trim().length > 0
);
}
protected _lastHttpResponseBytes: number = 0;
protected _lastRawResultsCount: number = 0;
public get lastHttpResponseBytes(): number {
return this._lastHttpResponseBytes;
}
public get lastRawResultsCount(): number {
return this._lastRawResultsCount;
}
public setLastHttpResponseBytes(bytes: number): void {
this._lastHttpResponseBytes = bytes;
}
public setLastRawResultsCount(count: number): void {
this._lastRawResultsCount = count;
}
/**
* Validate HTTP response status and decode JSON payload.
* Mapped from Python `BaseProvider._parse_json_response()`.
*/
protected parseJsonResponse<T = Record<string, unknown>>(response: HTTPResponse): T {
this._lastHttpResponseBytes = response.buffer?.length ?? Buffer.byteLength(response.text ?? "", "utf-8");
const providerLabel =
this._providerName.charAt(0).toUpperCase() + this._providerName.slice(1);
if (response.statusCode !== 200) {
if (this._rotationStatusCodes.includes(response.statusCode)) {
throw new KeyAuthOrQuotaError(
`${providerLabel} search failed with status code ${response.statusCode}: ${response.text}`,
response.statusCode
);
}
throw new ProviderError(
`${providerLabel} search failed with status code ${response.statusCode}: ${response.text}`
);
}
const data = response.data;
if (!data || typeof data !== "object") {
throw new ProviderError(
`Invalid response payload from ${providerLabel} API: expected JSON object`
);
}
return data as T;
}
/**
* Defensively slice result array to guarantee returning at most maxResults items.
*/
protected enforceResultLimit<T>(results: T[], maxResults?: number): T[] {
if (typeof maxResults === "number" && maxResults >= 0) {
return results.slice(0, maxResults);
}
return results;
}
/**
* Execute request function with automatic API key rotation on auth/quota errors.
* Mapped from Python `BaseProvider._execute_with_key_rotation()`.
*/
protected async executeWithKeyRotation<T>(
requestFn: (apiKey: string) => Promise<T>
): Promise<T> {
const totalKeys = this._keyManager.keyCount();
if (totalKeys === 0) {
throw new NoAPIKeysError(
`No API keys configured for provider '${this._providerName}'`
);
}
let lastKeyError: KeyAuthOrQuotaError | null = null;
let lastGeneralError: Error | null = null;
for (let attempt = 0; attempt < totalKeys; attempt++) {
const activeKey = this.currentKey();
try {
const result = await requestFn(activeKey);
this._keyManager.markKeySuccess(activeKey);
return result;
} catch (err: unknown) {
if (err instanceof KeyAuthOrQuotaError) {
lastKeyError = err;
if (err.statusCode === 401) {
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} received status 401 (Unauthorized). Disabling key permanently...`
);
this._keyManager.markKeyInvalid(activeKey);
} else {
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} received status ${err.statusCode} (Quota/Rate Limit). Disabling key (cooldown active)...`
);
this._keyManager.markKeyExhausted(activeKey, err.retryAfter);
}
if (attempt < totalKeys - 1) {
this.nextKey();
}
} else {
lastGeneralError = err instanceof Error ? err : new Error(String(err));
Logger.warn(
"[Provider]",
`${this.configIdentity}: Key #${attempt + 1} request failed (${lastGeneralError.message}). Trying next available key...`
);
if (attempt < totalKeys - 1) {
this.nextKey();
} else {
throw lastGeneralError;
}
}
}
}
if (lastGeneralError) {
throw lastGeneralError;
}
const lastStatus = lastKeyError ? lastKeyError.statusCode : "Unknown";
const msg = `All ${totalKeys} configured API keys for provider '${this._providerName}' failed authentication or quota limits. Last status: ${lastStatus}`;
throw new ProviderError(msg);
}
public abstract search(query: string, options?: SearchOptions): Promise<InternalProviderResult[]>;
}