src / generator.ts
import { type Chat, type GeneratorController } from "@lmstudio/sdk";
import { configSchematics, CONFIG_DEFAULTS } from "./config";
import { ShartiLogger, type LogLevel } from "./utils/logger";
import { asBool, asNumber, asString } from "./utils/coerce";
import { formatBlock } from "./utils/block";
import { toOpenAIMessages, toOpenAITools, sanitizeToolMessageOrdering, deriveConversationKey } from "./convert/messages";
import { consumeStream } from "./convert/stream";
import { createOpenAI, buildRequestPayload, logDebugPayload, executeCompletions } from "./request";
import { extractProviderError, formatErrorMessage } from "./errors";
import { logGenerationStats, getSessionAccumulator } from "./stats";
import type { OpenAICompatMessageParam } from "./types";
/* -------------------------------------------------------------------------- */
/* Generation entrypoint */
/* */
/* Thin orchestrator: wires validation, conversion, request building, abort */
/* handling, retry and stats. Domain mechanics live in dedicated modules. */
/* -------------------------------------------------------------------------- */
export async function generate(ctl: GeneratorController, history: Chat) {
const config = ctl.getPluginConfig(configSchematics);
const startTime = Date.now();
// --- Logging -------------------------------------------------------------
const logLevel = asString(config.get("logLevel"), CONFIG_DEFAULTS.logLevel) as LogLevel;
const logger = new ShartiLogger(ctl, { logLevel });
// --- Validate required connection fields ---------------------------------
const baseUrl = asString(config.get("baseUrl"));
const model = asString(config.get("model"));
if (!baseUrl) {
logger.error("Missing Base URL. Please set it in the plugin settings.");
return;
}
if (!model) {
logger.error("Missing Model name. Please set it in the plugin settings.");
return;
}
// --- Remaining config ----------------------------------------------------
const apiKey = asString(config.get("apiKey"));
const maxRetries = asNumber(config.get("maxRetries"), CONFIG_DEFAULTS.maxRetries);
// --- Assemble request inputs --------------------------------------------
let messages: OpenAICompatMessageParam[] = await toOpenAIMessages(history);
messages = sanitizeToolMessageOrdering(messages);
const tools = toOpenAITools(ctl);
const requestPayload = buildRequestPayload(config, messages, tools);
// --- Observability -------------------------------------------------------
const reqBlock = formatBlock("🚀 Request:", [
["baseUrl", baseUrl],
["model", model],
["messages", messages.length],
["tools", tools?.length ?? 0],
]);
if (reqBlock) logger.info(reqBlock);
logDebugPayload(logger, requestPayload);
// --- Abort plumbing ------------------------------------------------------
const abortController = new AbortController();
const onAbort = (): void => {
logger.warn("Generation aborted by user.");
abortController.abort();
};
ctl.abortSignal.addEventListener("abort", onAbort, { once: true });
// A single client is reused across retries (connection pooling preserved).
const openai = createOpenAI(baseUrl, apiKey);
// Persistent-per-conversation accumulator: survives the multi-round (tool handoff)
// regenerations of a single turn, isolated per conversation thread and concurrency-safe.
const turnStats = getSessionAccumulator(deriveConversationKey(history));
try {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
if (ctl.abortSignal.aborted) {
logger.warn("Aborted during retry wait.");
return;
}
let stream;
try {
stream = await executeCompletions(openai, requestPayload, abortController.signal);
} catch (error: unknown) {
if (ctl.abortSignal.aborted) {
logger.warn("Generation aborted.");
return;
}
const info = extractProviderError(error);
// Transient 503: retry with capped exponential backoff (1s → 2s → 4s → 8s → 10s).
if (info.status === 503 && attempt < maxRetries) {
const delay = Math.min(1000 * Math.pow(2, attempt), 10000);
logger.warn(`Server returned 503 (attempt ${attempt + 1}/${maxRetries + 1}). Retrying in ${delay}ms…`);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
const formatted = formatErrorMessage(info);
logger.error(formatted);
logger.debug(`Raw error details: ${JSON.stringify(info, null, 2)}`);
return;
}
// Stream established — consume (mid-stream failures are reported, not retried).
try {
const stats = await consumeStream(stream, ctl, logger, startTime);
logGenerationStats(logger, stats);
turnStats.record(stats, history.getLength());
turnStats.emitSummaryIfFinal(logger, stats.finishReason, asBool(config.get("showTurnSummaryInChat"), CONFIG_DEFAULTS.showTurnSummaryInChat));
} catch (error: unknown) {
if (ctl.abortSignal.aborted) {
logger.warn("Generation aborted during streaming.");
return;
}
const info = extractProviderError(error);
const formatted = formatErrorMessage(info);
logger.error(`Stream error: ${formatted}`);
logger.debug(`Raw stream error details: ${JSON.stringify(info, null, 2)}`);
}
return;
}
} finally {
ctl.abortSignal.removeEventListener("abort", onAbort);
}
}
src / generator.ts
import { type Chat, type GeneratorController } from "@lmstudio/sdk";
import { configSchematics, CONFIG_DEFAULTS } from "./config";
import { ShartiLogger, type LogLevel } from "./utils/logger";
import { asBool, asNumber, asString } from "./utils/coerce";
import { formatBlock } from "./utils/block";
import { toOpenAIMessages, toOpenAITools, sanitizeToolMessageOrdering, deriveConversationKey } from "./convert/messages";
import { consumeStream } from "./convert/stream";
import { createOpenAI, buildRequestPayload, logDebugPayload, executeCompletions } from "./request";
import { extractProviderError, formatErrorMessage } from "./errors";
import { logGenerationStats, getSessionAccumulator } from "./stats";
import type { OpenAICompatMessageParam } from "./types";
/* -------------------------------------------------------------------------- */
/* Generation entrypoint */
/* */
/* Thin orchestrator: wires validation, conversion, request building, abort */
/* handling, retry and stats. Domain mechanics live in dedicated modules. */
/* -------------------------------------------------------------------------- */
export async function generate(ctl: GeneratorController, history: Chat) {
const config = ctl.getPluginConfig(configSchematics);
const startTime = Date.now();
// --- Logging -------------------------------------------------------------
const logLevel = asString(config.get("logLevel"), CONFIG_DEFAULTS.logLevel) as LogLevel;
const logger = new ShartiLogger(ctl, { logLevel });
// --- Validate required connection fields ---------------------------------
const baseUrl = asString(config.get("baseUrl"));
const model = asString(config.get("model"));
if (!baseUrl) {
logger.error("Missing Base URL. Please set it in the plugin settings.");
return;
}
if (!model) {
logger.error("Missing Model name. Please set it in the plugin settings.");
return;
}
// --- Remaining config ----------------------------------------------------
const apiKey = asString(config.get("apiKey"));
const maxRetries = asNumber(config.get("maxRetries"), CONFIG_DEFAULTS.maxRetries);
// --- Assemble request inputs --------------------------------------------
let messages: OpenAICompatMessageParam[] = await toOpenAIMessages(history);
messages = sanitizeToolMessageOrdering(messages);
const tools = toOpenAITools(ctl);
const requestPayload = buildRequestPayload(config, messages, tools);
// --- Observability -------------------------------------------------------
const reqBlock = formatBlock("🚀 Request:", [
["baseUrl", baseUrl],
["model", model],
["messages", messages.length],
["tools", tools?.length ?? 0],
]);
if (reqBlock) logger.info(reqBlock);
logDebugPayload(logger, requestPayload);
// --- Abort plumbing ------------------------------------------------------
const abortController = new AbortController();
const onAbort = (): void => {
logger.warn("Generation aborted by user.");
abortController.abort();
};
ctl.abortSignal.addEventListener("abort", onAbort, { once: true });
// A single client is reused across retries (connection pooling preserved).
const openai = createOpenAI(baseUrl, apiKey);
// Persistent-per-conversation accumulator: survives the multi-round (tool handoff)
// regenerations of a single turn, isolated per conversation thread and concurrency-safe.
const turnStats = getSessionAccumulator(deriveConversationKey(history));
try {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
if (ctl.abortSignal.aborted) {
logger.warn("Aborted during retry wait.");
return;
}
let stream;
try {
stream = await executeCompletions(openai, requestPayload, abortController.signal);
} catch (error: unknown) {
if (ctl.abortSignal.aborted) {
logger.warn("Generation aborted.");
return;
}
const info = extractProviderError(error);
// Transient 503: retry with capped exponential backoff (1s → 2s → 4s → 8s → 10s).
if (info.status === 503 && attempt < maxRetries) {
const delay = Math.min(1000 * Math.pow(2, attempt), 10000);
logger.warn(`Server returned 503 (attempt ${attempt + 1}/${maxRetries + 1}). Retrying in ${delay}ms…`);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
const formatted = formatErrorMessage(info);
logger.error(formatted);
logger.debug(`Raw error details: ${JSON.stringify(info, null, 2)}`);
return;
}
// Stream established — consume (mid-stream failures are reported, not retried).
try {
const stats = await consumeStream(stream, ctl, logger, startTime);
logGenerationStats(logger, stats);
turnStats.record(stats, history.getLength());
turnStats.emitSummaryIfFinal(logger, stats.finishReason, asBool(config.get("showTurnSummaryInChat"), CONFIG_DEFAULTS.showTurnSummaryInChat));
} catch (error: unknown) {
if (ctl.abortSignal.aborted) {
logger.warn("Generation aborted during streaming.");
return;
}
const info = extractProviderError(error);
const formatted = formatErrorMessage(info);
logger.error(`Stream error: ${formatted}`);
logger.debug(`Raw stream error details: ${JSON.stringify(info, null, 2)}`);
}
return;
}
} finally {
ctl.abortSignal.removeEventListener("abort", onAbort);
}
}