vibevoice-tts

/**
 * @file types.ts
 * @description Type definitions for vibevoice-tts formatting plugin.
 *
 * VibeVoice TTS has NO native SSML support. Output is plain text
 * in a structured multi-speaker script format. Emotion is conveyed
 * via voice reference selection, not inline tags.
 */

/** Format mode: detect speaker structure or force single/multi. */
export type FormatMode = "single" | "multi" | "auto";

/** Output format for the formatted script. */
export type OutputFormat = "plain" | "script" | "json";

export type EmotionLabel =
  | "neutral"
  | "happy"
  | "sad"
  | "angry"
  | "excited"
  | "calm"
  | "thoughtful"
  | "whisper"
  | "serious"
  | "energetic"
  | "warm"
  | "cold"
  | "mysterious"
  | "dramatic";

/**
 * Delivery direction tags — hints for the user on how to adjust TTS params.
 * These are NOT sent to VibeVoice; they guide the human operator.
 */
export type DirectionTag =
  "emphasis" | "slower" | "faster" | "pause" | "whisper" | "louder" | "softer";

/**
 * A pause instruction inserted into the script.
 * VibeVoice interprets empty lines as natural pauses.
 */
export interface PauseMarker {
  /** Position in the script (line index). */
  atLine: number;
  /** Duration in seconds (informational — VibeVoice has no explicit duration control). */
  seconds: number;
  /** Reason for the pause: "section", "paragraph", "sentence", or explicit marker. */
  reason: "section" | "paragraph" | "sentence" | "explicit";
}

/**
 * TTS generation recommendations for the user.
 * These are metadata — not sent to VibeVoice directly.
 */
export interface TTSRecommendations {
  /** Temperature for generation (0.6-0.7 narration, 0.8-0.9 dialogue). */
  temperature: number;
  /** Speed multiplier (0.95x feels more natural than 1.0x). */
  speed: number;
  /** Target words per minute (150-170 for podcast). */
  wpm: number;
  /** Suggested chunk size in words. */
  chunkSize: number;
}

/**
 * A single chunk of a prepared script (~500 words).
 */
export interface ScriptChunk {
  /** Chunk index (1-based). */
  index: number;
  /** The formatted script text for this chunk. */
  text: string;
  /** Word count. */
  wordCount: number;
  /** Speaker IDs present in this chunk. */
  speakers: number[];
  /** Section title or boundary label (if any). */
  section?: string;
}

/**
 * Parameters for the `prepare` tool (podcast script preparation).
 */
export interface PrepareParams {
  /** The raw input script text. */
  text: string;
  /** Target chunk size in words. Default: 500. */
  chunkSize: number;
  /** Insert pauses between paragraphs. Default: true. */
  paragraphPauses: boolean;
  /** Insert pauses between sentences in long paragraphs. Default: false. */
  sentencePauses: boolean;
  /** Max sentences per paragraph before splitting. Default: 3. */
  maxSentencesPerParagraph: number;
  /** Expand common abbreviations. Default: true. */
  expandAbbreviations: boolean;
  /** Preserve direction tags as user-facing hints. Default: true. */
  preserveDirections: boolean;
  /** Output format for chunks. Default: "script". */
  output: "plain" | "script" | "json";
}

/**
 * Result from the `prepare` tool.
 */
export interface PrepareResult {
  /** The original unmodified input. */
  readonly original: string;
  /** The fully prepared script (single string, with pauses inserted). */
  readonly preparedScript: string;
  /** Chunks for incremental generation. */
  readonly chunks: ReadonlyArray<ScriptChunk>;
  /** Pause markers inserted. */
  readonly pauses: ReadonlyArray<PauseMarker>;
  /** TTS generation recommendations. */
  readonly recommendations: TTSRecommendations;
  /** Warnings about the input. */
  readonly warnings: ReadonlyArray<string>;
  /** Total word count. */
  readonly totalWords: number;
  /** Estimated duration in minutes (at recommended WPM). */
  readonly estimatedMinutes: number;
}

/**
 * Maps emotion labels to speaker slot numbers (PsiPi method).
 *
 * VibeVoice controls emotion through the voice reference audio assigned
 * to each speaker slot — never through inline tags. To vary emotion for a
 * single voice, assign the same voice to multiple slots with different
 * emotional reference clips, then route lines to slots via this map.
 */
export type EmotionMapping = Partial<Record<EmotionLabel, number>>;

/**
 * Parameters for the format tool.
 * VibeVoice TTS receives plain text — no SSML, no inline emotion tags.
 * Emotion is documented per-speaker so the user picks the right voice reference.
 */
export interface VibeVoiceFormatParams {
  /** The input text to format for VibeVoice TTS. */
  text: string;
  /** Speaker mode: "single", "multi", or "auto" (detect). Default: "auto". */
  mode: FormatMode;
  /** Number of speakers (1-4). Default: 1. */
  speakers: number;
  /** Voice names per speaker, e.g. ["Alice", "Frank"]. */
  voices: ReadonlyArray<string>;
  /**
   * Emotion labels per speaker — these are documentation annotations
   * telling the user which voice reference audio to select for each slot.
   * VibeVoice does NOT support inline emotion tags.
   */
  emotions: ReadonlyArray<EmotionLabel>;
  /** Apply text cleaning pipeline. Default: true. */
  clean: boolean;
  /** Output format: "plain", "script", or "json". Default: "script". */
  output: OutputFormat;
  /**
   * Emotion-to-speaker-slot mapping (PsiPi method). Lines annotated with
   * [emotion] / (emotion) are stripped of the annotation and reassigned to
   * the mapped speaker slot. Default: neutral/calm→1, happy/sad→2,
   * angry/excited→3.
   */
  emotion_map?: EmotionMapping;
}

/** Metadata about a single speaker in the formatted script. */
export interface SpeakerInfo {
  /** Speaker ID (1-based). */
  id: number;
  /** Optional voice name for this speaker. */
  voice?: string;
  /** Optional emotion label for voice reference selection. */
  emotion?: string;
  /** Number of lines spoken by this speaker. */
  lines: number;
}

/** A single line in the multi-speaker script. */
export interface ScriptLine {
  /** Speaker ID (1-based). */
  speaker: number;
  /** The text spoken on this line. */
  text: string;
}

/**
 * The complete result of formatting text for VibeVoice TTS.
 * No SSML — VibeVoice consumes plain text or structured script format natively.
 */
export interface VibeVoiceResult {
  /** The original unmodified input text. */
  readonly original: string;
  /** The text after the cleaning pipeline (identical to original if clean=false). */
  readonly cleaned: string;
  /** The formatted script with "Speaker N:" prefixes (for output="script"). */
  readonly script: string;
  /** Per-speaker metadata. */
  readonly speakers: ReadonlyArray<SpeakerInfo>;
  /** Non-fatal warnings about the input or processing. */
  readonly warnings: ReadonlyArray<string>;
}

// ---------------------------------------------------------------------------
// VibeVoice Expression Tags — OPEN SYSTEM
// ---------------------------------------------------------------------------

/**
 * VibeVoice expression tags are OPEN — any <tag> pattern is preserved.
 * The model can invent new expressions freely: <sing>, <roar>, <whispers>,
 * <coughs>, <gasps>, <clears throat>, etc.
 *
 * KNOWN tags (documented from real scripts):
 *   <laughs>, <giggles>, <chuckles>, <sighs>, <sniffs>,
 *   <coughs>, <gasps>, <groans>, <whimpers>, <screams>,
 *   <cries>, <sobbing>, <yawns>, <clears throat>,
 *   <inhales>, <exhales>, <pause>, <smacks lips>,
 *   <clicks tongue>, <gulps>, <swallows>, <purrs>,
 *   <hisses>, <growls>, <roars>, <whistles>,
 *   <humming>, <singing>, <singing softly>, <singing loudly>,
 *   <singing quietly>, <singing happily>, <singing sadly>,
 *   <singing angrily>, <singing excitedly>, <singing nervously>
 *
 * But ANY <word> or <phrase with spaces> is valid and will be preserved.
 */
export const KNOWN_EXPRESSION_TAGS: ReadonlyArray<string> = [
  "laughs",
  "giggles",
  "chuckles",
  "sighs",
  "sniffs",
  "coughs",
  "gasps",
  "groans",
  "whimpers",
  "screams",
  "cries",
  "sobbing",
  "yawns",
  "clears throat",
  "inhales",
  "exhales",
  "pause",
  "smacks lips",
  "clicks tongue",
  "gulps",
  "swallows",
  "purrs",
  "hisses",
  "growls",
  "roars",
  "whistles",
  "humming",
  "singing",
  "singing softly",
  "singing loudly",
  "singing quietly",
  "singing happily",
  "singing sadly",
  "singing angrily",
  "singing excitedly",
  "singing nervously",
];

/**
 * Regex matching VibeVoice expression tags: <word> or <phrase with spaces>.
 * Used to preserve expression tags during text cleaning.
 */
export const EXPRESSION_TAG_REGEX = /<([a-zA-Z][a-zA-Z0-9\s-]*)>/g;

// ---------------------------------------------------------------------------
// Dynamic Speaker Profiles — Emotion Changes Over Time
// ---------------------------------------------------------------------------

/**
 * A speaker's emotional/delivery state at a point in the script.
 * Speakers can change expression mid-script — this is NOT a fixed parameter.
 */
export interface SpeakerState {
  /** Position in script (line index or character offset). */
  at: number;
  /** Emotion label at this point. */
  emotion?: EmotionLabel;
  /** Delivery speed modifier (e.g., 0.8x slower, 1.2x faster). */
  speed?: number;
  /** Volume modifier (e.g., "softer", "louder", "whisper"). */
  volume?: "softer" | "louder" | "whisper" | "normal";
  /** Expression tag to apply (e.g., <laughs>, <sighs>). */
  expression?: string;
  /** Background music cue (if VibeVoice supports it). */
  bgMusic?: string;
}

/**
 * A speaker profile with a timeline of emotional/delivery changes.
 * Instead of fixed emotion per speaker, this tracks how expression
 * evolves throughout the script.
 */
export interface SpeakerProfile {
  /** Speaker ID (1-based). */
  id: number;
  /** Voice name (e.g., "Alice", "Frank"). */
  voice?: string;
  /** Timeline of state changes — each entry overrides previous state. */
  states: ReadonlyArray<SpeakerState>;
  /** Default emotion if no states defined. */
  defaultEmotion?: EmotionLabel;
  /** Default voice reference for this speaker. */
  defaultVoiceRef?: string;
}

/**
 * Background music configuration.
 * VibeVoice may support generating background music from text cues.
 */
export interface BGMusicConfig {
  /** Music style/genre cue (e.g., "ambient piano", "orchestral swell"). */
  style: string;
  /** Intensity (0.0-1.0). */
  intensity: number;
  /** When to start (line index or character offset). */
  startAt: number;
  /** When to fade out (line index or character offset). */
  fadeOutAt?: number;
}
// ---------------------------------------------------------------------------
// Narration Styles & Moods (for the `narrate` tool)
// ---------------------------------------------------------------------------

/**
 * Narration style presets — each configures pause patterns, speed,
 * temperature, and direction tag density.
 */
export type NarrationStyle =
  | "storytelling"
  | "children"
  | "christmas-tale"
  | "news"
  | "technical"
  | "dramatic";

/**
 * Mood modifiers — fine-tune the emotional tone within a style.
 */
export type NarrationMood =
  | "warm"
  | "mysterious"
  | "energetic"
  | "somber"
  | "playful"
  | "serious"
  | "neutral";

/**
 * Language code for language-specific processing.
 */
export type LanguageCode =
  "en" | "pl" | "de" | "fr" | "es" | "it" | "pt" | "ja" | "ko" | "zh";

/**
 * A phonetic hint — marks a word/phrase that needs special pronunciation.
 * Used for English words in non-English text (and vice versa).
 */
export interface PhoneticHint {
  /** The original word/phrase in the text. */
  original: string;
  /** How it should be pronounced (phonetic spelling or language tag). */
  pronunciation: string;
  /** Position in the text (character index). */
  position: number;
  /** Reason for the hint. */
  reason: "foreign-word" | "abbreviation" | "acronym" | "number" | "date";
}

/**
 * Detected narrative structure element.
 */
export interface NarrativeElement {
  /** Type of narrative element. */
  type:
    | "dialogue"
    | "narration"
    | "scene-change"
    | "chapter"
    | "transition"
    | "climax";
  /** Start position in text (character index). */
  start: number;
  /** End position in text (character index). */
  end: number;
  /** Optional label (e.g., chapter title). */
  label?: string;
  /** Confidence score (0-1). */
  confidence: number;
}

/**
 * Style-specific TTS configuration from a preset.
 */
export interface StylePreset {
  /** Pause duration between sentences (seconds). */
  sentencePause: number;
  /** Pause duration between paragraphs (seconds). */
  paragraphPause: number;
  /** Pause duration between scenes/chapters (seconds). */
  scenePause: number;
  /** Speed multiplier. */
  speed: number;
  /** Temperature for TTS generation. */
  temperature: number;
  /** Target WPM. */
  wpm: number;
  /** Whether to insert direction tags automatically. */
  autoDirections: boolean;
  /** Max sentences per paragraph before splitting. */
  maxSentences: number;
  /** Description of the preset. */
  description: string;
}

/**
 * Language-specific configuration.
 */
export interface LanguagePack {
  /** Language code. */
  code: LanguageCode;
  /** Native name of the language. */
  name: string;
  /** Abbreviation expansion map (abbreviation -> full form). */
  abbreviations: Readonly<Record<string, string>>;
  /** Common English loanwords that should be pronounced in English. */
  foreignWords: ReadonlyArray<string>;
  /** Acronyms that should be spelled out letter by letter. */
  acronyms: ReadonlyArray<string>;
  /** Number formatting rules. */
  numberFormat: "western" | "eastern";
}

/**
 * Parameters for the `narrate` tool (style-aware narration preparation).
 */
export interface NarrateParams {
  /** The raw input text. */
  text: string;
  /** Narration style preset. Default: "storytelling". */
  style: NarrationStyle;
  /** Mood modifier. Default: "neutral". */
  mood: NarrationMood;
  /** Language code for language-specific processing. Default: "en". */
  language: LanguageCode;
  /** Target chunk size in words. Default: 500. */
  chunkSize: number;
  /** Expand language-specific abbreviations. Default: true. */
  expandAbbreviations: boolean;
  /** Insert phonetic hints for foreign words. Default: true. */
  phoneticHints: boolean;
  /** Detect narrative structure (dialogue, scenes, etc.). Default: true. */
  detectStructure: boolean;
  /** Auto-insert expression tags based on context. Default: true. */
  autoExpressions: boolean;
  /** Output format. Default: "script". */
  output: "plain" | "script" | "json";
}

/**
 * Result from the `narrate` tool.
 */
export interface NarrateResult {
  /** The original unmodified input. */
  readonly original: string;
  /** The fully prepared narration script. */
  readonly script: string;
  /** Chunks for incremental generation. */
  readonly chunks: ReadonlyArray<ScriptChunk>;
  /** Detected narrative structure elements. */
  readonly structure: ReadonlyArray<NarrativeElement>;
  /** Phonetic hints inserted. */
  readonly phoneticHints: ReadonlyArray<PhoneticHint>;
  /** Expression tags inserted (auto or manual). */
  readonly expressions: ReadonlyArray<{
    tag: string;
    position: number;
    reason: string;
  }>;
  /** Speaker emotion timeline changes. */
  readonly speakerChanges: ReadonlyArray<{
    speaker: number;
    at: number;
    emotion: string;
  }>;
  /** TTS recommendations based on style + mood. */
  readonly recommendations: TTSRecommendations;
  /** Style preset that was applied. */
  readonly appliedStyle: StylePreset;
  /** Warnings about the input. */
  readonly warnings: ReadonlyArray<string>;
  /** Total word count. */
  readonly totalWords: number;
  /** Estimated duration in minutes. */
  readonly estimatedMinutes: number;
  /** Language pack that was used. */
  readonly language: LanguageCode;
}