266 lines
9.6 KiB
TypeScript
266 lines
9.6 KiB
TypeScript
export const MAX_STT_AUDIO_BYTES = 25 * 1024 * 1024
|
||
export const STT_PROVIDER_TIMEOUT_MS = 60_000
|
||
|
||
/** 16 kHz 16-bit mono PCM — the most common upload; compressed audio only shortens the estimate. */
|
||
const PCM_BYTES_PER_SECOND = 32_000
|
||
const GATEWAY_BASE_MS = 5_000
|
||
const GATEWAY_MAX_WITH_FALLBACK_MS = 30_000
|
||
|
||
/**
|
||
* How long to wait for the self-hosted gateway before a direct provider takes over.
|
||
*
|
||
* The gateway tries a GPU box that may be off and a CPU-only NAS Whisper that
|
||
* needed 50–90 s for a 3 s clip (2 cores under load, 2026-09-27). Waiting the
|
||
* full provider timeout turned a short recording into a minute-long wait, and
|
||
* the phone gave up to its tiny local model. With a direct provider configured,
|
||
* give the gateway 5 s plus the clip length (enough for a GPU, far too little
|
||
* for the NAS) and fall through; without one, keep the full timeout.
|
||
*/
|
||
const LANGUAGE_NAME_TO_CODE: Readonly<Record<string, string>> = {
|
||
korean: 'ko', english: 'en', japanese: 'ja', chinese: 'zh', spanish: 'es', french: 'fr',
|
||
german: 'de', portuguese: 'pt', russian: 'ru', vietnamese: 'vi', thai: 'th', italian: 'it',
|
||
dutch: 'nl', indonesian: 'id', turkish: 'tr', arabic: 'ar', hindi: 'hi', polish: 'pl',
|
||
}
|
||
|
||
/**
|
||
* Whisper verbose_json (OpenAI, Groq) reports the language by name ("korean"), while the
|
||
* STT contract carries a code. A name used to fail normalizeSttResult, so every direct
|
||
* OpenAI/Groq fallback was thrown away as an invalid result. Accept a code, map a name,
|
||
* otherwise use what the client asked for.
|
||
*/
|
||
export function providerLanguageCode(reported: unknown, requested: string): string {
|
||
const fallback = requested === 'auto' || requested === 'multi' ? 'und' : requested
|
||
if (typeof reported !== 'string') return fallback
|
||
const value = reported.trim().toLowerCase()
|
||
if (/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/.test(value)) return value
|
||
return LANGUAGE_NAME_TO_CODE[value] ?? fallback
|
||
}
|
||
|
||
export function gatewayDeadlineMs(audioBytes: number, hasDirectFallback: boolean): number {
|
||
if (!hasDirectFallback) return STT_PROVIDER_TIMEOUT_MS
|
||
const estimatedSeconds = Math.max(0, audioBytes) / PCM_BYTES_PER_SECOND
|
||
return Math.min(GATEWAY_MAX_WITH_FALLBACK_MS, Math.round(GATEWAY_BASE_MS + estimatedSeconds * 1_000))
|
||
}
|
||
|
||
export interface NormalizedSttResult {
|
||
transcript: string
|
||
confidence: number
|
||
language_code: string
|
||
duration_seconds: number
|
||
provider: string
|
||
}
|
||
|
||
export function normalizeSttResult(value: unknown): NormalizedSttResult {
|
||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||
throw new Error('invalid_stt_result')
|
||
}
|
||
const result = value as Record<string, unknown>
|
||
if (
|
||
typeof result.transcript !== 'string'
|
||
|| !result.transcript.trim()
|
||
|| result.transcript.length > 1_000_000
|
||
|| typeof result.confidence !== 'number'
|
||
|| !Number.isFinite(result.confidence)
|
||
|| result.confidence < 0
|
||
|| result.confidence > 1
|
||
|| typeof result.language_code !== 'string'
|
||
|| !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})?$/i.test(result.language_code)
|
||
|| typeof result.duration_seconds !== 'number'
|
||
|| !Number.isFinite(result.duration_seconds)
|
||
|| result.duration_seconds < 0
|
||
|| result.duration_seconds > 24 * 60 * 60
|
||
|| typeof result.provider !== 'string'
|
||
|| !/^[a-z0-9._-]{1,64}$/.test(result.provider)
|
||
) {
|
||
throw new Error('invalid_stt_result')
|
||
}
|
||
return {
|
||
transcript: result.transcript.trim(),
|
||
confidence: result.confidence,
|
||
language_code: result.language_code.toLowerCase(),
|
||
duration_seconds: result.duration_seconds,
|
||
provider: result.provider,
|
||
}
|
||
}
|
||
|
||
export function createInternalSttGatewayUrl(baseUrl: string): string {
|
||
const url = new URL('/api/stt/internal/transcribe', baseUrl)
|
||
const localHttp = url.protocol === 'http:' && [
|
||
'localhost',
|
||
'127.0.0.1',
|
||
'host.docker.internal',
|
||
'd3ro-api-server',
|
||
].includes(url.hostname)
|
||
if ((url.protocol !== 'https:' && !localHttp) || url.username || url.password) {
|
||
throw new Error('invalid_stt_gateway_url')
|
||
}
|
||
return url.toString()
|
||
}
|
||
|
||
const AUDIO_EXTENSIONS: Readonly<Record<string, string>> = Object.freeze({
|
||
'audio/aac': 'aac',
|
||
'audio/flac': 'flac',
|
||
'audio/mp3': 'mp3',
|
||
'audio/mp4': 'm4a',
|
||
'audio/mpeg': 'mp3',
|
||
'audio/ogg': 'ogg',
|
||
'audio/wav': 'wav',
|
||
'audio/webm': 'webm',
|
||
'audio/x-flac': 'flac',
|
||
'audio/x-m4a': 'm4a',
|
||
'audio/x-wav': 'wav',
|
||
'application/ogg': 'ogg',
|
||
'video/mp4': 'mp4',
|
||
'video/quicktime': 'mov',
|
||
'video/webm': 'webm',
|
||
})
|
||
|
||
export class SttInputError extends Error {
|
||
constructor(
|
||
readonly code: 'audio_empty' | 'audio_too_large' | 'unsupported_audio_type' | 'invalid_language',
|
||
readonly status: 400 | 413 | 415,
|
||
) {
|
||
super(code)
|
||
this.name = 'SttInputError'
|
||
}
|
||
}
|
||
|
||
export interface DictionaryHintRow {
|
||
word?: unknown
|
||
pronunciation?: unknown
|
||
}
|
||
|
||
export interface DictionaryHints {
|
||
keyterms: string[]
|
||
prompt: string
|
||
}
|
||
|
||
function compactText(value: unknown, maxLength: number): string {
|
||
if (typeof value !== 'string') return ''
|
||
const withoutControls = Array.from(value, (character) => {
|
||
const code = character.charCodeAt(0)
|
||
return code < 32 || code === 127 ? ' ' : character
|
||
}).join('')
|
||
return withoutControls.replace(/\s+/g, ' ').trim().slice(0, maxLength)
|
||
}
|
||
|
||
export function validateSttAudio(audio: Blob, languageCode: string): {
|
||
contentType: string
|
||
fileName: string
|
||
languageCode: string
|
||
} {
|
||
if (audio.size <= 0) throw new SttInputError('audio_empty', 400)
|
||
if (audio.size > MAX_STT_AUDIO_BYTES) throw new SttInputError('audio_too_large', 413)
|
||
|
||
const contentType = audio.type.toLowerCase().split(';', 1)[0].trim()
|
||
const extension = AUDIO_EXTENSIONS[contentType]
|
||
if (!extension) throw new SttInputError('unsupported_audio_type', 415)
|
||
|
||
const language = languageCode.trim()
|
||
if (!/^(auto|multi|[a-z]{2,3}(?:-[a-z0-9]{2,8})?)$/i.test(language)) {
|
||
throw new SttInputError('invalid_language', 400)
|
||
}
|
||
|
||
return {
|
||
contentType,
|
||
fileName: `audio.${extension}`,
|
||
languageCode: language.toLowerCase(),
|
||
}
|
||
}
|
||
|
||
export function buildDictionaryHints(rows: readonly DictionaryHintRow[]): DictionaryHints {
|
||
const seen = new Set<string>()
|
||
const keyterms: string[] = []
|
||
const promptParts: string[] = []
|
||
let promptLength = 0
|
||
|
||
for (const row of rows.slice(0, 100)) {
|
||
const word = compactText(row.word, 120)
|
||
if (!word) continue
|
||
const identity = word.toLocaleLowerCase()
|
||
if (seen.has(identity)) continue
|
||
|
||
const pronunciation = compactText(row.pronunciation, 120)
|
||
const promptPart = pronunciation ? `${word} (${pronunciation})` : word
|
||
const nextLength = promptLength + promptPart.length + (promptParts.length > 0 ? 2 : 0)
|
||
if (nextLength > 900) break
|
||
|
||
seen.add(identity)
|
||
keyterms.push(word)
|
||
promptParts.push(promptPart)
|
||
promptLength = nextLength
|
||
if (keyterms.length >= 50) break
|
||
}
|
||
|
||
return { keyterms, prompt: promptParts.join(', ') }
|
||
}
|
||
|
||
export function createDeepgramSttUrl(languageCode: string, keyterms: readonly string[]): string {
|
||
const url = new URL('https://api.deepgram.com/v1/listen')
|
||
url.searchParams.set('model', 'nova-3')
|
||
url.searchParams.set('language', languageCode === 'auto' ? 'multi' : languageCode)
|
||
url.searchParams.set('smart_format', 'true')
|
||
url.searchParams.set('punctuate', 'true')
|
||
for (const keyterm of keyterms.slice(0, 50)) url.searchParams.append('keyterm', keyterm)
|
||
return url.toString()
|
||
}
|
||
|
||
/**
|
||
* Fixed failure codes reported to callers in `attempts`. Never a raw exception
|
||
* message: Deno fetch errors carry the request URL (the internal gateway host)
|
||
* and JSON parse errors quote part of the provider body.
|
||
*/
|
||
export type SttFailureCode =
|
||
| 'timeout'
|
||
| 'network_error'
|
||
| 'invalid_response'
|
||
| 'misconfigured'
|
||
| 'error'
|
||
| `http_${number}`
|
||
|
||
/**
|
||
* What one provider call produced.
|
||
*
|
||
* - `ok`: a 2xx answer with a real transcript.
|
||
* - `no_speech`: a 2xx answer with an empty transcript. The provider has billed the
|
||
* audio, so this is terminal: it is never retried on the next paid provider.
|
||
* - `retryable`: transport error, non-2xx status or a malformed 2xx body. `billed`
|
||
* is true when the provider answered 2xx (it charged us even though the body was
|
||
* unusable), so the quota reservation must be consumed, not refunded.
|
||
* `unavailable` marks an explicit "not available" answer (gateway 503).
|
||
*/
|
||
export type SttOutcome =
|
||
| { kind: 'ok'; result: NormalizedSttResult }
|
||
| { kind: 'no_speech' }
|
||
| { kind: 'retryable'; failure: SttFailureCode; billed: boolean; unavailable: boolean }
|
||
|
||
export function sttHttpFailure(status: number): SttFailureCode {
|
||
return `http_${Math.trunc(status)}`
|
||
}
|
||
|
||
/** Map a thrown transport error to a fixed code. The error text is never kept. */
|
||
export function sttTransportFailure(err: unknown): SttFailureCode {
|
||
if (!(err instanceof Error)) return 'error'
|
||
if (err.name === 'TimeoutError' || err.name === 'AbortError') return 'timeout'
|
||
// Deno fetch rejects DNS, connection and TLS failures with a TypeError.
|
||
if (err.name === 'TypeError') return 'network_error'
|
||
return 'error'
|
||
}
|
||
|
||
/**
|
||
* Classify a 2xx provider answer. The provider has already billed the audio, so
|
||
* every outcome here is `billed`. An empty transcript is a genuine "no speech"
|
||
* result; normalizeSttResult stays strict and is only used for real transcripts.
|
||
*/
|
||
export function sttOutcomeFromBilledAnswer(candidate: Readonly<Record<string, unknown>>): SttOutcome {
|
||
const transcript = candidate.transcript
|
||
if (typeof transcript !== 'string') {
|
||
return { kind: 'retryable', failure: 'invalid_response', billed: true, unavailable: false }
|
||
}
|
||
if (!transcript.trim()) return { kind: 'no_speech' }
|
||
try {
|
||
return { kind: 'ok', result: normalizeSttResult(candidate) }
|
||
} catch {
|
||
return { kind: 'retryable', failure: 'invalid_response', billed: true, unavailable: false }
|
||
}
|
||
}
|