Loading files
import { env } from '../config/env.js'
import { logger } from '../utils/logger.js'
// Gemini batch STT for the device voice endpoint. The device streams raw linear16/16k/mono PCM;
// Gemini's inline audio wants a container, so we prepend a 44-byte WAV header and send audio/wav.
const CHANNELS = 1
const BITS = 16
// Minimal canonical WAV (PCM) header for the given data length at `sampleRate` Hz.
function wavHeader(dataLen: number, sampleRate: number): Buffer {
const byteRate = (sampleRate * CHANNELS * BITS) / 8
const blockAlign = (CHANNELS * BITS) / 8
const h = Buffer.alloc(44)
h.write('RIFF', 0)
h.writeUInt32LE(36 + dataLen, 4)
h.write('WAVE', 8)
h.write('fmt ', 12)
h.writeUInt32LE(16, 16) // fmt chunk size
h.writeUInt16LE(1, 20) // PCM
h.writeUInt16LE(CHANNELS, 22)
h.writeUInt32LE(sampleRate, 24)
h.writeUInt32LE(byteRate, 28)
h.writeUInt16LE(blockAlign, 32)
h.writeUInt16LE(BITS, 34)
h.write('data', 36)
h.writeUInt32LE(dataLen, 40)
return h
}
// Instruction goes in systemInstruction (not the user turn) so the model treats it as a rule,
// not content to reason about — this keeps thinking-capable models from echoing deliberation.
const SYSTEM =
'You transcribe audio to plain text. Output ONLY the exact words spoken, as a single line of ' +
'plain text. Absolutely do NOT add timestamps, time codes, "-->" ranges, SRT or WebVTT/caption ' +
'formatting, speaker names, brackets, or notes. Do NOT apologize, refuse, or explain — even if ' +
'the audio is short, noisy, or unclear, just write the words you hear. If you truly hear no ' +
'words at all, output nothing (an empty response).'
// Belt-and-suspenders: strip caption artifacts the model sometimes emits despite the instruction
// (SRT/VTT timestamps, cue numbers, the WEBVTT header) so they never reach the agent as a turn.
function cleanTranscript(raw: string): string {
return raw
.replace(/^?WEBVTT.*$/gim, '')
// "00:00:00,000 --> 00:00:01,000" (also ':' or '.' as the ms separator)
.replace(/\d{1,2}:\d{2}:\d{2}[.,:]\d{1,3}\s*-->\s*\d{1,2}:\d{2}:\d{2}[.,:]\d{1,3}.*$/gim, '')
.replace(/^\s*\d+\s*$/gm, '') // lone SRT cue numbers
.replace(/\s+/g, ' ')
.trim()
}
/** Transcribe raw linear16/mono PCM at `sampleRate` Hz via Gemini. Returns the transcript ('' if none). */
export async function transcribe(pcm: Buffer, sampleRate = 16000): Promise<string> {
if (!env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY not set')
const wav = Buffer.concat([wavHeader(pcm.length, sampleRate), pcm])
const url =
`https://generativelanguage.googleapis.com/v1beta/models/${env.GEMINI_STT_MODEL}:generateContent` +
`?key=${env.GEMINI_API_KEY}`
const payload = JSON.stringify({
systemInstruction: { parts: [{ text: SYSTEM }] },
contents: [
{
parts: [
{ text: 'Transcribe:' },
{ inline_data: { mime_type: 'audio/wav', data: wav.toString('base64') } },
],
},
],
// thinkingBudget 0 disables 2.5 "thinking" — STT needs none, and it keeps latency low and
// stops the model from deliberating/refusing instead of just transcribing.
generationConfig: { temperature: 0, thinkingConfig: { thinkingBudget: 0 } },
})
// Retry transient overload (503/UNAVAILABLE, 429) with backoff — flash-lite spikes are common.
let res!: Response
for (let attempt = 0; attempt < 4; attempt++) {
res = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: payload,
})
if (res.ok || (res.status !== 503 && res.status !== 429)) break
const backoff = 400 * 2 ** attempt // 400, 800, 1600 ms
logger.warn('gemini transient error — retrying', { status: res.status, attempt, backoff })
await new Promise((r) => setTimeout(r, backoff))
}
if (!res.ok) {
const body = await res.text().catch(() => '')
throw new Error(`Gemini HTTP ${res.status}: ${body.slice(0, 200)}`)
}
const data = (await res.json()) as {
candidates?: Array<{ content?: { parts?: Array<{ text?: string; thought?: boolean }> } }>
}
const rawText = (data.candidates?.[0]?.content?.parts ?? [])
.filter((p) => !p.thought) // drop reasoning parts from thinking models
.map((p) => p.text ?? '')
.join('')
const transcript = cleanTranscript(rawText)
logger.info('gemini transcript', { chars: transcript.length, model: env.GEMINI_STT_MODEL })
return transcript
}
442335385This file is pinned to the selected release. Download the original to open it in your design tool.