971 lines
34 KiB
TypeScript
971 lines
34 KiB
TypeScript
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
|
import {
|
|
ActivityIndicator,
|
|
AppState,
|
|
KeyboardAvoidingView,
|
|
Platform,
|
|
Pressable,
|
|
ScrollView,
|
|
StyleSheet,
|
|
TextInput,
|
|
View,
|
|
type ViewStyle,
|
|
} from 'react-native'
|
|
import { useFocusEffect } from '@react-navigation/native'
|
|
import { useSafeAreaInsets } from 'react-native-safe-area-context'
|
|
import { useI18n } from '@d3ro/i18n'
|
|
import { ThemeButton, ThemeText } from '../theme/themed-components'
|
|
import { useAuth } from '../lib/auth-context'
|
|
import { useMobilePreferences } from '../lib/preferences-context'
|
|
import { audioRecorder, type RecordingProgress } from '../lib/audio-recorder'
|
|
import {
|
|
ChatServiceError,
|
|
type ChatErrorCode,
|
|
type ChatRequestMessage,
|
|
} from '../features/chat/chat-service'
|
|
import { AudioPipelineError } from '../features/import/audio-import-types'
|
|
import { streamTalkResponse } from '../features/talk/llm-stream-service'
|
|
import { transcribeTalkRecording } from '../features/talk/talk-transcription-service'
|
|
import {
|
|
shutdownTalkSpeech,
|
|
speakTalkText,
|
|
stopTalkSpeech,
|
|
TalkTtsError,
|
|
} from '../features/talk/talk-tts'
|
|
import {
|
|
isDeterministicTalkAudioAvailable,
|
|
synthesizeDeterministicTalkAudio,
|
|
} from '../features/talk/talk-debug-audio'
|
|
import ContentReportSheet, {
|
|
type ContentReportTarget,
|
|
} from '../components/ContentReportSheet'
|
|
|
|
interface TalkMessage extends ChatRequestMessage {
|
|
id: string
|
|
createdAt: number
|
|
generationId?: string
|
|
}
|
|
|
|
type TalkPhase = 'idle' | 'recording' | 'transcribing' | 'thinking' | 'speaking'
|
|
type TalkErrorCode = ChatErrorCode
|
|
| 'MIC_PERMISSION_DENIED'
|
|
| 'RECORDING_FAILED'
|
|
| 'RECORDING_TOO_SHORT'
|
|
| 'STT_QUOTA_EXCEEDED'
|
|
| 'STT_FAILED'
|
|
| 'STT_NO_SPEECH'
|
|
| 'TTS_UNAVAILABLE'
|
|
| 'TTS_FAILED'
|
|
|
|
const MIN_RECORDING_MS = 450
|
|
|
|
function errorTranslationKey(code: TalkErrorCode) {
|
|
switch (code) {
|
|
case 'AUTH_REQUIRED': return 'mobile.talk.error.auth'
|
|
case 'INVALID_REQUEST': return 'mobile.talk.error.invalid'
|
|
case 'QUOTA_EXCEEDED': return 'mobile.talk.error.quota'
|
|
case 'MODEL_NOT_ALLOWED': return 'mobile.talk.error.model'
|
|
case 'PROVIDER_UNAVAILABLE': return 'mobile.talk.error.provider'
|
|
case 'TIMEOUT': return 'mobile.talk.error.timeout'
|
|
case 'NETWORK': return 'mobile.talk.error.network'
|
|
case 'INVALID_RESPONSE': return 'mobile.talk.error.response'
|
|
case 'CANCELLED': return 'mobile.talk.error.cancelled'
|
|
case 'MIC_PERMISSION_DENIED': return 'mobile.talk.error.micPermission'
|
|
case 'RECORDING_FAILED': return 'mobile.talk.error.recording'
|
|
case 'RECORDING_TOO_SHORT': return 'mobile.talk.error.tooShort'
|
|
case 'STT_QUOTA_EXCEEDED': return 'mobile.talk.error.sttQuota'
|
|
case 'STT_FAILED': return 'mobile.talk.error.stt'
|
|
case 'STT_NO_SPEECH': return 'mobile.talk.error.noSpeech'
|
|
case 'TTS_UNAVAILABLE': return 'mobile.talk.error.ttsUnavailable'
|
|
case 'TTS_FAILED': return 'mobile.talk.error.tts'
|
|
default: return 'mobile.talk.error.server'
|
|
}
|
|
}
|
|
|
|
function transcriptionErrorCode(error: unknown): TalkErrorCode {
|
|
if (!(error instanceof AudioPipelineError)) return 'STT_FAILED'
|
|
switch (error.code) {
|
|
case 'auth': return 'AUTH_REQUIRED'
|
|
case 'cancelled': return 'CANCELLED'
|
|
case 'quota': return 'STT_QUOTA_EXCEEDED'
|
|
case 'no-speech': return 'STT_NO_SPEECH'
|
|
default: return 'STT_FAILED'
|
|
}
|
|
}
|
|
|
|
function formatDuration(durationMs: number): string {
|
|
const seconds = Math.max(0, Math.floor(durationMs / 1000))
|
|
return `${Math.floor(seconds / 60).toString().padStart(2, '0')}:${(seconds % 60)
|
|
.toString()
|
|
.padStart(2, '0')}`
|
|
}
|
|
|
|
function phaseTranslationKey(phase: TalkPhase) {
|
|
switch (phase) {
|
|
case 'recording': return 'mobile.talk.phase.recording' as const
|
|
case 'transcribing': return 'mobile.talk.phase.transcribing' as const
|
|
case 'thinking': return 'mobile.talk.phase.thinking' as const
|
|
case 'speaking': return 'mobile.talk.phase.speaking' as const
|
|
default: return 'mobile.talk.phase.idle' as const
|
|
}
|
|
}
|
|
|
|
export default function TalkScreen(): React.ReactElement {
|
|
const insets = useSafeAreaInsets()
|
|
const { t } = useI18n()
|
|
const { session } = useAuth()
|
|
const { palette, preferences } = useMobilePreferences()
|
|
const scrollRef = useRef<ScrollView>(null)
|
|
const mountedRef = useRef(true)
|
|
const messagesRef = useRef<TalkMessage[]>([])
|
|
const phaseRef = useRef<TalkPhase>('idle')
|
|
const operationRef = useRef(0)
|
|
const speechOperationRef = useRef(0)
|
|
const requestControllerRef = useRef<AbortController | null>(null)
|
|
const transcriptionControllerRef = useRef<AbortController | null>(null)
|
|
const activeAssistantIdRef = useRef<string | null>(null)
|
|
const recordingActiveRef = useRef(false)
|
|
const recordingStartingRef = useRef(false)
|
|
const recordingFinishingRef = useRef(false)
|
|
const recorderCleanupRef = useRef<Promise<void>>(Promise.resolve())
|
|
const pressHeldRef = useRef(false)
|
|
const textSendPendingRef = useRef(false)
|
|
const retryMessagesRef = useRef<TalkMessage[] | null>(null)
|
|
const retryAutoSpeakRef = useRef(false)
|
|
const sessionRef = useRef(session)
|
|
sessionRef.current = session
|
|
|
|
const initialMessage = useMemo<TalkMessage>(() => ({
|
|
id: 'welcome',
|
|
role: 'assistant',
|
|
content: t('mobile.talk.greeting'),
|
|
createdAt: Date.now(),
|
|
}), [t])
|
|
const [messages, setMessages] = useState<TalkMessage[]>(() => [initialMessage])
|
|
const [input, setInput] = useState('')
|
|
const [phase, setPhase] = useState<TalkPhase>('idle')
|
|
const [recordingProgress, setRecordingProgress] = useState<RecordingProgress>({
|
|
durationMs: 0,
|
|
meteringDb: null,
|
|
})
|
|
const [errorCode, setErrorCode] = useState<TalkErrorCode | null>(null)
|
|
const [canRetry, setCanRetry] = useState(false)
|
|
const [speakingMessageId, setSpeakingMessageId] = useState<string | null>(null)
|
|
const [reportTarget, setReportTarget] = useState<ContentReportTarget | null>(null)
|
|
const styles = useMemo(() => createStyles(palette), [palette])
|
|
const deterministicAudioAvailable = isDeterministicTalkAudioAvailable()
|
|
|
|
const replaceMessages = useCallback((next: TalkMessage[]): void => {
|
|
messagesRef.current = next
|
|
if (mountedRef.current) setMessages(next)
|
|
}, [])
|
|
|
|
const updateMessages = useCallback((update: (current: TalkMessage[]) => TalkMessage[]): void => {
|
|
replaceMessages(update(messagesRef.current))
|
|
}, [replaceMessages])
|
|
|
|
const updatePhase = useCallback((next: TalkPhase): void => {
|
|
phaseRef.current = next
|
|
if (mountedRef.current) setPhase(next)
|
|
}, [])
|
|
|
|
const isCurrentOperation = useCallback((operationId: number): boolean => (
|
|
mountedRef.current && operationRef.current === operationId
|
|
), [])
|
|
|
|
const stopSpeech = useCallback(async (): Promise<void> => {
|
|
speechOperationRef.current += 1
|
|
try {
|
|
await stopTalkSpeech()
|
|
} catch {
|
|
// An unavailable engine has no audio to stop.
|
|
}
|
|
if (!mountedRef.current) return
|
|
setSpeakingMessageId(null)
|
|
if (phaseRef.current === 'speaking') updatePhase('idle')
|
|
}, [updatePhase])
|
|
|
|
const cancelActiveOperation = useCallback((showCancelled: boolean): void => {
|
|
operationRef.current += 1
|
|
pressHeldRef.current = false
|
|
requestControllerRef.current?.abort()
|
|
requestControllerRef.current = null
|
|
transcriptionControllerRef.current?.abort()
|
|
transcriptionControllerRef.current = null
|
|
const assistantId = activeAssistantIdRef.current
|
|
activeAssistantIdRef.current = null
|
|
if (assistantId !== null) {
|
|
updateMessages((current) => current.filter((message) => message.id !== assistantId))
|
|
}
|
|
if (recordingStartingRef.current || recordingActiveRef.current || recordingFinishingRef.current) {
|
|
const cleanup = audioRecorder.cancel().catch(() => undefined)
|
|
recorderCleanupRef.current = cleanup
|
|
}
|
|
recordingStartingRef.current = false
|
|
recordingActiveRef.current = false
|
|
recordingFinishingRef.current = false
|
|
void stopSpeech()
|
|
if (!mountedRef.current) return
|
|
setRecordingProgress({ durationMs: 0, meteringDb: null })
|
|
setCanRetry(false)
|
|
retryMessagesRef.current = null
|
|
updatePhase('idle')
|
|
if (showCancelled) setErrorCode('CANCELLED')
|
|
}, [stopSpeech, updateMessages, updatePhase])
|
|
|
|
useEffect(() => {
|
|
messagesRef.current = messages
|
|
}, [messages])
|
|
|
|
useEffect(() => {
|
|
mountedRef.current = true
|
|
const subscription = AppState.addEventListener('change', (state) => {
|
|
if (state !== 'active' && phaseRef.current !== 'idle') cancelActiveOperation(false)
|
|
})
|
|
return () => {
|
|
mountedRef.current = false
|
|
subscription.remove()
|
|
cancelActiveOperation(false)
|
|
void shutdownTalkSpeech().catch(() => undefined)
|
|
}
|
|
}, [cancelActiveOperation])
|
|
|
|
useFocusEffect(useCallback(() => (
|
|
() => cancelActiveOperation(false)
|
|
), [cancelActiveOperation]))
|
|
|
|
const authIdentity = session === null ? 'signed-out' : `${session.user.id}:${session.access_token}`
|
|
const previousAuthIdentityRef = useRef(authIdentity)
|
|
useEffect(() => {
|
|
if (previousAuthIdentityRef.current !== authIdentity) {
|
|
previousAuthIdentityRef.current = authIdentity
|
|
cancelActiveOperation(false)
|
|
}
|
|
}, [authIdentity, cancelActiveOperation])
|
|
|
|
useEffect(() => {
|
|
const timer = setTimeout(() => scrollRef.current?.scrollToEnd({ animated: true }), 0)
|
|
return () => clearTimeout(timer)
|
|
}, [messages, phase])
|
|
|
|
const playSpeech = useCallback(async (
|
|
messageId: string,
|
|
content: string,
|
|
languageTag: string,
|
|
): Promise<void> => {
|
|
const speechId = ++speechOperationRef.current
|
|
try {
|
|
await stopTalkSpeech().catch(() => false)
|
|
if (!mountedRef.current || speechOperationRef.current !== speechId) return
|
|
setSpeakingMessageId(messageId)
|
|
updatePhase('speaking')
|
|
const result = await speakTalkText({
|
|
text: content,
|
|
languageTag,
|
|
utteranceId: `talk-${messageId}-${speechId}`,
|
|
})
|
|
if (!mountedRef.current || speechOperationRef.current !== speechId) return
|
|
if (result.status === 'completed' || result.status === 'stopped') {
|
|
setSpeakingMessageId(null)
|
|
updatePhase('idle')
|
|
}
|
|
} catch (error) {
|
|
if (!mountedRef.current || speechOperationRef.current !== speechId) return
|
|
setSpeakingMessageId(null)
|
|
updatePhase('idle')
|
|
setErrorCode(error instanceof TalkTtsError && error.code === 'UNAVAILABLE'
|
|
? 'TTS_UNAVAILABLE'
|
|
: 'TTS_FAILED')
|
|
}
|
|
}, [updatePhase])
|
|
|
|
const runConversation = useCallback(async (
|
|
conversation: TalkMessage[],
|
|
autoSpeak: boolean,
|
|
operationId: number,
|
|
): Promise<void> => {
|
|
const currentSession = sessionRef.current
|
|
if (currentSession === null) {
|
|
if (isCurrentOperation(operationId)) {
|
|
setErrorCode('AUTH_REQUIRED')
|
|
setCanRetry(false)
|
|
updatePhase('idle')
|
|
}
|
|
return
|
|
}
|
|
|
|
const controller = new AbortController()
|
|
requestControllerRef.current = controller
|
|
const assistantId = `assistant-${Date.now()}-${operationId}`
|
|
activeAssistantIdRef.current = assistantId
|
|
const placeholder: TalkMessage = {
|
|
id: assistantId,
|
|
role: 'assistant',
|
|
content: '',
|
|
createdAt: Date.now(),
|
|
}
|
|
replaceMessages([...conversation, placeholder])
|
|
updatePhase('thinking')
|
|
setErrorCode(null)
|
|
setCanRetry(false)
|
|
|
|
try {
|
|
const { text: content, generationId } = await streamTalkResponse(
|
|
conversation.map(({ role, content: messageContent }) => ({
|
|
role,
|
|
content: messageContent,
|
|
})),
|
|
{
|
|
accessToken: currentSession.access_token,
|
|
model: preferences.preferredLlmModel,
|
|
signal: controller.signal,
|
|
onTextDelta: (_delta, accumulated) => {
|
|
if (!isCurrentOperation(operationId)) return
|
|
updateMessages((current) => current.map((message) => (
|
|
message.id === assistantId ? { ...message, content: accumulated } : message
|
|
)))
|
|
},
|
|
},
|
|
)
|
|
if (!isCurrentOperation(operationId)) return
|
|
const latestSession = sessionRef.current
|
|
if (
|
|
latestSession === null
|
|
|| latestSession.user.id !== currentSession.user.id
|
|
|| latestSession.access_token !== currentSession.access_token
|
|
) {
|
|
throw new ChatServiceError('AUTH_REQUIRED', false, 401)
|
|
}
|
|
|
|
updateMessages((current) => current.map((message) => (
|
|
message.id === assistantId ? { ...message, content, generationId } : message
|
|
)))
|
|
activeAssistantIdRef.current = null
|
|
requestControllerRef.current = null
|
|
retryMessagesRef.current = null
|
|
setCanRetry(false)
|
|
updatePhase('idle')
|
|
if (autoSpeak) {
|
|
await playSpeech(
|
|
assistantId,
|
|
content,
|
|
preferences.locale === 'en' ? 'en-US' : 'ko-KR',
|
|
)
|
|
}
|
|
} catch (error) {
|
|
if (!isCurrentOperation(operationId)) return
|
|
const serviceError = error instanceof ChatServiceError
|
|
? error
|
|
: new ChatServiceError('SERVER_ERROR', true)
|
|
updateMessages((current) => current.filter((message) => message.id !== assistantId))
|
|
activeAssistantIdRef.current = null
|
|
requestControllerRef.current = null
|
|
retryMessagesRef.current = conversation
|
|
retryAutoSpeakRef.current = autoSpeak
|
|
setErrorCode(serviceError.code)
|
|
setCanRetry(serviceError.retryable)
|
|
updatePhase('idle')
|
|
}
|
|
}, [
|
|
isCurrentOperation,
|
|
playSpeech,
|
|
preferences.locale,
|
|
preferences.preferredLlmModel,
|
|
replaceMessages,
|
|
updateMessages,
|
|
updatePhase,
|
|
])
|
|
|
|
const submitMessage = useCallback((content: string, autoSpeak: boolean): void => {
|
|
const normalized = content.trim()
|
|
if (normalized.length === 0 || normalized.length > 8_000) return
|
|
const operationId = ++operationRef.current
|
|
const userMessage: TalkMessage = {
|
|
id: `user-${Date.now()}-${operationId}`,
|
|
role: 'user',
|
|
content: normalized,
|
|
createdAt: Date.now(),
|
|
}
|
|
const conversation = [...messagesRef.current, userMessage].slice(-40)
|
|
replaceMessages(conversation)
|
|
setErrorCode(null)
|
|
setCanRetry(false)
|
|
retryMessagesRef.current = null
|
|
void runConversation(conversation, autoSpeak, operationId)
|
|
}, [replaceMessages, runConversation])
|
|
|
|
const finishRecording = useCallback(async (operationId: number): Promise<void> => {
|
|
if (recordingFinishingRef.current || !recordingActiveRef.current) return
|
|
recordingFinishingRef.current = true
|
|
recordingActiveRef.current = false
|
|
|
|
try {
|
|
const recording = await audioRecorder.stop()
|
|
if (!isCurrentOperation(operationId)) {
|
|
await audioRecorder.cleanup(recording)
|
|
return
|
|
}
|
|
if (recording.durationMs < MIN_RECORDING_MS) {
|
|
await audioRecorder.cleanup(recording)
|
|
setErrorCode('RECORDING_TOO_SHORT')
|
|
updatePhase('idle')
|
|
return
|
|
}
|
|
|
|
const controller = new AbortController()
|
|
transcriptionControllerRef.current = controller
|
|
updatePhase('transcribing')
|
|
const currentSession = sessionRef.current
|
|
if (currentSession === null) {
|
|
await audioRecorder.cleanup(recording)
|
|
throw new AudioPipelineError('auth', 'Sign in to transcribe speech')
|
|
}
|
|
|
|
const result = await transcribeTalkRecording(recording, {
|
|
accessToken: currentSession.access_token,
|
|
languageCode: preferences.locale,
|
|
signal: controller.signal,
|
|
disposeRecording: () => audioRecorder.cleanup(recording),
|
|
})
|
|
transcriptionControllerRef.current = null
|
|
if (!isCurrentOperation(operationId)) return
|
|
const latestSession = sessionRef.current
|
|
if (
|
|
latestSession === null
|
|
|| latestSession.user.id !== currentSession.user.id
|
|
|| latestSession.access_token !== currentSession.access_token
|
|
) {
|
|
throw new AudioPipelineError('auth', 'The account changed during transcription')
|
|
}
|
|
setInput('')
|
|
submitMessage(result.text, true)
|
|
} catch (error) {
|
|
// 진단: 클라우드/기기 전사의 실제 원인(상태코드·whisper 오류)은
|
|
// UI 번역키로 덮이기 전에 로그로 남긴다. Metro/logcat에서 확인 가능.
|
|
const causeChain: string[] = []
|
|
for (
|
|
let e: unknown = error;
|
|
e instanceof Error && causeChain.length < 4;
|
|
e = (e as Error & { cause?: unknown }).cause
|
|
) {
|
|
causeChain.push(e.message)
|
|
}
|
|
console.warn(
|
|
`[Talk] transcription failed (${error instanceof AudioPipelineError ? error.code : 'unknown'}):`,
|
|
causeChain.join(' | '),
|
|
)
|
|
if (!isCurrentOperation(operationId)) return
|
|
transcriptionControllerRef.current = null
|
|
setErrorCode(transcriptionErrorCode(error))
|
|
setCanRetry(false)
|
|
updatePhase('idle')
|
|
} finally {
|
|
recordingFinishingRef.current = false
|
|
if (mountedRef.current) setRecordingProgress({ durationMs: 0, meteringDb: null })
|
|
}
|
|
}, [isCurrentOperation, preferences.locale, submitMessage, updatePhase])
|
|
|
|
const beginRecording = useCallback(async (): Promise<void> => {
|
|
if (recordingStartingRef.current || recordingActiveRef.current) return
|
|
if (phaseRef.current !== 'idle' && phaseRef.current !== 'speaking') return
|
|
pressHeldRef.current = true
|
|
const operationId = ++operationRef.current
|
|
recordingStartingRef.current = true
|
|
setErrorCode(null)
|
|
setCanRetry(false)
|
|
retryMessagesRef.current = null
|
|
await stopSpeech()
|
|
|
|
try {
|
|
await recorderCleanupRef.current
|
|
if (!isCurrentOperation(operationId)) return
|
|
const currentSession = sessionRef.current
|
|
if (currentSession === null) {
|
|
setErrorCode('AUTH_REQUIRED')
|
|
return
|
|
}
|
|
const granted = await audioRecorder.requestPermission()
|
|
if (!isCurrentOperation(operationId)) return
|
|
if (!granted) {
|
|
setErrorCode('MIC_PERMISSION_DENIED')
|
|
return
|
|
}
|
|
|
|
await audioRecorder.start((progress) => {
|
|
if (isCurrentOperation(operationId)) setRecordingProgress(progress)
|
|
})
|
|
if (!isCurrentOperation(operationId)) {
|
|
await audioRecorder.cancel().catch(() => undefined)
|
|
return
|
|
}
|
|
recordingActiveRef.current = true
|
|
updatePhase('recording')
|
|
if (!pressHeldRef.current) await finishRecording(operationId)
|
|
} catch {
|
|
if (isCurrentOperation(operationId)) {
|
|
setErrorCode('RECORDING_FAILED')
|
|
updatePhase('idle')
|
|
}
|
|
await audioRecorder.cancel().catch(() => undefined)
|
|
} finally {
|
|
recordingStartingRef.current = false
|
|
}
|
|
}, [finishRecording, isCurrentOperation, stopSpeech, updatePhase])
|
|
|
|
const releaseRecording = useCallback((): void => {
|
|
pressHeldRef.current = false
|
|
if (!recordingActiveRef.current) return
|
|
void finishRecording(operationRef.current)
|
|
}, [finishRecording])
|
|
|
|
const handleSend = useCallback((): void => {
|
|
const content = input.trim()
|
|
if (textSendPendingRef.current || content.length === 0 || phaseRef.current === 'recording'
|
|
|| phaseRef.current === 'transcribing' || phaseRef.current === 'thinking') return
|
|
textSendPendingRef.current = true
|
|
setInput('')
|
|
void stopSpeech()
|
|
.then(() => {
|
|
if (mountedRef.current) submitMessage(content, false)
|
|
})
|
|
.finally(() => {
|
|
textSendPendingRef.current = false
|
|
})
|
|
}, [input, stopSpeech, submitMessage])
|
|
|
|
const handleRetry = useCallback((): void => {
|
|
const retryMessages = retryMessagesRef.current
|
|
if (!canRetry || retryMessages === null || phaseRef.current !== 'idle') return
|
|
const operationId = ++operationRef.current
|
|
void runConversation(retryMessages, retryAutoSpeakRef.current, operationId)
|
|
}, [canRetry, runConversation])
|
|
|
|
const runDeterministicAudioE2E = useCallback(async (): Promise<void> => {
|
|
if (!deterministicAudioAvailable || phaseRef.current !== 'idle') return
|
|
const currentSession = sessionRef.current
|
|
if (currentSession === null) {
|
|
setErrorCode('AUTH_REQUIRED')
|
|
return
|
|
}
|
|
const operationId = ++operationRef.current
|
|
setErrorCode(null)
|
|
setCanRetry(false)
|
|
await stopSpeech()
|
|
let fixture: Awaited<ReturnType<typeof synthesizeDeterministicTalkAudio>> | null = null
|
|
try {
|
|
fixture = await synthesizeDeterministicTalkAudio({
|
|
text: preferences.locale === 'en'
|
|
? 'Mobile voice transcription verified'
|
|
: '모바일 음성 전사 검증',
|
|
languageTag: preferences.locale === 'en' ? 'en-US' : 'ko-KR',
|
|
})
|
|
if (!isCurrentOperation(operationId)) {
|
|
await fixture.dispose()
|
|
return
|
|
}
|
|
const controller = new AbortController()
|
|
transcriptionControllerRef.current = controller
|
|
updatePhase('transcribing')
|
|
const result = await transcribeTalkRecording(fixture.recording, {
|
|
accessToken: currentSession.access_token,
|
|
languageCode: preferences.locale,
|
|
signal: controller.signal,
|
|
disposeRecording: fixture.dispose,
|
|
})
|
|
fixture = null
|
|
transcriptionControllerRef.current = null
|
|
if (!isCurrentOperation(operationId)) return
|
|
const latestSession = sessionRef.current
|
|
if (
|
|
latestSession === null ||
|
|
latestSession.user.id !== currentSession.user.id ||
|
|
latestSession.access_token !== currentSession.access_token
|
|
) {
|
|
throw new AudioPipelineError('auth', 'The account changed during transcription')
|
|
}
|
|
submitMessage(result.text, true)
|
|
} catch (injectionError) {
|
|
if (fixture !== null) await fixture.dispose().catch(() => undefined)
|
|
transcriptionControllerRef.current = null
|
|
if (!isCurrentOperation(operationId)) return
|
|
setErrorCode(transcriptionErrorCode(injectionError))
|
|
setCanRetry(false)
|
|
updatePhase('idle')
|
|
}
|
|
}, [
|
|
deterministicAudioAvailable,
|
|
isCurrentOperation,
|
|
preferences.locale,
|
|
stopSpeech,
|
|
submitMessage,
|
|
updatePhase,
|
|
])
|
|
|
|
const handleClear = useCallback((): void => {
|
|
cancelActiveOperation(false)
|
|
replaceMessages([])
|
|
setInput('')
|
|
setErrorCode(null)
|
|
setCanRetry(false)
|
|
}, [cancelActiveOperation, replaceMessages])
|
|
|
|
const handleSpeechPress = useCallback((message: TalkMessage): void => {
|
|
if (phaseRef.current !== 'idle' && phaseRef.current !== 'speaking') return
|
|
if (speakingMessageId === message.id) {
|
|
void stopSpeech()
|
|
return
|
|
}
|
|
void playSpeech(
|
|
message.id,
|
|
message.content,
|
|
preferences.locale === 'en' ? 'en-US' : 'ko-KR',
|
|
)
|
|
}, [playSpeech, preferences.locale, speakingMessageId, stopSpeech])
|
|
|
|
const inputBlocked = phase === 'recording' || phase === 'transcribing' || phase === 'thinking'
|
|
const meterLevel = recordingProgress.meteringDb === null
|
|
? 0.15
|
|
: Math.max(0.08, Math.min(1, (recordingProgress.meteringDb + 60) / 60))
|
|
|
|
return (
|
|
<KeyboardAvoidingView
|
|
style={styles.container}
|
|
behavior={Platform.OS === 'ios' ? 'padding' : 'height'}
|
|
keyboardVerticalOffset={80}
|
|
>
|
|
<View style={[styles.header, { paddingTop: insets.top + 10 }]}>
|
|
<View>
|
|
<ThemeText variant="title" accessibilityRole="header">{t('mobile.talk.title')}</ThemeText>
|
|
<ThemeText variant="label" color={phase === 'idle' ? 'muted' : 'accent'}>
|
|
{t(phaseTranslationKey(phase))}
|
|
</ThemeText>
|
|
</View>
|
|
<Pressable
|
|
accessibilityRole="button"
|
|
accessibilityLabel={t('mobile.talk.clear')}
|
|
onPress={handleClear}
|
|
style={styles.clearButton}
|
|
testID="talk-clear"
|
|
>
|
|
<ThemeText variant="label" color="muted">{t('mobile.talk.clear')}</ThemeText>
|
|
</Pressable>
|
|
</View>
|
|
|
|
<ScrollView
|
|
ref={scrollRef}
|
|
style={styles.chatArea}
|
|
contentContainerStyle={styles.chatContent}
|
|
keyboardShouldPersistTaps="handled"
|
|
testID="talk-messages"
|
|
>
|
|
{messages.length === 0 && (
|
|
<ThemeText color="muted" style={styles.emptyText}>{t('mobile.talk.empty')}</ThemeText>
|
|
)}
|
|
{messages.map((message) => (
|
|
<View
|
|
key={message.id}
|
|
style={[
|
|
styles.bubble,
|
|
message.role === 'user' ? styles.userBubble : styles.assistantBubble,
|
|
]}
|
|
>
|
|
{message.content.length > 0 && (
|
|
<ThemeText color={message.role === 'user' ? 'accent' : 'primary'}>
|
|
{message.content}
|
|
</ThemeText>
|
|
)}
|
|
{message.content.length === 0 && message.id === activeAssistantIdRef.current && (
|
|
<ActivityIndicator size="small" color={palette.accent.main} />
|
|
)}
|
|
<View style={styles.bubbleFooter}>
|
|
<ThemeText variant="label" color="muted" style={styles.bubbleLabel}>
|
|
{message.role === 'user' ? t('mobile.talk.you') : t('mobile.talk.claude')}
|
|
</ThemeText>
|
|
{message.role === 'assistant' && message.content.length > 0 && message.id !== 'welcome' && (
|
|
<View style={styles.bubbleActions}>
|
|
{message.generationId !== undefined && (
|
|
<Pressable
|
|
accessibilityRole="button"
|
|
accessibilityLabel={t('mobile.report.action')}
|
|
onPress={() => setReportTarget({
|
|
sourceType: 'talk_response',
|
|
generationId: message.generationId!,
|
|
snapshot: message.content.slice(0, 4_000),
|
|
})}
|
|
hitSlop={8}
|
|
testID={`talk-report-${message.id}`}
|
|
>
|
|
<ThemeText variant="label" color="muted">
|
|
{t('mobile.report.action')}
|
|
</ThemeText>
|
|
</Pressable>
|
|
)}
|
|
<Pressable
|
|
accessibilityRole="button"
|
|
accessibilityLabel={speakingMessageId === message.id
|
|
? t('mobile.talk.stopSpeech')
|
|
: t('mobile.talk.playSpeech')}
|
|
onPress={() => handleSpeechPress(message)}
|
|
hitSlop={8}
|
|
testID={`talk-speech-${message.id}`}
|
|
>
|
|
<ThemeText variant="label" color={speakingMessageId === message.id ? 'accent' : 'muted'}>
|
|
{speakingMessageId === message.id ? '■' : '▶'}
|
|
</ThemeText>
|
|
</Pressable>
|
|
</View>
|
|
)}
|
|
</View>
|
|
</View>
|
|
))}
|
|
|
|
{(phase === 'transcribing' || phase === 'thinking') && (
|
|
<View style={styles.busyRow} accessibilityLiveRegion="polite" testID="talk-busy">
|
|
<ActivityIndicator size="small" color={palette.accent.main} />
|
|
<ThemeText color="muted">
|
|
{phase === 'transcribing'
|
|
? t('mobile.talk.transcribing')
|
|
: t('mobile.talk.thinking')}
|
|
</ThemeText>
|
|
<ThemeButton
|
|
label={t('mobile.talk.cancel')}
|
|
variant="quiet"
|
|
onPress={() => cancelActiveOperation(true)}
|
|
style={styles.inlineCancel}
|
|
testID="talk-cancel"
|
|
/>
|
|
</View>
|
|
)}
|
|
|
|
{errorCode !== null && (
|
|
<View style={styles.errorCard} accessibilityLiveRegion="assertive" testID="talk-error">
|
|
<ThemeText color="danger">{t(errorTranslationKey(errorCode))}</ThemeText>
|
|
{canRetry && (
|
|
<ThemeButton
|
|
label={t('mobile.talk.retry')}
|
|
variant="secondary"
|
|
onPress={handleRetry}
|
|
style={styles.retryButton}
|
|
testID="talk-retry"
|
|
/>
|
|
)}
|
|
</View>
|
|
)}
|
|
</ScrollView>
|
|
|
|
{phase === 'recording' && (
|
|
<View style={styles.recordingPanel} testID="talk-recording">
|
|
<View style={styles.recordingStatus}>
|
|
<View style={styles.liveDot} />
|
|
<ThemeText color="danger">{t('mobile.talk.recording')}</ThemeText>
|
|
<ThemeText color="muted">{formatDuration(recordingProgress.durationMs)}</ThemeText>
|
|
</View>
|
|
<View style={styles.meterTrack}>
|
|
<View style={[styles.meterFill, { width: `${Math.round(meterLevel * 100)}%` }]} />
|
|
</View>
|
|
<ThemeText variant="label" color="muted">{t('mobile.talk.releaseToSend')}</ThemeText>
|
|
</View>
|
|
)}
|
|
|
|
<View style={[styles.inputArea, { paddingBottom: Math.max(insets.bottom, 8) }]}>
|
|
{deterministicAudioAvailable && (
|
|
<ThemeButton
|
|
label={t('mobile.talk.debugAudioE2e')}
|
|
variant="quiet"
|
|
disabled={phase !== 'idle'}
|
|
onPress={() => { void runDeterministicAudioE2E() }}
|
|
testID="talk-debug-audio-e2e"
|
|
/>
|
|
)}
|
|
<View style={styles.inputRow}>
|
|
<Pressable
|
|
accessibilityRole="button"
|
|
accessibilityLabel={t('mobile.talk.holdToTalk')}
|
|
disabled={phase === 'transcribing' || phase === 'thinking'}
|
|
onPressIn={() => { void beginRecording() }}
|
|
onPressOut={releaseRecording}
|
|
style={({ pressed }) => [
|
|
styles.micButton,
|
|
(pressed || phase === 'recording') && styles.micButtonActive,
|
|
(phase === 'transcribing' || phase === 'thinking') && styles.disabledButton,
|
|
]}
|
|
testID="talk-ptt"
|
|
>
|
|
<ThemeText color={phase === 'recording' ? 'onAccent' : 'accent'}>●</ThemeText>
|
|
</Pressable>
|
|
<TextInput
|
|
accessibilityLabel={t('mobile.talk.placeholder')}
|
|
style={styles.textInput}
|
|
placeholder={t('mobile.talk.placeholder')}
|
|
placeholderTextColor={palette.text.muted}
|
|
value={input}
|
|
onChangeText={setInput}
|
|
onSubmitEditing={handleSend}
|
|
editable={!inputBlocked}
|
|
maxLength={8_000}
|
|
multiline
|
|
returnKeyType="send"
|
|
blurOnSubmit
|
|
testID="talk-input"
|
|
/>
|
|
<Pressable
|
|
accessibilityRole="button"
|
|
accessibilityLabel={t('mobile.talk.send')}
|
|
disabled={inputBlocked || input.trim().length === 0}
|
|
onPress={handleSend}
|
|
style={({ pressed }) => [
|
|
styles.sendButton,
|
|
(inputBlocked || input.trim().length === 0) && styles.disabledButton,
|
|
pressed && styles.pressedButton,
|
|
]}
|
|
testID="talk-send"
|
|
>
|
|
<ThemeText color="onAccent">↑</ThemeText>
|
|
</Pressable>
|
|
</View>
|
|
<ThemeText variant="label" color="muted" style={styles.pttHint}>
|
|
{t('mobile.talk.pttHint')}
|
|
</ThemeText>
|
|
</View>
|
|
|
|
<ContentReportSheet
|
|
visible={reportTarget !== null}
|
|
accessToken={session?.access_token ?? null}
|
|
target={reportTarget}
|
|
onRequestClose={() => setReportTarget(null)}
|
|
/>
|
|
</KeyboardAvoidingView>
|
|
)
|
|
}
|
|
|
|
type Palette = ReturnType<typeof useMobilePreferences>['palette']
|
|
|
|
function createStyles(palette: Palette): ReturnType<typeof StyleSheet.create> {
|
|
const bubbleBase: ViewStyle = {
|
|
maxWidth: '88%',
|
|
padding: 16,
|
|
borderRadius: 16,
|
|
borderWidth: 1,
|
|
}
|
|
return StyleSheet.create({
|
|
container: { flex: 1, backgroundColor: palette.bg.app },
|
|
header: {
|
|
minHeight: 76,
|
|
paddingHorizontal: 20,
|
|
paddingBottom: 12,
|
|
flexDirection: 'row',
|
|
alignItems: 'center',
|
|
justifyContent: 'space-between',
|
|
borderBottomWidth: 1,
|
|
borderBottomColor: palette.border.default,
|
|
backgroundColor: palette.bg.sidebar,
|
|
},
|
|
clearButton: { minWidth: 48, minHeight: 44, alignItems: 'center', justifyContent: 'center' },
|
|
chatArea: { flex: 1 },
|
|
chatContent: { padding: 20, paddingBottom: 28, gap: 14 },
|
|
emptyText: { textAlign: 'center', paddingVertical: 48 },
|
|
bubble: bubbleBase,
|
|
assistantBubble: {
|
|
alignSelf: 'flex-start',
|
|
backgroundColor: palette.bg.card,
|
|
borderColor: palette.border.default,
|
|
borderTopLeftRadius: 4,
|
|
},
|
|
userBubble: {
|
|
alignSelf: 'flex-end',
|
|
backgroundColor: palette.accent.dim,
|
|
borderColor: palette.accent.main,
|
|
borderTopRightRadius: 4,
|
|
},
|
|
bubbleFooter: {
|
|
marginTop: 8,
|
|
minHeight: 20,
|
|
flexDirection: 'row',
|
|
alignItems: 'center',
|
|
justifyContent: 'space-between',
|
|
gap: 16,
|
|
},
|
|
bubbleLabel: { fontSize: 9, letterSpacing: 1 },
|
|
bubbleActions: { flexDirection: 'row', alignItems: 'center', gap: 18 },
|
|
busyRow: { flexDirection: 'row', alignItems: 'center', gap: 10, paddingVertical: 12 },
|
|
inlineCancel: { minHeight: 36, marginLeft: 'auto', paddingVertical: 4 },
|
|
errorCard: {
|
|
backgroundColor: palette.bg.card,
|
|
borderColor: palette.tag.red,
|
|
borderWidth: 1,
|
|
borderRadius: 12,
|
|
padding: 14,
|
|
gap: 10,
|
|
},
|
|
retryButton: { alignSelf: 'flex-start', minHeight: 42 },
|
|
recordingPanel: {
|
|
backgroundColor: palette.bg.card,
|
|
borderTopWidth: 1,
|
|
borderTopColor: palette.tag.red,
|
|
paddingHorizontal: 20,
|
|
paddingVertical: 12,
|
|
gap: 8,
|
|
},
|
|
recordingStatus: { flexDirection: 'row', alignItems: 'center', gap: 10 },
|
|
liveDot: {
|
|
width: 9,
|
|
height: 9,
|
|
borderRadius: 5,
|
|
backgroundColor: palette.tag.red,
|
|
},
|
|
meterTrack: {
|
|
height: 4,
|
|
overflow: 'hidden',
|
|
borderRadius: 2,
|
|
backgroundColor: palette.bg.inset,
|
|
},
|
|
meterFill: { height: 4, borderRadius: 2, backgroundColor: palette.accent.main },
|
|
inputArea: {
|
|
backgroundColor: palette.bg.sidebar,
|
|
borderTopWidth: 1,
|
|
borderTopColor: palette.border.default,
|
|
paddingHorizontal: 12,
|
|
paddingTop: 12,
|
|
},
|
|
inputRow: {
|
|
flexDirection: 'row',
|
|
alignItems: 'flex-end',
|
|
backgroundColor: palette.bg.inset,
|
|
borderRadius: 12,
|
|
borderWidth: 1,
|
|
borderColor: palette.border.default,
|
|
padding: 8,
|
|
gap: 8,
|
|
},
|
|
textInput: {
|
|
flex: 1,
|
|
minHeight: 40,
|
|
maxHeight: 120,
|
|
color: palette.text.primary,
|
|
fontSize: 14,
|
|
paddingHorizontal: 8,
|
|
paddingVertical: 8,
|
|
},
|
|
micButton: {
|
|
width: 40,
|
|
height: 40,
|
|
borderRadius: 20,
|
|
borderWidth: 1,
|
|
borderColor: palette.accent.main,
|
|
justifyContent: 'center',
|
|
alignItems: 'center',
|
|
},
|
|
micButtonActive: { backgroundColor: palette.accent.main },
|
|
sendButton: {
|
|
width: 40,
|
|
height: 40,
|
|
borderRadius: 20,
|
|
backgroundColor: palette.accent.main,
|
|
justifyContent: 'center',
|
|
alignItems: 'center',
|
|
},
|
|
disabledButton: { opacity: 0.4 },
|
|
pressedButton: { backgroundColor: palette.accent.pressed },
|
|
pttHint: { paddingTop: 8, paddingBottom: 2, textAlign: 'center' },
|
|
})
|
|
}
|