fix(llm): stop leaking instruction prompts instead of processed text
Running a custom instruction (translate, summarise, rewrite, explain code,
free prompt) inserted the instruction's own wording instead of the result.
Two faults stacked:
The instruction was passed as the text to process, leaving the system-prompt
argument empty. `BASE_SYSTEM_PROMPTS` has no `custom` key, so resolution fell
back to `refine` without saying so, and the model dutifully polished the
instruction it had been handed. The transcript never reached it.
And only `{{text}}` was substituted, which none of the five built-in
instructions use — they carry `{{targetLanguage}}`, `{{userPrompt}}`, or no
placeholder at all. The substitution was a no-op from the day it was written:
the presets landed ten hours before the code that expected them.
- Instruction prompts now go to the system-prompt argument and the transcript
to the text argument. Instructions that spell out `{{text}}` keep their old
meaning, so hand-written ones still work.
- `renderInstructionPrompt` resolves `{{text}}`, `{{userPrompt}}` and
`{{targetLanguage}}` in one place, and warns by name when a placeholder is
left standing rather than letting it reach the model.
- `resolveSystemPrompt` no longer drops silently to `refine` for `custom`.
- Voice shortcuts no longer die at the `defaultLLMAction === 'none'` gate; an
explicitly named instruction outranks the default. Without one, `none` still
passes the transcript through untouched.
- `translate` receives its target language instead of relying on a default two
call frames away. It is still always English — `AppConfig` has no key for it,
and neither `language` (UI locale) nor `sttLanguage` (source language) can
stand in. Choosing a target language needs a setting and is not in this fix.
- Chains ran instructions with placeholders intact; they share the same
resolution now.
- The command screen's pipeline bench called `llm.generate`, which preload does
not expose, so every run threw and the catch showed the input back as if it
had succeeded. It uses `llm.process` now, over the same path production
takes, and a failure reads as a failure.
Present since the feature shipped: the custom-instruction path has never
worked. Plain actions (refine, summarise, grammar, expand) were unaffected and
are now covered by tests so they stay that way.
This commit is contained in:
Yun Chan
parent
30d51c952f
commit
99f06c253c
14 changed files with 992 additions and 44 deletions
|
|
@ -60,15 +60,49 @@ vi.mock('../../../src/main/services/KeyBindingService', () => ({
|
|||
getKeyBindingService: () => mockKeyBinding
|
||||
}))
|
||||
|
||||
const CONFIG_DEFAULTS: Record<string, unknown> = {
|
||||
sttModelId: 'base',
|
||||
defaultLLMAction: 'refine',
|
||||
ollamaServerUrl: 'http://localhost:11434',
|
||||
llmModelId: 'gemma4:e4b'
|
||||
}
|
||||
|
||||
const config = vi.hoisted(() => ({ values: {} as Record<string, unknown> }))
|
||||
|
||||
vi.mock('../../../src/main/services/ConfigService', () => ({
|
||||
configGet: vi.fn((key: string) => {
|
||||
const defaults: Record<string, unknown> = {
|
||||
sttModelId: 'base',
|
||||
defaultLLMAction: 'refine',
|
||||
ollamaServerUrl: 'http://localhost:11434',
|
||||
llmModelId: 'gemma4:e4b'
|
||||
}
|
||||
return defaults[key]
|
||||
configGet: vi.fn((key: string) => config.values[key])
|
||||
}))
|
||||
|
||||
const instructionStore = vi.hoisted(() => ({
|
||||
byId: {} as Record<string, { id: string; name: string; prompt: string }>
|
||||
}))
|
||||
|
||||
vi.mock('../../../src/main/services/CustomInstructionService', () => ({
|
||||
getCustomInstructionService: () => ({
|
||||
getById: (id: string) => instructionStore.byId[id] ?? null
|
||||
})
|
||||
}))
|
||||
|
||||
// 음성 단축키(두 번째 진입점) — 기본은 비활성
|
||||
const voiceCommand = vi.hoisted(() => ({
|
||||
enabled: false,
|
||||
instructionId: null as string | null,
|
||||
cleanedText: ''
|
||||
}))
|
||||
|
||||
vi.mock('../../../src/main/services/VoiceCommandService', () => ({
|
||||
getVoiceCommandService: () => ({
|
||||
isEnabled: () => voiceCommand.enabled,
|
||||
match: (text: string) =>
|
||||
voiceCommand.enabled && voiceCommand.instructionId
|
||||
? {
|
||||
matched: true,
|
||||
ruleId: 'rule-1',
|
||||
instructionId: voiceCommand.instructionId,
|
||||
cleanedText: voiceCommand.cleanedText || text,
|
||||
matchedKeyword: '번역'
|
||||
}
|
||||
: { matched: false, ruleId: null, instructionId: null, cleanedText: text, matchedKeyword: null }
|
||||
})
|
||||
}))
|
||||
|
||||
|
|
@ -97,6 +131,11 @@ beforeEach(async () => {
|
|||
vi.resetModules()
|
||||
vi.clearAllMocks()
|
||||
audioBus.removeAllListeners()
|
||||
config.values = { ...CONFIG_DEFAULTS }
|
||||
instructionStore.byId = {}
|
||||
voiceCommand.enabled = false
|
||||
voiceCommand.instructionId = null
|
||||
voiceCommand.cleanedText = ''
|
||||
mockSTT.initialize.mockResolvedValue(undefined as never)
|
||||
mockSTT.transcribe.mockResolvedValue({
|
||||
text: '테스트 전사',
|
||||
|
|
@ -223,6 +262,211 @@ describe('VoiceModeService', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('LLM 후처리 인자 전달', () => {
|
||||
const TRANSCRIPT = '회의 끝나고 배포하자'
|
||||
|
||||
/** 한 세션을 끝까지 돌린다. */
|
||||
async function runSession(): Promise<void> {
|
||||
mockSTT.transcribe.mockResolvedValue({
|
||||
text: TRANSCRIPT,
|
||||
segments: [],
|
||||
language: 'ko',
|
||||
duration: 2,
|
||||
processingTime: 50,
|
||||
} as never)
|
||||
|
||||
const svc = getVoiceModeService()
|
||||
const settled = new Promise<void>((resolve) => {
|
||||
svc.once('session-completed', () => resolve())
|
||||
svc.once('session-cancelled', () => resolve())
|
||||
})
|
||||
|
||||
await svc.startSession('dictation')
|
||||
audioBus.emit('audio-data', { buffer: Buffer.alloc(16000 * 2) })
|
||||
await new Promise((r) => setTimeout(r, 850))
|
||||
await svc.stopSession()
|
||||
await Promise.race([settled, new Promise((r) => setTimeout(r, 2000))])
|
||||
}
|
||||
|
||||
function registerInstruction(inst: { id: string; name: string; prompt: string }): void {
|
||||
instructionStore.byId[inst.id] = inst
|
||||
}
|
||||
|
||||
// ── 고친 경로: 커스텀 지시문 ───────────────────────────
|
||||
describe('커스텀 지시문 경로 (고친 경로)', () => {
|
||||
it('{{text}} 없는 지시문은 시스템 프롬프트로, 전사 텍스트는 처리 대상으로 전달한다', async () => {
|
||||
registerInstruction({
|
||||
id: 'builtin-summarize',
|
||||
name: '요약',
|
||||
prompt: '다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.\n요약문만 출력하세요.',
|
||||
})
|
||||
config.values.defaultLLMAction = 'custom'
|
||||
config.values.activeInstructionId = 'builtin-summarize'
|
||||
|
||||
await runSession()
|
||||
|
||||
expect(mockLLM.processText).toHaveBeenCalledTimes(1)
|
||||
const [text, action, targetLanguage, systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(action).toBe('custom')
|
||||
expect(targetLanguage).toBeUndefined()
|
||||
expect(systemPrompt).toBe(
|
||||
'다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.\n요약문만 출력하세요.',
|
||||
)
|
||||
})
|
||||
|
||||
it('builtin-translate의 {{targetLanguage}}를 치환해 시스템 프롬프트로 전달한다', async () => {
|
||||
registerInstruction({
|
||||
id: 'builtin-translate',
|
||||
name: '번역',
|
||||
prompt:
|
||||
'다음 텍스트를 {{targetLanguage}}로 번역해주세요.\n자연스럽고 정확한 번역만 출력하세요.',
|
||||
})
|
||||
config.values.defaultLLMAction = 'custom'
|
||||
config.values.activeInstructionId = 'builtin-translate'
|
||||
|
||||
await runSession()
|
||||
const [text, , , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(systemPrompt).toContain('English로 번역')
|
||||
expect(systemPrompt).not.toContain('{{')
|
||||
})
|
||||
|
||||
it('builtin-free-prompt의 {{userPrompt}}를 전사 텍스트로 치환한다', async () => {
|
||||
registerInstruction({
|
||||
id: 'builtin-free-prompt',
|
||||
name: '자유 프롬프트',
|
||||
prompt: '{{userPrompt}}',
|
||||
})
|
||||
config.values.defaultLLMAction = 'custom'
|
||||
config.values.activeInstructionId = 'builtin-free-prompt'
|
||||
|
||||
await runSession()
|
||||
const [text, , , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(systemPrompt).toBe(TRANSCRIPT)
|
||||
})
|
||||
|
||||
it('{{text}}를 쓰는 사용자 정의 지시문은 치환 결과를 처리 대상 텍스트로 넘긴다', async () => {
|
||||
registerInstruction({
|
||||
id: 'user-bullets',
|
||||
name: '불릿 정리',
|
||||
prompt: '아래 내용을 불릿으로 정리해줘:\n{{text}}',
|
||||
})
|
||||
config.values.defaultLLMAction = 'custom'
|
||||
config.values.activeInstructionId = 'user-bullets'
|
||||
|
||||
await runSession()
|
||||
const [text, action, , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(`아래 내용을 불릿으로 정리해줘:\n${TRANSCRIPT}`)
|
||||
expect(action).toBe('custom')
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
})
|
||||
|
||||
it('지시문을 찾지 못하면 전사 텍스트만 전달한다', async () => {
|
||||
config.values.defaultLLMAction = 'custom'
|
||||
config.values.activeInstructionId = 'does-not-exist'
|
||||
|
||||
await runSession()
|
||||
const [text, action, , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(action).toBe('custom')
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
// ── 두 번째 진입점: 음성 단축키 ────────────────────────
|
||||
describe('음성 단축키 진입점 (overrideInstructionId)', () => {
|
||||
it('음성 명령으로 지목된 지시문도 시스템 프롬프트로 전달한다', async () => {
|
||||
registerInstruction({
|
||||
id: 'builtin-translate',
|
||||
name: '번역',
|
||||
prompt: '다음 텍스트를 {{targetLanguage}}로 번역해주세요.',
|
||||
})
|
||||
voiceCommand.enabled = true
|
||||
voiceCommand.instructionId = 'builtin-translate'
|
||||
voiceCommand.cleanedText = '회의 끝나고 배포하자'
|
||||
config.values.defaultLLMAction = 'refine'
|
||||
|
||||
await runSession()
|
||||
const [text, action, , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(action).toBe('custom')
|
||||
expect(systemPrompt).toBe('다음 텍스트를 English로 번역해주세요.')
|
||||
})
|
||||
|
||||
it("defaultLLMAction이 'none'이어도 음성 명령은 스킵되지 않는다", async () => {
|
||||
registerInstruction({
|
||||
id: 'builtin-summarize',
|
||||
name: '요약',
|
||||
prompt: '다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.',
|
||||
})
|
||||
voiceCommand.enabled = true
|
||||
voiceCommand.instructionId = 'builtin-summarize'
|
||||
config.values.defaultLLMAction = 'none'
|
||||
|
||||
await runSession()
|
||||
|
||||
expect(mockLLM.processText).toHaveBeenCalledTimes(1)
|
||||
const [text, action, , systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(action).toBe('custom')
|
||||
expect(systemPrompt).toBe('다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.')
|
||||
})
|
||||
|
||||
it("음성 명령이 없으면 'none'은 기존대로 LLM을 건너뛴다", async () => {
|
||||
config.values.defaultLLMAction = 'none'
|
||||
|
||||
await runSession()
|
||||
|
||||
expect(mockLLM.processText).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
// ── 안 고친 경로: 일반 액션이 원래 맞았고 계속 맞다 ────
|
||||
describe('일반 액션 경로 (수정 대상 아님 — 계속 정상이어야 한다)', () => {
|
||||
it.each(['refine', 'summarize', 'grammar', 'expand'])(
|
||||
"'%s' 액션은 전사 텍스트를 text로 넘기고 customPrompt를 넘기지 않는다",
|
||||
async (action) => {
|
||||
config.values.defaultLLMAction = action
|
||||
// 활성 지시문이 있어도 일반 액션 경로는 지시문을 타지 않아야 한다.
|
||||
registerInstruction({ id: 'builtin-summarize', name: '요약', prompt: '요약해줘' })
|
||||
config.values.activeInstructionId = 'builtin-summarize'
|
||||
|
||||
await runSession()
|
||||
|
||||
expect(mockLLM.processText).toHaveBeenCalledTimes(1)
|
||||
const [text, passedAction, targetLanguage, systemPrompt] =
|
||||
mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(passedAction).toBe(action)
|
||||
expect(targetLanguage).toBeUndefined()
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
},
|
||||
)
|
||||
|
||||
it("'translate' 액션은 대상 언어를 함께 넘긴다", async () => {
|
||||
config.values.defaultLLMAction = 'translate'
|
||||
|
||||
await runSession()
|
||||
const [text, action, targetLanguage, systemPrompt] = mockLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe(TRANSCRIPT)
|
||||
expect(action).toBe('translate')
|
||||
expect(targetLanguage).toBe('English')
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getState', () => {
|
||||
it('현재 상태를 VoiceState 형태로 반환한다', () => {
|
||||
const svc = getVoiceModeService()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue