fix(llm): stop leaking instruction prompts instead of processed text
Running a custom instruction (translate, summarise, rewrite, explain code,
free prompt) inserted the instruction's own wording instead of the result.
Two faults stacked:
The instruction was passed as the text to process, leaving the system-prompt
argument empty. `BASE_SYSTEM_PROMPTS` has no `custom` key, so resolution fell
back to `refine` without saying so, and the model dutifully polished the
instruction it had been handed. The transcript never reached it.
And only `{{text}}` was substituted, which none of the five built-in
instructions use — they carry `{{targetLanguage}}`, `{{userPrompt}}`, or no
placeholder at all. The substitution was a no-op from the day it was written:
the presets landed ten hours before the code that expected them.
- Instruction prompts now go to the system-prompt argument and the transcript
to the text argument. Instructions that spell out `{{text}}` keep their old
meaning, so hand-written ones still work.
- `renderInstructionPrompt` resolves `{{text}}`, `{{userPrompt}}` and
`{{targetLanguage}}` in one place, and warns by name when a placeholder is
left standing rather than letting it reach the model.
- `resolveSystemPrompt` no longer drops silently to `refine` for `custom`.
- Voice shortcuts no longer die at the `defaultLLMAction === 'none'` gate; an
explicitly named instruction outranks the default. Without one, `none` still
passes the transcript through untouched.
- `translate` receives its target language instead of relying on a default two
call frames away. It is still always English — `AppConfig` has no key for it,
and neither `language` (UI locale) nor `sttLanguage` (source language) can
stand in. Choosing a target language needs a setting and is not in this fix.
- Chains ran instructions with placeholders intact; they share the same
resolution now.
- The command screen's pipeline bench called `llm.generate`, which preload does
not expose, so every run threw and the catch showed the input back as if it
had succeeded. It uses `llm.process` now, over the same path production
takes, and a failure reads as a failure.
Present since the feature shipped: the custom-instruction path has never
worked. Plain actions (refine, summarise, grammar, expand) were unaffected and
are now covered by tests so they stay that way.
This commit is contained in:
parent
30d51c952f
commit
99f06c253c
14 changed files with 992 additions and 44 deletions
163
apps/desktop/tests/main/ipc/llm-handlers.test.ts
Normal file
163
apps/desktop/tests/main/ipc/llm-handlers.test.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
// tests/main/ipc/llm-handlers.test.ts
|
||||
// LLM.PROCESS 핸들러가 지시문 프롬프트를 시스템 프롬프트 자리로 정규화하는지 잠근다.
|
||||
// 명령 화면 테스트 벤치가 이 경로를 타므로, 프로덕션(VoiceModeService)과
|
||||
// 동일한 인자 배치·치환이 적용되어야 한다.
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { IPC_CHANNELS } from '@d3ro/core/ipc-channels'
|
||||
import type { LLMProcessParams } from '@d3ro/core/types'
|
||||
|
||||
vi.mock('../../../src/main/services/LoggerService', () => ({
|
||||
getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() })
|
||||
}))
|
||||
|
||||
const handlers = vi.hoisted(
|
||||
() => new Map<string, (event: unknown, params: unknown) => Promise<unknown>>()
|
||||
)
|
||||
|
||||
vi.mock('electron', async (importOriginal) => {
|
||||
const actual = await importOriginal<Record<string, unknown>>()
|
||||
return {
|
||||
...actual,
|
||||
ipcMain: {
|
||||
handle: vi.fn((channel: string, fn: (event: unknown, params: unknown) => Promise<unknown>) => {
|
||||
handlers.set(channel, fn)
|
||||
}),
|
||||
on: vi.fn(),
|
||||
removeHandler: vi.fn()
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('../../../src/main/services/ConfigService', () => ({
|
||||
configGet: vi.fn(() => 'local'),
|
||||
configSet: vi.fn()
|
||||
}))
|
||||
|
||||
const mockLocalLLM = vi.hoisted(() => ({
|
||||
processText: vi.fn(),
|
||||
cancelGeneration: vi.fn(),
|
||||
on: vi.fn(),
|
||||
off: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('../../../src/main/services/LocalLLMService', () => ({
|
||||
getLocalLLMService: () => mockLocalLLM
|
||||
}))
|
||||
|
||||
vi.mock('../../../src/main/services/PremiumLLMService', () => ({
|
||||
getPremiumLLMService: () => ({ cancelGeneration: vi.fn(), on: vi.fn(), off: vi.fn() })
|
||||
}))
|
||||
|
||||
vi.mock('../../../src/main/services/OnlineLLMService', () => ({
|
||||
getOnlineLLMService: () => mockLocalLLM
|
||||
}))
|
||||
|
||||
async function invokeProcess(params: LLMProcessParams): Promise<unknown> {
|
||||
const handler = handlers.get(IPC_CHANNELS.LLM.PROCESS)
|
||||
if (!handler) throw new Error('LLM.PROCESS handler not registered')
|
||||
return handler({}, params)
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules()
|
||||
vi.clearAllMocks()
|
||||
handlers.clear()
|
||||
mockLocalLLM.processText.mockResolvedValue('LLM 결과')
|
||||
|
||||
const mod = await import('../../../src/main/ipc/llm-handlers')
|
||||
mod.registerLLMHandlers()
|
||||
})
|
||||
|
||||
describe('LLM.PROCESS — 지시문 인자 정규화', () => {
|
||||
it('custom 액션의 지시문을 시스템 프롬프트로, 입력 텍스트를 처리 대상으로 넘긴다', async () => {
|
||||
await invokeProcess({
|
||||
text: '오늘 배포 일정을 정했습니다',
|
||||
action: 'custom',
|
||||
customPrompt: '다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.'
|
||||
})
|
||||
|
||||
const [text, action, , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe('오늘 배포 일정을 정했습니다')
|
||||
expect(action).toBe('custom')
|
||||
expect(systemPrompt).toBe('다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.')
|
||||
})
|
||||
|
||||
it('{{targetLanguage}}를 치환해 시스템 프롬프트로 넘긴다', async () => {
|
||||
await invokeProcess({
|
||||
text: '안녕하세요',
|
||||
action: 'custom',
|
||||
customPrompt: '다음 텍스트를 {{targetLanguage}}로 번역해주세요.'
|
||||
})
|
||||
|
||||
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe('안녕하세요')
|
||||
expect(systemPrompt).toBe('다음 텍스트를 English로 번역해주세요.')
|
||||
expect(systemPrompt).not.toContain('{{')
|
||||
})
|
||||
|
||||
it('{{userPrompt}}를 입력 텍스트로 치환한다', async () => {
|
||||
await invokeProcess({
|
||||
text: '피보나치 짜줘',
|
||||
action: 'custom',
|
||||
customPrompt: '{{userPrompt}}'
|
||||
})
|
||||
|
||||
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe('피보나치 짜줘')
|
||||
expect(systemPrompt).toBe('피보나치 짜줘')
|
||||
})
|
||||
|
||||
it('{{text}}를 쓰는 지시문은 치환 결과를 처리 대상 텍스트로 넘긴다', async () => {
|
||||
await invokeProcess({
|
||||
text: '가 나 다',
|
||||
action: 'custom',
|
||||
customPrompt: '아래를 불릿으로 정리해줘:\n{{text}}'
|
||||
})
|
||||
|
||||
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe('아래를 불릿으로 정리해줘:\n가 나 다')
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
})
|
||||
|
||||
it('명시적 targetLanguage를 그대로 쓴다', async () => {
|
||||
await invokeProcess({
|
||||
text: '안녕',
|
||||
action: 'custom',
|
||||
customPrompt: '{{targetLanguage}}로 번역해줘.',
|
||||
targetLanguage: '프랑스어'
|
||||
})
|
||||
|
||||
const [, , targetLanguage, systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(targetLanguage).toBe('프랑스어')
|
||||
expect(systemPrompt).toBe('프랑스어로 번역해줘.')
|
||||
})
|
||||
|
||||
it('일반 액션은 지시문 정규화를 거치지 않고 그대로 전달한다', async () => {
|
||||
await invokeProcess({ text: '다듬어줘 이 문장', action: 'refine' })
|
||||
|
||||
const [text, action, targetLanguage, systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
||||
|
||||
expect(text).toBe('다듬어줘 이 문장')
|
||||
expect(action).toBe('refine')
|
||||
expect(targetLanguage).toBeUndefined()
|
||||
expect(systemPrompt).toBeUndefined()
|
||||
})
|
||||
|
||||
it('실패를 성공으로 위장하지 않는다', async () => {
|
||||
mockLocalLLM.processText.mockRejectedValue(new Error('Ollama unreachable'))
|
||||
|
||||
const result = (await invokeProcess({ text: '아무 말', action: 'refine' })) as {
|
||||
success: boolean
|
||||
error: { message: string }
|
||||
}
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error.message).toContain('Ollama unreachable')
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue