Running a custom instruction (translate, summarise, rewrite, explain code,
free prompt) inserted the instruction's own wording instead of the result.
Two faults stacked:
The instruction was passed as the text to process, leaving the system-prompt
argument empty. `BASE_SYSTEM_PROMPTS` has no `custom` key, so resolution fell
back to `refine` without saying so, and the model dutifully polished the
instruction it had been handed. The transcript never reached it.
And only `{{text}}` was substituted, which none of the five built-in
instructions use — they carry `{{targetLanguage}}`, `{{userPrompt}}`, or no
placeholder at all. The substitution was a no-op from the day it was written:
the presets landed ten hours before the code that expected them.
- Instruction prompts now go to the system-prompt argument and the transcript
to the text argument. Instructions that spell out `{{text}}` keep their old
meaning, so hand-written ones still work.
- `renderInstructionPrompt` resolves `{{text}}`, `{{userPrompt}}` and
`{{targetLanguage}}` in one place, and warns by name when a placeholder is
left standing rather than letting it reach the model.
- `resolveSystemPrompt` no longer drops silently to `refine` for `custom`.
- Voice shortcuts no longer die at the `defaultLLMAction === 'none'` gate; an
explicitly named instruction outranks the default. Without one, `none` still
passes the transcript through untouched.
- `translate` receives its target language instead of relying on a default two
call frames away. It is still always English — `AppConfig` has no key for it,
and neither `language` (UI locale) nor `sttLanguage` (source language) can
stand in. Choosing a target language needs a setting and is not in this fix.
- Chains ran instructions with placeholders intact; they share the same
resolution now.
- The command screen's pipeline bench called `llm.generate`, which preload does
not expose, so every run threw and the catch showed the input back as if it
had succeeded. It uses `llm.process` now, over the same path production
takes, and a failure reads as a failure.
Present since the feature shipped: the custom-instruction path has never
worked. Plain actions (refine, summarise, grammar, expand) were unaffected and
are now covered by tests so they stay that way.
163 lines
5.5 KiB
TypeScript
163 lines
5.5 KiB
TypeScript
// tests/main/ipc/llm-handlers.test.ts
|
|
// LLM.PROCESS 핸들러가 지시문 프롬프트를 시스템 프롬프트 자리로 정규화하는지 잠근다.
|
|
// 명령 화면 테스트 벤치가 이 경로를 타므로, 프로덕션(VoiceModeService)과
|
|
// 동일한 인자 배치·치환이 적용되어야 한다.
|
|
|
|
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
|
import { IPC_CHANNELS } from '@d3ro/core/ipc-channels'
|
|
import type { LLMProcessParams } from '@d3ro/core/types'
|
|
|
|
vi.mock('../../../src/main/services/LoggerService', () => ({
|
|
getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() })
|
|
}))
|
|
|
|
const handlers = vi.hoisted(
|
|
() => new Map<string, (event: unknown, params: unknown) => Promise<unknown>>()
|
|
)
|
|
|
|
vi.mock('electron', async (importOriginal) => {
|
|
const actual = await importOriginal<Record<string, unknown>>()
|
|
return {
|
|
...actual,
|
|
ipcMain: {
|
|
handle: vi.fn((channel: string, fn: (event: unknown, params: unknown) => Promise<unknown>) => {
|
|
handlers.set(channel, fn)
|
|
}),
|
|
on: vi.fn(),
|
|
removeHandler: vi.fn()
|
|
}
|
|
}
|
|
})
|
|
|
|
vi.mock('../../../src/main/services/ConfigService', () => ({
|
|
configGet: vi.fn(() => 'local'),
|
|
configSet: vi.fn()
|
|
}))
|
|
|
|
const mockLocalLLM = vi.hoisted(() => ({
|
|
processText: vi.fn(),
|
|
cancelGeneration: vi.fn(),
|
|
on: vi.fn(),
|
|
off: vi.fn()
|
|
}))
|
|
|
|
vi.mock('../../../src/main/services/LocalLLMService', () => ({
|
|
getLocalLLMService: () => mockLocalLLM
|
|
}))
|
|
|
|
vi.mock('../../../src/main/services/PremiumLLMService', () => ({
|
|
getPremiumLLMService: () => ({ cancelGeneration: vi.fn(), on: vi.fn(), off: vi.fn() })
|
|
}))
|
|
|
|
vi.mock('../../../src/main/services/OnlineLLMService', () => ({
|
|
getOnlineLLMService: () => mockLocalLLM
|
|
}))
|
|
|
|
async function invokeProcess(params: LLMProcessParams): Promise<unknown> {
|
|
const handler = handlers.get(IPC_CHANNELS.LLM.PROCESS)
|
|
if (!handler) throw new Error('LLM.PROCESS handler not registered')
|
|
return handler({}, params)
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules()
|
|
vi.clearAllMocks()
|
|
handlers.clear()
|
|
mockLocalLLM.processText.mockResolvedValue('LLM 결과')
|
|
|
|
const mod = await import('../../../src/main/ipc/llm-handlers')
|
|
mod.registerLLMHandlers()
|
|
})
|
|
|
|
describe('LLM.PROCESS — 지시문 인자 정규화', () => {
|
|
it('custom 액션의 지시문을 시스템 프롬프트로, 입력 텍스트를 처리 대상으로 넘긴다', async () => {
|
|
await invokeProcess({
|
|
text: '오늘 배포 일정을 정했습니다',
|
|
action: 'custom',
|
|
customPrompt: '다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.'
|
|
})
|
|
|
|
const [text, action, , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('오늘 배포 일정을 정했습니다')
|
|
expect(action).toBe('custom')
|
|
expect(systemPrompt).toBe('다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.')
|
|
})
|
|
|
|
it('{{targetLanguage}}를 치환해 시스템 프롬프트로 넘긴다', async () => {
|
|
await invokeProcess({
|
|
text: '안녕하세요',
|
|
action: 'custom',
|
|
customPrompt: '다음 텍스트를 {{targetLanguage}}로 번역해주세요.'
|
|
})
|
|
|
|
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('안녕하세요')
|
|
expect(systemPrompt).toBe('다음 텍스트를 English로 번역해주세요.')
|
|
expect(systemPrompt).not.toContain('{{')
|
|
})
|
|
|
|
it('{{userPrompt}}를 입력 텍스트로 치환한다', async () => {
|
|
await invokeProcess({
|
|
text: '피보나치 짜줘',
|
|
action: 'custom',
|
|
customPrompt: '{{userPrompt}}'
|
|
})
|
|
|
|
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('피보나치 짜줘')
|
|
expect(systemPrompt).toBe('피보나치 짜줘')
|
|
})
|
|
|
|
it('{{text}}를 쓰는 지시문은 치환 결과를 처리 대상 텍스트로 넘긴다', async () => {
|
|
await invokeProcess({
|
|
text: '가 나 다',
|
|
action: 'custom',
|
|
customPrompt: '아래를 불릿으로 정리해줘:\n{{text}}'
|
|
})
|
|
|
|
const [text, , , systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('아래를 불릿으로 정리해줘:\n가 나 다')
|
|
expect(systemPrompt).toBeUndefined()
|
|
})
|
|
|
|
it('명시적 targetLanguage를 그대로 쓴다', async () => {
|
|
await invokeProcess({
|
|
text: '안녕',
|
|
action: 'custom',
|
|
customPrompt: '{{targetLanguage}}로 번역해줘.',
|
|
targetLanguage: '프랑스어'
|
|
})
|
|
|
|
const [, , targetLanguage, systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(targetLanguage).toBe('프랑스어')
|
|
expect(systemPrompt).toBe('프랑스어로 번역해줘.')
|
|
})
|
|
|
|
it('일반 액션은 지시문 정규화를 거치지 않고 그대로 전달한다', async () => {
|
|
await invokeProcess({ text: '다듬어줘 이 문장', action: 'refine' })
|
|
|
|
const [text, action, targetLanguage, systemPrompt] = mockLocalLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('다듬어줘 이 문장')
|
|
expect(action).toBe('refine')
|
|
expect(targetLanguage).toBeUndefined()
|
|
expect(systemPrompt).toBeUndefined()
|
|
})
|
|
|
|
it('실패를 성공으로 위장하지 않는다', async () => {
|
|
mockLocalLLM.processText.mockRejectedValue(new Error('Ollama unreachable'))
|
|
|
|
const result = (await invokeProcess({ text: '아무 말', action: 'refine' })) as {
|
|
success: boolean
|
|
error: { message: string }
|
|
}
|
|
|
|
expect(result.success).toBe(false)
|
|
expect(result.error.message).toContain('Ollama unreachable')
|
|
})
|
|
})
|