Running a custom instruction (translate, summarise, rewrite, explain code,
free prompt) inserted the instruction's own wording instead of the result.
Two faults stacked:
The instruction was passed as the text to process, leaving the system-prompt
argument empty. `BASE_SYSTEM_PROMPTS` has no `custom` key, so resolution fell
back to `refine` without saying so, and the model dutifully polished the
instruction it had been handed. The transcript never reached it.
And only `{{text}}` was substituted, which none of the five built-in
instructions use — they carry `{{targetLanguage}}`, `{{userPrompt}}`, or no
placeholder at all. The substitution was a no-op from the day it was written:
the presets landed ten hours before the code that expected them.
- Instruction prompts now go to the system-prompt argument and the transcript
to the text argument. Instructions that spell out `{{text}}` keep their old
meaning, so hand-written ones still work.
- `renderInstructionPrompt` resolves `{{text}}`, `{{userPrompt}}` and
`{{targetLanguage}}` in one place, and warns by name when a placeholder is
left standing rather than letting it reach the model.
- `resolveSystemPrompt` no longer drops silently to `refine` for `custom`.
- Voice shortcuts no longer die at the `defaultLLMAction === 'none'` gate; an
explicitly named instruction outranks the default. Without one, `none` still
passes the transcript through untouched.
- `translate` receives its target language instead of relying on a default two
call frames away. It is still always English — `AppConfig` has no key for it,
and neither `language` (UI locale) nor `sttLanguage` (source language) can
stand in. Choosing a target language needs a setting and is not in this fix.
- Chains ran instructions with placeholders intact; they share the same
resolution now.
- The command screen's pipeline bench called `llm.generate`, which preload does
not expose, so every run threw and the catch showed the input back as if it
had succeeded. It uses `llm.process` now, over the same path production
takes, and a failure reads as a failure.
Present since the feature shipped: the custom-instruction path has never
worked. Plain actions (refine, summarise, grammar, expand) were unaffected and
are now covered by tests so they stay that way.
174 lines
6.1 KiB
TypeScript
174 lines
6.1 KiB
TypeScript
// tests/main/services/ChainService.test.ts
|
|
// 체인 스텝이 지시문을 시스템 프롬프트로 넘기고 플레이스홀더를 치환하는지 잠근다.
|
|
// 인자 자리는 원래 옳았으나 치환이 없어 {{targetLanguage}} 등이 그대로 새어나갔다.
|
|
|
|
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
|
import type { LLMChain } from '@d3ro/core/types'
|
|
|
|
vi.mock('../../../src/main/services/LoggerService', () => ({
|
|
getLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() })
|
|
}))
|
|
|
|
const config = vi.hoisted(() => ({ values: {} as Record<string, unknown> }))
|
|
|
|
vi.mock('../../../src/main/services/ConfigService', () => ({
|
|
configGet: vi.fn((key: string) => config.values[key]),
|
|
configSet: vi.fn((key: string, value: unknown) => {
|
|
config.values[key] = value
|
|
})
|
|
}))
|
|
|
|
const instructionStore = vi.hoisted(() => ({
|
|
byId: {} as Record<string, { id: string; name: string; prompt: string }>
|
|
}))
|
|
|
|
vi.mock('../../../src/main/services/CustomInstructionService', () => ({
|
|
getCustomInstructionService: () => ({
|
|
getById: (id: string) => instructionStore.byId[id] ?? null
|
|
})
|
|
}))
|
|
|
|
const mockLLM = vi.hoisted(() => ({
|
|
processText: vi.fn(),
|
|
cancelGeneration: vi.fn()
|
|
}))
|
|
|
|
vi.mock('../../../src/main/services/PremiumLLMService', () => ({
|
|
getPremiumLLMService: () => mockLLM
|
|
}))
|
|
|
|
let getChainService: typeof import('../../../src/main/services/ChainService')['getChainService']
|
|
|
|
function chain(steps: Array<{ instructionId: string }>): LLMChain {
|
|
return {
|
|
id: 'chain-1',
|
|
name: '테스트 체인',
|
|
steps: steps.map((s) => ({ instructionId: s.instructionId, inputSource: 'previous' as const })),
|
|
createdAt: 0,
|
|
updatedAt: 0
|
|
}
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules()
|
|
vi.clearAllMocks()
|
|
config.values = {}
|
|
instructionStore.byId = {}
|
|
mockLLM.processText.mockResolvedValue('LLM 결과')
|
|
|
|
const mod = await import('../../../src/main/services/ChainService')
|
|
mod.resetChainServiceForTests()
|
|
getChainService = mod.getChainService
|
|
})
|
|
|
|
describe('ChainService.execute — 지시문 인자 전달', () => {
|
|
it('지시문을 시스템 프롬프트로, 스텝 입력을 처리 대상 텍스트로 넘긴다', async () => {
|
|
instructionStore.byId['builtin-summarize'] = {
|
|
id: 'builtin-summarize',
|
|
name: '요약',
|
|
prompt: '다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.'
|
|
}
|
|
config.values.llmChains = [chain([{ instructionId: 'builtin-summarize' }])]
|
|
|
|
const svc = getChainService()
|
|
svc.initialize()
|
|
await svc.execute('chain-1', '오늘 배포 일정을 정했습니다')
|
|
|
|
expect(mockLLM.processText).toHaveBeenCalledTimes(1)
|
|
const [text, action, targetLanguage, systemPrompt] = mockLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('오늘 배포 일정을 정했습니다')
|
|
expect(action).toBe('custom')
|
|
expect(targetLanguage).toBeUndefined()
|
|
expect(systemPrompt).toBe('다음 텍스트의 핵심 내용을 3줄 이내로 요약해주세요.')
|
|
})
|
|
|
|
it('{{targetLanguage}}를 치환해 시스템 프롬프트로 넘긴다', async () => {
|
|
instructionStore.byId['builtin-translate'] = {
|
|
id: 'builtin-translate',
|
|
name: '번역',
|
|
prompt: '다음 텍스트를 {{targetLanguage}}로 번역해주세요.\n자연스럽고 정확한 번역만 출력하세요.'
|
|
}
|
|
config.values.llmChains = [chain([{ instructionId: 'builtin-translate' }])]
|
|
|
|
const svc = getChainService()
|
|
svc.initialize()
|
|
await svc.execute('chain-1', '안녕하세요')
|
|
|
|
const [text, , , systemPrompt] = mockLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('안녕하세요')
|
|
expect(systemPrompt).toBe(
|
|
'다음 텍스트를 English로 번역해주세요.\n자연스럽고 정확한 번역만 출력하세요.'
|
|
)
|
|
expect(systemPrompt).not.toContain('{{')
|
|
})
|
|
|
|
it('{{userPrompt}}를 스텝 입력으로 치환한다', async () => {
|
|
instructionStore.byId['builtin-free-prompt'] = {
|
|
id: 'builtin-free-prompt',
|
|
name: '자유 프롬프트',
|
|
prompt: '{{userPrompt}}'
|
|
}
|
|
config.values.llmChains = [chain([{ instructionId: 'builtin-free-prompt' }])]
|
|
|
|
const svc = getChainService()
|
|
svc.initialize()
|
|
await svc.execute('chain-1', '피보나치 짜줘')
|
|
|
|
const [text, , , systemPrompt] = mockLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('피보나치 짜줘')
|
|
expect(systemPrompt).toBe('피보나치 짜줘')
|
|
})
|
|
|
|
it('{{text}}를 쓰는 지시문은 치환 결과를 처리 대상 텍스트로 넘긴다', async () => {
|
|
instructionStore.byId['user-bullets'] = {
|
|
id: 'user-bullets',
|
|
name: '불릿 정리',
|
|
prompt: '아래를 불릿으로 정리해줘:\n{{text}}'
|
|
}
|
|
config.values.llmChains = [chain([{ instructionId: 'user-bullets' }])]
|
|
|
|
const svc = getChainService()
|
|
svc.initialize()
|
|
await svc.execute('chain-1', '가 나 다')
|
|
|
|
const [text, action, , systemPrompt] = mockLLM.processText.mock.calls[0]
|
|
|
|
expect(text).toBe('아래를 불릿으로 정리해줘:\n가 나 다')
|
|
expect(action).toBe('custom')
|
|
expect(systemPrompt).toBeUndefined()
|
|
})
|
|
|
|
it('두 번째 스텝은 이전 스텝 출력을 처리 대상 텍스트로 받는다', async () => {
|
|
instructionStore.byId['builtin-summarize'] = {
|
|
id: 'builtin-summarize',
|
|
name: '요약',
|
|
prompt: '요약해줘.'
|
|
}
|
|
instructionStore.byId['builtin-translate'] = {
|
|
id: 'builtin-translate',
|
|
name: '번역',
|
|
prompt: '{{targetLanguage}}로 번역해줘.'
|
|
}
|
|
config.values.llmChains = [
|
|
chain([{ instructionId: 'builtin-summarize' }, { instructionId: 'builtin-translate' }])
|
|
]
|
|
mockLLM.processText.mockResolvedValueOnce('요약된 텍스트')
|
|
|
|
const svc = getChainService()
|
|
svc.initialize()
|
|
await svc.execute('chain-1', '긴 원문')
|
|
|
|
expect(mockLLM.processText).toHaveBeenCalledTimes(2)
|
|
|
|
const [firstText, , , firstSystem] = mockLLM.processText.mock.calls[0]
|
|
expect(firstText).toBe('긴 원문')
|
|
expect(firstSystem).toBe('요약해줘.')
|
|
|
|
const [secondText, , , secondSystem] = mockLLM.processText.mock.calls[1]
|
|
expect(secondText).toBe('요약된 텍스트')
|
|
expect(secondSystem).toBe('English로 번역해줘.')
|
|
})
|
|
})
|