fix(meeting): keep learned text from injecting prompt lines
Learned text becomes part of a prompt, so a dictionary word or phrase containing a newline could add its own instruction line. Every value that reaches the recognition hint or the polish block is now flattened to one line and length-capped, and a test proves an injected line stays plain text. Also cover the invariants that keep the correction safe (random inputs, Hangul untouched, idempotent, no token growth, large transcript) and the file-transcription wiring that attaches the learned hint to every chunk and corrects each chunk and segment before it is stored.
This commit is contained in:
parent
03e7b7f507
commit
bcd0cf4f23
3 changed files with 260 additions and 3 deletions
|
|
@ -102,7 +102,7 @@ export function splitGlossaryVariants(pronunciation: string | null | undefined,
|
|||
const canonicalLower = canonical.toLowerCase()
|
||||
const out: string[] = []
|
||||
for (const part of raw.split(/[,/;]+/u)) {
|
||||
const value = part.trim()
|
||||
const value = sanitizeGlossaryLine(part)
|
||||
if (!value) continue
|
||||
if (value.toLowerCase() === canonicalLower) continue
|
||||
if (out.includes(value)) continue
|
||||
|
|
@ -111,6 +111,24 @@ export function splitGlossaryVariants(pronunciation: string | null | undefined,
|
|||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* 프롬프트·힌트에 넣을 문자열 한 줄로 정리한다.
|
||||
*
|
||||
* 학습 데이터는 사용자가 입력한 텍스트에서 오므로 줄바꿈·제어문자로 지시문 구조를 흔들 수 있다.
|
||||
* (예: 사전 단어에 줄바꿈을 넣어 가짜 지시 줄을 추가) 프롬프트에 넣는 모든 값은 이 함수를 거친다.
|
||||
*/
|
||||
export function sanitizeGlossaryLine(value: string, maxChars = 120): string {
|
||||
const withoutControls = Array.from(value)
|
||||
.map((char) => {
|
||||
const code = char.codePointAt(0) ?? 0
|
||||
return code < 0x20 || code === 0x7f ? ' ' : char
|
||||
})
|
||||
.join('')
|
||||
const compact = withoutControls.replace(/\s+/gu, ' ').trim()
|
||||
if (compact.length <= maxChars) return compact
|
||||
return compact.slice(0, Math.max(0, maxChars - 1)).trimEnd() + '…'
|
||||
}
|
||||
|
||||
function bump(map: Map<string, number>, key: string, amount: number): void {
|
||||
map.set(key, (map.get(key) ?? 0) + amount)
|
||||
}
|
||||
|
|
@ -143,7 +161,7 @@ export function buildTranscriptGlossary(input: {
|
|||
const seen = new Set<string>()
|
||||
|
||||
for (const entry of (input.dictionary ?? []).slice(0, limits.dictionaryEntries)) {
|
||||
const word = entry.word?.trim()
|
||||
const word = sanitizeGlossaryLine(entry.word ?? '')
|
||||
if (!word) continue
|
||||
const key = word.toLowerCase()
|
||||
if (seen.has(key)) continue
|
||||
|
|
@ -161,7 +179,7 @@ export function buildTranscriptGlossary(input: {
|
|||
/** 같은 뜻의 한글 표기가 코퍼스에 있는 영문 토큰(한영 자판 오타 산물) — 힌트·교정·예시에서 뺀다. */
|
||||
const hangulObserved = new Set<string>()
|
||||
for (const phrase of (input.phrases ?? []).slice(0, limits.phrases)) {
|
||||
const text = phrase.text?.replace(/\s+/gu, ' ').trim()
|
||||
const text = sanitizeGlossaryLine(phrase.text ?? '', 200)
|
||||
if (!text) continue
|
||||
const tokens: string[] = []
|
||||
const weight = Math.max(1, phrase.count)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue