feat: ship Wave 5-6 multi-surface hygiene, optimization, and bug fixes
This commit is contained in:
parent
01ef3293cf
commit
66b6b45ba9
399 changed files with 17828 additions and 11330 deletions
|
|
@ -6,27 +6,30 @@
|
|||
// HTML 이스케이프가 없어 `List<String>`·`<TBD>` 가 PDF에서 사라지고 줄바꿈·표가 깨졌다.
|
||||
// 이제 parseMarkdownBlocks 한 곳에서 블록으로 나누고, HTML/DOCX 는 블록을 렌더링만 한다.
|
||||
|
||||
import { formatDurationClockFromMs } from '../format'
|
||||
import type { MeetingSessionDetail } from '../types'
|
||||
|
||||
export function formatTime(ms: number): string {
|
||||
const totalSeconds = Math.floor(ms / 1000)
|
||||
const minutes = Math.floor(totalSeconds / 60)
|
||||
const seconds = totalSeconds % 60
|
||||
return `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`
|
||||
return formatDurationClockFromMs(ms, { padMinutes: true })
|
||||
}
|
||||
|
||||
export function formatDateFile(epochMs: number): string {
|
||||
if (!Number.isFinite(epochMs)) return ''
|
||||
const d = new Date(epochMs)
|
||||
if (!Number.isFinite(d.getTime())) return ''
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}_${String(d.getHours()).padStart(2, '0')}${String(d.getMinutes()).padStart(2, '0')}`
|
||||
}
|
||||
|
||||
export function formatDateRange(startedAt: number, endedAt: number | null): string {
|
||||
const fmt = (ts: number): string => {
|
||||
const d = new Date(ts)
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')} ${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`
|
||||
}
|
||||
if (!endedAt) return fmt(startedAt)
|
||||
return `${fmt(startedAt)} ~ ${new Date(endedAt).getHours()}:${String(new Date(endedAt).getMinutes()).padStart(2, '0')}`
|
||||
if (!Number.isFinite(startedAt)) return ''
|
||||
const startDate = new Date(startedAt)
|
||||
if (!Number.isFinite(startDate.getTime())) return ''
|
||||
const fmt = (d: Date): string =>
|
||||
`${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')} ${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`
|
||||
if (endedAt === null || !Number.isFinite(endedAt)) return fmt(startDate)
|
||||
const endDate = new Date(endedAt)
|
||||
if (!Number.isFinite(endDate.getTime())) return fmt(startDate)
|
||||
return `${fmt(startDate)} ~ ${endDate.getHours()}:${String(endDate.getMinutes()).padStart(2, '0')}`
|
||||
}
|
||||
|
||||
export function buildExportMarkdown(session: MeetingSessionDetail): string {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,63 @@ export const DEFAULT_REDACTION_RULES: RedactionRule[] = [
|
|||
},
|
||||
]
|
||||
|
||||
function escapeRegExp(text: string): string {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
||||
interface RedactStringContext {
|
||||
existingTexts: readonly string[]
|
||||
mode: 'mask' | 'tokenize'
|
||||
activeRules: readonly RedactionRule[]
|
||||
tokens: Map<string, string>
|
||||
nextCounter: () => number
|
||||
categories?: Set<string>
|
||||
}
|
||||
|
||||
function allocateUniqueToken(
|
||||
ruleId: string,
|
||||
ctx: RedactStringContext,
|
||||
): string {
|
||||
const prefix = `[REDACTED_${ruleId.toUpperCase()}_`
|
||||
let candidate = `${prefix}${ctx.nextCounter()}]`
|
||||
while (ctx.tokens.has(candidate) || ctx.existingTexts.some((chunk) => chunk.includes(candidate))) {
|
||||
candidate = `${prefix}${ctx.nextCounter()}]`
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
|
||||
function applyRedactionRules(
|
||||
text: string,
|
||||
ctx: RedactStringContext,
|
||||
): { redacted: string; count: number } {
|
||||
let redacted = text
|
||||
let count = 0
|
||||
|
||||
for (const rule of ctx.activeRules) {
|
||||
rule.pattern.lastIndex = 0
|
||||
redacted = redacted.replace(rule.pattern, (match, ...groups) => {
|
||||
count += 1
|
||||
ctx.categories?.add(rule.name)
|
||||
|
||||
if (ctx.mode === 'tokenize') {
|
||||
const placeholder = allocateUniqueToken(rule.id, ctx)
|
||||
ctx.tokens.set(placeholder, match)
|
||||
return placeholder
|
||||
}
|
||||
|
||||
return rule.mask(match, ...groups)
|
||||
})
|
||||
}
|
||||
|
||||
return { redacted, count }
|
||||
}
|
||||
|
||||
function resolveActiveRules(enabledRules?: string[]): RedactionRule[] {
|
||||
return enabledRules
|
||||
? DEFAULT_REDACTION_RULES.filter((r) => enabledRules.includes(r.id))
|
||||
: DEFAULT_REDACTION_RULES
|
||||
}
|
||||
|
||||
/**
|
||||
* 텍스트 내의 민감 개인정보(PII)를 검출하여 마스킹 또는 토큰화
|
||||
* @param text 원본 전사 텍스트
|
||||
|
|
@ -93,36 +150,26 @@ export function redactPII(
|
|||
}
|
||||
}
|
||||
|
||||
let redacted = text
|
||||
let totalCount = 0
|
||||
const categories = new Set<string>()
|
||||
const tokens = new Map<string, string>()
|
||||
let tokenCounter = 0
|
||||
|
||||
const activeRules = enabledRules
|
||||
? DEFAULT_REDACTION_RULES.filter((r) => enabledRules.includes(r.id))
|
||||
: DEFAULT_REDACTION_RULES
|
||||
|
||||
for (const rule of activeRules) {
|
||||
redacted = redacted.replace(rule.pattern, (match, ...groups) => {
|
||||
totalCount++
|
||||
categories.add(rule.name)
|
||||
|
||||
if (mode === 'tokenize') {
|
||||
tokenCounter++
|
||||
const placeholder = `[REDACTED_${rule.id.toUpperCase()}_${tokenCounter}]`
|
||||
tokens.set(placeholder, match)
|
||||
return placeholder
|
||||
}
|
||||
|
||||
return rule.mask(match, ...groups)
|
||||
})
|
||||
}
|
||||
const { redacted, count } = applyRedactionRules(text, {
|
||||
existingTexts: [text],
|
||||
mode,
|
||||
activeRules: resolveActiveRules(enabledRules),
|
||||
tokens,
|
||||
nextCounter: () => {
|
||||
tokenCounter += 1
|
||||
return tokenCounter
|
||||
},
|
||||
categories,
|
||||
})
|
||||
|
||||
return {
|
||||
originalText: text,
|
||||
redactedText: redacted,
|
||||
redactionCount: totalCount,
|
||||
redactionCount: count,
|
||||
matchedCategories: Array.from(categories),
|
||||
tokens,
|
||||
}
|
||||
|
|
@ -130,7 +177,7 @@ export function redactPII(
|
|||
|
||||
/**
|
||||
* 토큰화된 텍스트를 원본 값으로 복원 (Rehydration)
|
||||
* LLM 추론 완료 후 클라이언트 화면 표시 시 안전하게 원문 복원
|
||||
* 단일 패스로 치환하므로 복원 값 안에 다른 플레이스홀더 문자열이 섞여 있어도 이중 복원되지 않는다.
|
||||
*/
|
||||
export function rehydrateText(
|
||||
redactedText: string,
|
||||
|
|
@ -138,9 +185,59 @@ export function rehydrateText(
|
|||
): string {
|
||||
if (!redactedText || tokens.size === 0) return redactedText
|
||||
|
||||
let result = redactedText
|
||||
tokens.forEach((originalValue, placeholder) => {
|
||||
result = result.split(placeholder).join(originalValue)
|
||||
})
|
||||
return result
|
||||
const keys = Array.from(tokens.keys())
|
||||
.filter((key) => key.length > 0)
|
||||
.sort((a, b) => b.length - a.length)
|
||||
if (keys.length === 0) return redactedText
|
||||
|
||||
const matcher = new RegExp(keys.map(escapeRegExp).join('|'), 'g')
|
||||
return redactedText.replace(matcher, (match) => tokens.get(match) ?? match)
|
||||
}
|
||||
|
||||
export interface RedactedMessagesResult<T extends { role: string; content: string }> {
|
||||
messages: T[]
|
||||
tokens: Map<string, string>
|
||||
redactionCount: number
|
||||
}
|
||||
|
||||
/**
|
||||
* 다중 채팅 메시지 배열 전체에 대해 단일 플레이스홀더 카운터로 PII 토큰화를 수행한다.
|
||||
* 여러 턴에 전화번호·이메일이 중복 등장해도 토큰 번호가 충돌하지 않는다.
|
||||
*/
|
||||
export function redactPIIMessages<T extends { role: string; content: string }>(
|
||||
messages: readonly T[],
|
||||
enabledRules?: string[],
|
||||
): RedactedMessagesResult<T> {
|
||||
const tokens = new Map<string, string>()
|
||||
let tokenCounter = 0
|
||||
let redactionCount = 0
|
||||
|
||||
const activeRules = resolveActiveRules(enabledRules)
|
||||
const existingTexts = messages
|
||||
.map((m) => (typeof m.content === 'string' ? m.content : ''))
|
||||
.filter(Boolean)
|
||||
|
||||
const redactedMessages = messages.map((message) => {
|
||||
if (!message.content || typeof message.content !== 'string') {
|
||||
return { ...message }
|
||||
}
|
||||
const { redacted, count } = applyRedactionRules(message.content, {
|
||||
existingTexts,
|
||||
mode: 'tokenize',
|
||||
activeRules,
|
||||
tokens,
|
||||
nextCounter: () => {
|
||||
tokenCounter += 1
|
||||
return tokenCounter
|
||||
},
|
||||
})
|
||||
redactionCount += count
|
||||
return { ...message, content: redacted }
|
||||
})
|
||||
|
||||
return {
|
||||
messages: redactedMessages,
|
||||
tokens,
|
||||
redactionCount,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue