yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

전 저장소 리팩터링과 SSOT 정비

Browse source
This commit is contained in:
Yun Chan 2026-07-15 21:31:30 +09:00
parent 14ecbd4e7d
commit 3dfddcac6f
173 changed files with 19679 additions and 6952 deletions
Download patch file Download diff file Expand all files Collapse all files

173
apps/api/app/services/evaluator.py
View file

@ -49,15 +49,16 @@ from ..taxonomy import (
TECHNIQUE_CATEGORY,
TECHNIQUE_KO,
ClientState,
CommentKind,
Technique,
TechniqueCategory,
speaker_ko_label,
)
from . import guardrail
from .evaluation_contract import APPROPRIATENESS_VALUES, Appropriateness
from .llm_audit import LlmAuditHook, generate_with_audit
if TYPE_CHECKING: # 런타임 import 회피(순환·소유권 경계). 타입 힌트 전용.
from .orchestrator import LlmAuditHook, TurnContext
from .orchestrator import TurnContext
# ════════════════════════════════════════════════════════════════════════════
@ -66,11 +67,12 @@ if TYPE_CHECKING: # 런타임 import 회피(순환·소유권 경계). 타입
# LLM 은 후보로 한글 라벨을 받지만, 코드값(value)을 돌려줄 수도 있어 둘 다 받는다.
_TECHNIQUE_BY_KO: dict[str, Technique] = {ko: t for t, ko in TECHNIQUE_KO.items()}
_TECHNIQUE_BY_CODE: dict[str, Technique] = {t.value: t for t in Technique}
_CLIENT_STATE_BY_KO: dict[str, ClientState] = {ko: s for s, ko in CLIENT_STATE_KO.items()}
_CLIENT_STATE_BY_KO: dict[str, ClientState] = {
ko: s for s, ko in CLIENT_STATE_KO.items()
}
_CLIENT_STATE_BY_CODE: dict[str, ClientState] = {s.value: s for s in ClientState}
# 적절성 신호 — fast-loop 의 경량 판단(상태머신 라포 추정과 별개 차원).
_APPROPRIATENESS = ("pos", "warn", "neutral")
# 의도이탈 심각도 (taxonomy.SupervisorComment.severity 와 동일 어휘).
_SEVERITY = ("minor", "moderate", "major")
@ -207,17 +209,17 @@ class IntentDeviation(BaseModel):
class TechniqueTag(BaseModel):
"""fast-loop 기법 태그 1건 — taxonomy.Technique 코드 + 한글 + 군집 + 근거."""
code: str # taxonomy.Technique.value
label_ko: str # TECHNIQUE_KO
category: str # TechniqueCategory.value (분포 집계축)
code: str # taxonomy.Technique.value
label_ko: str # TECHNIQUE_KO
category: str # TechniqueCategory.value (분포 집계축)
rationale: Optional[str] = None # 왜 이 기법으로 봤는지(근거 요구)
class ClientStateRead(BaseModel):
"""내담자 상태 '읽기' — 학습자 발화 직후 내담자 응답에서 관측된 상태(읽기 채점 근거)."""
code: str # taxonomy.ClientState.value
label_ko: str # CLIENT_STATE_KO
code: str # taxonomy.ClientState.value
label_ko: str # CLIENT_STATE_KO
rationale: Optional[str] = None
@ -236,12 +238,14 @@ class TurnEvaluation(BaseModel):
stage: str
techniques: list[TechniqueTag] = Field(default_factory=list)
client_state_read: list[ClientStateRead] = Field(default_factory=list)
appropriateness: str = "neutral" # pos | warn | neutral
appropriateness: Appropriateness = "neutral"
appropriateness_note: Optional[str] = None
intent_deviation: Optional[IntentDeviation] = None # 있을 때만(1급 시민)
rapport_signal: Optional[float] = None # 평가 AI 가 본 라포 신호(−1~+1, 상태머신 주입 가능)
rapport_signal: Optional[float] = (
None # 평가 AI 가 본 라포 신호(−1~+1, 상태머신 주입 가능)
)
theory_mode: Optional[str] = None
error: Optional[str] = None # 평가 실패 시 사유(비치명적; None 이면 정상)
error: Optional[str] = None # 평가 실패 시 사유(비치명적; None 이면 정상)
def to_hook_dict(self) -> dict[str, Any]:
"""orchestrator.EvalHook 가 기대하는 평가 dict(turns.evaluation 적재용)."""
@ -251,10 +255,12 @@ class TurnEvaluation(BaseModel):
class TechniqueDistribution(BaseModel):
"""deep-loop 기법 분포 — 군집별 카운트 + 과다/과소 진단."""
by_category: dict[str, int] = Field(default_factory=dict) # category.value -> count
by_technique: dict[str, int] = Field(default_factory=dict) # technique.value -> count
by_category: dict[str, int] = Field(default_factory=dict) # category.value -> count
by_technique: dict[str, int] = Field(
default_factory=dict
) # technique.value -> count
total: int = 0
overused: list[str] = Field(default_factory=list) # 과다 사용 군집(category.value)
overused: list[str] = Field(default_factory=list) # 과다 사용 군집(category.value)
underused: list[str] = Field(default_factory=list) # 과소/미사용 군집
@ -266,15 +272,15 @@ class SessionEvaluation(BaseModel):
loop: str = "deep"
session_id: str
stage: str # 평가 시점 단계(전환 트리거면 from-stage)
scope: str = "session_end" # 'session_end' | 'stage_transition'
stage: str # 평가 시점 단계(전환 트리거면 from-stage)
scope: str = "session_end" # 'session_end' | 'stage_transition'
turns_evaluated: int = 0
distribution: TechniqueDistribution = Field(default_factory=TechniqueDistribution)
strengths: list[str] = Field(default_factory=list) # 잘한 순간(근거 포함 문장)
improvements: list[str] = Field(default_factory=list) # 개선점(최대 3)
strengths: list[str] = Field(default_factory=list) # 잘한 순간(근거 포함 문장)
improvements: list[str] = Field(default_factory=list) # 개선점(최대 3)
intent_deviations: list[IntentDeviation] = Field(default_factory=list)
supervisor_rationale: Optional[str] = None # CommentKind.RATIONALE 종합
supervisor_critique: Optional[str] = None # CommentKind.CRITIQUE 종합
supervisor_rationale: Optional[str] = None # CommentKind.RATIONALE 종합
supervisor_critique: Optional[str] = None # CommentKind.CRITIQUE 종합
alternative_utterances: list[str] = Field(default_factory=list) # 대안 발화 제시
theory_mode: Optional[str] = None
error: Optional[str] = None
@ -324,7 +330,7 @@ def _fast_schema() -> dict[str, Any]:
"required": ["code"],
},
},
"appropriateness": {"type": "string", "enum": list(_APPROPRIATENESS)},
"appropriateness": {"type": "string", "enum": list(APPROPRIATENESS_VALUES)},
"appropriateness_note": {"type": "string"},
"rapport_signal": {"type": "number", "minimum": -1, "maximum": 1},
"intent_deviation": {
@ -350,7 +356,11 @@ def _deep_schema() -> dict[str, Any]:
"additionalProperties": False,
"properties": {
"strengths": {"type": "array", "items": {"type": "string"}},
"improvements": {"type": "array", "items": {"type": "string"}, "maxItems": 3},
"improvements": {
"type": "array",
"items": {"type": "string"},
"maxItems": 3,
},
"intent_deviations": {
"type": "array",
"items": {
@ -405,7 +415,9 @@ def _client_state_candidates_block() -> str:
# ─ few-shot 골든셋 예시(data/golden) — 명시적으로 켠 환경에서만 로딩 ─────────
# 골든셋은 학습/평가 보정 자료이지 운영 런타임의 기본 데이터가 아니다.
_GOLDEN_FEWSHOT_ENABLED = os.environ.get("EVALUATOR_GOLDEN_FEWSHOT_ENABLED", "").lower() in {
_GOLDEN_FEWSHOT_ENABLED = os.environ.get(
"EVALUATOR_GOLDEN_FEWSHOT_ENABLED", ""
).lower() in {
"1",
"true",
"yes",
@ -456,7 +468,11 @@ def _fewshot_block() -> str:
techs = ",".join(e.get("techniques", []))
text = (e.get("text") or "").replace("\n", " ")[:70]
rat = next(
(c.get("text", "") for c in e.get("comments", []) if c.get("kind") == "rationale"),
(
c.get("text", "")
for c in e.get("comments", [])
if c.get("kind") == "rationale"
),
"",
)
line = f'- "{text}…" → {techs}'
@ -482,10 +498,13 @@ def build_fast_messages(ctx: "TurnContext", client_reply: str) -> list[EngineMes
st = ctx.state_after or ctx.state_before
theory = _theory_mode(ctx)
client_reply_masked = guardrail.mask_pii(client_reply).text_masked
recent = "\n".join(
f"{speaker_ko_label(t.get('speaker'))}: {t.get('text', '')}"
for t in (ctx.memory.recent_turns or [])[-4:]
) or "(직전 맥락 없음)"
recent = (
"\n".join(
f"{speaker_ko_label(t.get('speaker'))}: {t.get('text', '')}"
for t in (ctx.memory.recent_turns or [])[-4:]
)
or "(직전 맥락 없음)"
)
crisis_note = ""
if ctx.crisis is not None and getattr(ctx.crisis, "escalate", False):
@ -495,7 +514,8 @@ def build_fast_messages(ctx: "TurnContext", client_reply: str) -> list[EngineMes
)
system = "\n\n".join(
p for p in [
p
for p in [
_EVAL_ROLE,
_technique_candidates_block(),
_client_state_candidates_block(),
@ -509,7 +529,8 @@ def build_fast_messages(ctx: "TurnContext", client_reply: str) -> list[EngineMes
"없으면 null. 이 항목은 가장 중요하다 — 무리한 생성 금지, 진짜 이탈만.\n"
"추가로 rapport_signal(−1~+1): 이 발화가 라포에 끼친 방향(공감·반영=+, 조언점프·평가=−)."
),
] if p
]
if p
)
user = (
@ -537,11 +558,16 @@ def build_deep_messages(
distribution: "TechniqueDistribution",
) -> list[EngineMessage]:
"""deep-loop 평가 프롬프트(전체 회기 + 코드 집계 분포 + 골든라벨 후보)."""
transcript = "\n".join(
f"{t.get('seq', '')}{speaker_ko_label(t.get('speaker'))}: {t.get('text', '')}"
for t in masked_turns
) or "(축어록 없음)"
dist_lines = ", ".join(f"{k}:{v}" for k, v in distribution.by_category.items()) or "(없음)"
transcript = (
"\n".join(
f"{t.get('seq', '')}{speaker_ko_label(t.get('speaker'))}: {t.get('text', '')}"
for t in masked_turns
)
or "(축어록 없음)"
)
dist_lines = (
", ".join(f"{k}:{v}" for k, v in distribution.by_category.items()) or "(없음)"
)
over = ", ".join(distribution.overused) or "(없음)"
under = ", ".join(distribution.underused) or "(없음)"
@ -590,8 +616,9 @@ def _parse_intent_deviation(d: Any) -> Optional[IntentDeviation]:
return IntentDeviation(dimension=dim, expected=exp, actual=act, severity=sev)
def _parse_fast(payload: dict[str, Any], *, turn_seq: int, stage: str,
theory: Optional[str]) -> TurnEvaluation:
def _parse_fast(
payload: dict[str, Any], *, turn_seq: int, stage: str, theory: Optional[str]
) -> TurnEvaluation:
techniques: list[TechniqueTag] = []
for item in payload.get("techniques") or []:
if not isinstance(item, dict):
@ -604,7 +631,9 @@ def _parse_fast(payload: dict[str, Any], *, turn_seq: int, stage: str,
code=t.value,
label_ko=TECHNIQUE_KO[t],
category=TECHNIQUE_CATEGORY[t].value,
rationale=(str(item.get("rationale")).strip() or None) if item.get("rationale") else None,
rationale=(str(item.get("rationale")).strip() or None)
if item.get("rationale")
else None,
)
)
@ -619,12 +648,14 @@ def _parse_fast(payload: dict[str, Any], *, turn_seq: int, stage: str,
ClientStateRead(
code=s.value,
label_ko=CLIENT_STATE_KO[s],
rationale=(str(item.get("rationale")).strip() or None) if item.get("rationale") else None,
rationale=(str(item.get("rationale")).strip() or None)
if item.get("rationale")
else None,
)
)
appro = str(payload.get("appropriateness") or "neutral").strip()
if appro not in _APPROPRIATENESS:
if appro not in APPROPRIATENESS_VALUES:
appro = "neutral"
rapport = payload.get("rapport_signal")
@ -706,7 +737,9 @@ async def evaluate_turn(
"""
st = ctx.state_after or ctx.state_before
theory = _theory_mode(ctx)
base = TurnEvaluation(loop="fast", turn_seq=st.turn_seq, stage=st.stage.value, theory_mode=theory)
base = TurnEvaluation(
loop="fast", turn_seq=st.turn_seq, stage=st.stage.value, theory_mode=theory
)
try:
req = GenerateRequest(
@ -715,7 +748,7 @@ async def evaluate_turn(
structured_schema=_fast_schema(),
model=_configured_model(settings.evaluator_fast_model),
max_tokens=900,
temperature=0.2, # 평가는 보수적·재현적으로
temperature=0.2, # 평가는 보수적·재현적으로
session_id=ctx.session_id,
metadata={"loop": "fast", "stage": st.stage.value, "turn_seq": st.turn_seq},
)
@ -723,20 +756,7 @@ async def evaluate_turn(
cached = _evaluator_cache_get(cache_key)
if cached is not None:
return TurnEvaluation.model_validate(cached)
started = time.perf_counter()
resp = await engine.generate(req)
latency_ms = int((time.perf_counter() - started) * 1000)
await _record_llm_audit(
audit_hook,
session_id=ctx.session_id,
provider=resp.provider,
model=resp.model,
tokens_in=resp.tokens_in,
tokens_out=resp.tokens_out,
cost_usd=resp.cost_usd,
inference_geo=resp.inference_geo,
latency_ms=latency_ms,
)
resp = await generate_with_audit(engine, req, audit_hook)
except EngineError:
base.error = "engine_error"
return base
@ -749,7 +769,9 @@ async def evaluate_turn(
base.error = "no_structured_output"
return base
try:
result = _parse_fast(payload, turn_seq=st.turn_seq, stage=st.stage.value, theory=theory)
result = _parse_fast(
payload, turn_seq=st.turn_seq, stage=st.stage.value, theory=theory
)
_evaluator_cache_put(cache_key, result.model_dump())
return result
except Exception: # 파싱 방어
@ -806,20 +828,7 @@ async def evaluate_session(
cached = _evaluator_cache_get(cache_key)
if cached is not None:
return SessionEvaluation.model_validate(cached)
started = time.perf_counter()
resp = await engine.generate(req)
latency_ms = int((time.perf_counter() - started) * 1000)
await _record_llm_audit(
audit_hook,
session_id=session_id,
provider=resp.provider,
model=resp.model,
tokens_in=resp.tokens_in,
tokens_out=resp.tokens_out,
cost_usd=resp.cost_usd,
inference_geo=resp.inference_geo,
latency_ms=latency_ms,
)
resp = await generate_with_audit(engine, req, audit_hook)
except EngineError as e:
base.error = f"engine_error: {e}"
return base
@ -834,7 +843,9 @@ async def evaluate_session(
base.strengths = _coerce_str_list(payload.get("strengths"))
base.improvements = _coerce_str_list(payload.get("improvements"))[:3] # 최대 3
base.alternative_utterances = _coerce_str_list(payload.get("alternative_utterances"))
base.alternative_utterances = _coerce_str_list(
payload.get("alternative_utterances")
)
rationale = payload.get("supervisor_rationale")
critique = payload.get("supervisor_critique")
base.supervisor_rationale = str(rationale).strip() if rationale else None
@ -850,18 +861,6 @@ async def evaluate_session(
# ════════════════════════════════════════════════════════════════════════════
# 7. orchestrator EvalHook 어댑터 — 주입형 클로저(엔진 바인딩)
# ════════════════════════════════════════════════════════════════════════════
async def _record_llm_audit(
audit_hook: Optional["LlmAuditHook"],
**payload: Any,
) -> None:
if audit_hook is None:
return
try:
await audit_hook(payload)
except Exception:
return
def make_eval_hook(
engine: EngineClient,
*,
@ -874,7 +873,9 @@ def make_eval_hook(
"""
async def _hook(ctx: "TurnContext", client_reply: str) -> Optional[dict[str, Any]]:
ev = await evaluate_turn(ctx, client_reply, engine=engine, audit_hook=audit_hook)
ev = await evaluate_turn(
ctx, client_reply, engine=engine, audit_hook=audit_hook
)
d = ev.to_hook_dict()
return d if d else None