현재 작업 전체 반영

apps/api/app/services/evaluator.py

@@ -26,6 +26,7 @@ from __future__ import annotations
 
 import json
 import os
+import time
 from typing import TYPE_CHECKING, Any, Optional
 
 from pydantic import BaseModel, Field
@@ -48,7 +49,7 @@ from ..taxonomy import (
 )
 
 if TYPE_CHECKING:  # 런타임 import 회피(순환·소유권 경계). 타입 힌트 전용.
-    from .orchestrator import TurnContext
+    from .orchestrator import LlmAuditHook, TurnContext
 
 
 # ════════════════════════════════════════════════════════════════════════════
@@ -624,6 +625,7 @@ async def evaluate_turn(
     client_reply: str,
     *,
     engine: EngineClient,
+    audit_hook: Optional["LlmAuditHook"] = None,
 ) -> TurnEvaluation:
     """fast-loop 턴 평가 — 턴 직후 경량 4차원 태깅(비치명적).
 
@@ -644,7 +646,20 @@ async def evaluate_turn(
             session_id=ctx.session_id,
             metadata={"loop": "fast", "stage": st.stage.value, "turn_seq": st.turn_seq},
         )
+        started = time.perf_counter()
         resp = await engine.generate(req)
+        latency_ms = int((time.perf_counter() - started) * 1000)
+        await _record_llm_audit(
+            audit_hook,
+            session_id=ctx.session_id,
+            provider=resp.provider,
+            model=resp.model,
+            tokens_in=resp.tokens_in,
+            tokens_out=resp.tokens_out,
+            cost_usd=resp.cost_usd,
+            inference_geo=resp.inference_geo,
+            latency_ms=latency_ms,
+        )
     except EngineError as e:
         base.error = f"engine_error: {e}"
         return base
@@ -672,6 +687,7 @@ async def evaluate_session(
     technique_codes: Optional[list[str]] = None,
     theory_mode: Optional[str] = None,
     scope: str = "session_end",
+    audit_hook: Optional["LlmAuditHook"] = None,
 ) -> SessionEvaluation:
     """deep-loop 정밀 평가 — 단계전환/회기말. 전체 축어록 + 코드 집계 분포 + LLM 정성 평가.
 
@@ -706,7 +722,20 @@ async def evaluate_session(
             session_id=session_id,
             metadata={"loop": "deep", "scope": scope, "stage": stage},
         )
+        started = time.perf_counter()
         resp = await engine.generate(req)
+        latency_ms = int((time.perf_counter() - started) * 1000)
+        await _record_llm_audit(
+            audit_hook,
+            session_id=session_id,
+            provider=resp.provider,
+            model=resp.model,
+            tokens_in=resp.tokens_in,
+            tokens_out=resp.tokens_out,
+            cost_usd=resp.cost_usd,
+            inference_geo=resp.inference_geo,
+            latency_ms=latency_ms,
+        )
     except EngineError as e:
         base.error = f"engine_error: {e}"
         return base
@@ -736,7 +765,23 @@ async def evaluate_session(
 # ════════════════════════════════════════════════════════════════════════════
 # 7. orchestrator EvalHook 어댑터 — 주입형 클로저(엔진 바인딩)
 # ════════════════════════════════════════════════════════════════════════════
-def make_eval_hook(engine: EngineClient):
+async def _record_llm_audit(
+    audit_hook: Optional["LlmAuditHook"],
+    **payload: Any,
+) -> None:
+    if audit_hook is None:
+        return
+    try:
+        await audit_hook(payload)
+    except Exception:
+        return
+
+
+def make_eval_hook(
+    engine: EngineClient,
+    *,
+    audit_hook: Optional["LlmAuditHook"] = None,
+):
     """orchestrator.EvalHook(Callable[[TurnContext, str], Awaitable[Optional[dict]]]) 호환 클로저.
 
     sessions 라우트가 run_turn_generate(ctx, engine, eval_hook=make_eval_hook(engine_client)) 로
@@ -744,7 +789,7 @@ def make_eval_hook(engine: EngineClient):
     """
 
     async def _hook(ctx: "TurnContext", client_reply: str) -> Optional[dict[str, Any]]:
-        ev = await evaluate_turn(ctx, client_reply, engine=engine)
+        ev = await evaluate_turn(ctx, client_reply, engine=engine, audit_hook=audit_hook)
         d = ev.to_hook_dict()
         return d if d else None