Jev 기반 내담자 감정 상태와 응답 일관성 개선
This commit is contained in:
Yun Chan
parent
77f8421818
commit
8344bc2ad2
23 changed files with 3384 additions and 25 deletions
432
scripts/evaluate-jev-client.py
Normal file
432
scripts/evaluate-jev-client.py
Normal file
|
|
@ -0,0 +1,432 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Jev 감정 판단 API의 공개 합성 한국어 사례 실측 러너."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps" / "api"
|
||||
FIXTURE_PATH = REPO_ROOT / "scripts" / "fixtures" / "jev-client-korean-cases.json"
|
||||
REQUIRED_STATE_KEYS = frozenset(
|
||||
{
|
||||
"persona",
|
||||
"memory",
|
||||
"recent_turns",
|
||||
"counselor_utterance",
|
||||
"previous_emotions",
|
||||
"current_state",
|
||||
}
|
||||
)
|
||||
SENSITIVE_FIELD_PATTERN = re.compile(r"(?:api[_-]?key|authorization|password|secret|token)", re.IGNORECASE)
|
||||
SECRET_VALUE_PATTERN = re.compile(r"(?:sk-|bearer\s+|AIza|AKIA)[A-Za-z0-9_\-]{8,}", re.IGNORECASE)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="공개 합성 한국어 가상 내담자 사례로 Jev 감정 판단 API를 실측한다."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
required=True,
|
||||
help="로컬 JSON 보고서 저장 경로",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fixtures",
|
||||
type=Path,
|
||||
default=FIXTURE_PATH,
|
||||
help="공개 합성 사례 JSON 경로",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repeats",
|
||||
type=int,
|
||||
default=1,
|
||||
help="각 사례의 반복 횟수(1~20, 기본 1)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[dict[str, Any]]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("fixture_load_failed") from exc
|
||||
if not isinstance(payload, dict) or payload.get("provenance") != "public_synthetic":
|
||||
raise ValueError("fixture_provenance_invalid")
|
||||
cases = payload.get("cases")
|
||||
if not isinstance(cases, list) or not 8 <= len(cases) <= 12:
|
||||
raise ValueError("fixture_case_count_invalid")
|
||||
|
||||
identifiers: set[str] = set()
|
||||
for case in cases:
|
||||
validate_case(case, identifiers)
|
||||
return cases
|
||||
|
||||
|
||||
def validate_case(case: Any, identifiers: set[str]) -> None:
|
||||
if not isinstance(case, dict) or set(case) != {
|
||||
"id",
|
||||
"description",
|
||||
"state",
|
||||
"review_questions",
|
||||
}:
|
||||
raise ValueError("fixture_case_shape_invalid")
|
||||
case_id = case["id"]
|
||||
if not isinstance(case_id, str) or not case_id or case_id in identifiers:
|
||||
raise ValueError("fixture_case_id_invalid")
|
||||
identifiers.add(case_id)
|
||||
if not isinstance(case["description"], str) or not case["description"]:
|
||||
raise ValueError("fixture_description_invalid")
|
||||
state = case["state"]
|
||||
if not isinstance(state, dict) or set(state) != REQUIRED_STATE_KEYS:
|
||||
raise ValueError("fixture_state_contract_invalid")
|
||||
persona = state["persona"]
|
||||
memory = state["memory"]
|
||||
recent_turns = state["recent_turns"]
|
||||
emotions = state["previous_emotions"]
|
||||
current_state = state["current_state"]
|
||||
context = persona.get("context") if isinstance(persona, dict) else None
|
||||
if (
|
||||
not isinstance(persona, dict)
|
||||
or set(persona) != {"affect_baseline", "context"}
|
||||
or not isinstance(persona["affect_baseline"], dict)
|
||||
or not all(isinstance(value, (int, float)) and math.isfinite(value) for value in persona["affect_baseline"].values())
|
||||
or not isinstance(context, dict)
|
||||
or set(context) != {
|
||||
"big5", "resistance", "speech_style", "presenting", "history", "ccd", "triggers"
|
||||
}
|
||||
or not all(isinstance(context[key], dict) for key in ("big5", "resistance", "speech_style", "ccd"))
|
||||
or not all(isinstance(context[key], str) and context[key] for key in ("presenting", "history"))
|
||||
or not isinstance(context["triggers"], list)
|
||||
or not all(isinstance(trigger, str) and trigger for trigger in context["triggers"])
|
||||
):
|
||||
raise ValueError("fixture_persona_invalid")
|
||||
if (
|
||||
not isinstance(memory, dict)
|
||||
or set(memory) != {"recall_summary", "pinned_facts"}
|
||||
or not isinstance(memory["recall_summary"], str)
|
||||
or not isinstance(memory["pinned_facts"], list)
|
||||
or not all(isinstance(item, str) for item in memory["pinned_facts"])
|
||||
):
|
||||
raise ValueError("fixture_memory_invalid")
|
||||
if (
|
||||
not isinstance(recent_turns, list)
|
||||
or len(recent_turns) > 12
|
||||
or not all(
|
||||
isinstance(turn, dict)
|
||||
and set(turn) == {"speaker", "text"}
|
||||
and turn["speaker"] in {"counselor", "client"}
|
||||
and isinstance(turn["text"], str)
|
||||
for turn in recent_turns
|
||||
)
|
||||
or not isinstance(state["counselor_utterance"], str)
|
||||
):
|
||||
raise ValueError("fixture_recent_turns_invalid")
|
||||
if (
|
||||
not isinstance(emotions, dict)
|
||||
or set(emotions) != {
|
||||
"anxiety", "sadness", "anger", "shame", "guilt", "loneliness", "relief", "hope", "trust"
|
||||
}
|
||||
or not all(isinstance(value, (int, float)) and 0.0 <= value <= 1.0 for value in emotions.values())
|
||||
):
|
||||
raise ValueError("fixture_previous_emotions_invalid")
|
||||
if (
|
||||
not isinstance(current_state, dict)
|
||||
or set(current_state) != {"resistance", "effective_openness"}
|
||||
or not all(isinstance(value, (int, float)) and 0.0 <= value <= 1.0 for value in current_state.values())
|
||||
):
|
||||
raise ValueError("fixture_current_state_invalid")
|
||||
questions = case["review_questions"]
|
||||
if (
|
||||
not isinstance(questions, list)
|
||||
or len(questions) != 2
|
||||
or not all(isinstance(question, str) and question for question in questions)
|
||||
):
|
||||
raise ValueError("fixture_review_questions_invalid")
|
||||
if contains_sensitive_content(case):
|
||||
raise ValueError("fixture_sensitive_content")
|
||||
|
||||
|
||||
def contains_sensitive_content(value: Any, field_name: str = "") -> bool:
|
||||
if SENSITIVE_FIELD_PATTERN.search(field_name):
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
return bool(SECRET_VALUE_PATTERN.search(value))
|
||||
if isinstance(value, dict):
|
||||
return any(contains_sensitive_content(item, str(key)) for key, item in value.items())
|
||||
if isinstance(value, list):
|
||||
return any(contains_sensitive_content(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def percentile(values: list[int], percent: float) -> int | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
position = (len(ordered) - 1) * percent
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower))
|
||||
|
||||
|
||||
def base_report(
|
||||
*,
|
||||
fixture_path: Path,
|
||||
case_count: int,
|
||||
repeats: int,
|
||||
provider: str,
|
||||
requested_model: str,
|
||||
timeout_seconds: float,
|
||||
confidence_threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "blocked",
|
||||
"fixture": {
|
||||
"path": str(fixture_path),
|
||||
"sha256": fixture_sha256(fixture_path),
|
||||
"case_count": case_count,
|
||||
"repeats": repeats,
|
||||
"provenance": "public_synthetic",
|
||||
"contains_real_patient_data": False,
|
||||
},
|
||||
"measurement_started_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"configuration": {
|
||||
"route_provider": provider,
|
||||
"requested_model": requested_model,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"confidence_threshold": confidence_threshold,
|
||||
},
|
||||
"metrics": {
|
||||
"attempted_calls": 0,
|
||||
"success_count": 0,
|
||||
"failure_count": 0,
|
||||
"appraisal_latency_ms": {"sample_count": 0, "p50": None, "p95": None},
|
||||
"input_tokens_total": 0,
|
||||
"output_tokens_total": 0,
|
||||
"cost_usd_total": None,
|
||||
"known_success_cost_usd": None,
|
||||
"failed_calls_cost_known": None,
|
||||
"actual_models": [],
|
||||
},
|
||||
"results": [],
|
||||
"limitations": [
|
||||
"이 결과는 Jev 감정 판단 API 실측이며 전체 응답 지연, TTFT, 임상 타당성의 증거가 아니다.",
|
||||
"전문가 검토 질문은 사례별 결과와 분리해 fixture에만 보관하며 자동 정답 또는 정확도 판정에 사용하지 않는다.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def fixture_sha256(path: Path) -> str | None:
|
||||
try:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def write_report(path: Path, report: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
async def collect(report: dict[str, Any], cases: list[dict[str, Any]], repeats: int) -> None:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
from app.services.jev_client import JevError, jev_client
|
||||
|
||||
confidence_threshold = report["configuration"]["confidence_threshold"]
|
||||
latencies: list[int] = []
|
||||
models: set[str] = set()
|
||||
known_costs: list[float] = []
|
||||
try:
|
||||
await jev_client.startup()
|
||||
for repeat in range(1, repeats + 1):
|
||||
for case in cases:
|
||||
case_id = case["id"]
|
||||
report["metrics"]["attempted_calls"] += 1
|
||||
try:
|
||||
appraisal = await jev_client.appraise(case["state"])
|
||||
except JevError as exc:
|
||||
report["metrics"]["failure_count"] += 1
|
||||
report["results"].append(
|
||||
{"case_id": case_id, "repeat": repeat, "status": "failed", "error_code": exc.code}
|
||||
)
|
||||
continue
|
||||
except Exception:
|
||||
report["metrics"]["failure_count"] += 1
|
||||
report["results"].append(
|
||||
{"case_id": case_id, "repeat": repeat, "status": "failed", "error_code": "unexpected_error"}
|
||||
)
|
||||
continue
|
||||
|
||||
dimensions = {}
|
||||
for name, estimate in appraisal.emotions.items():
|
||||
confidence_missing = estimate.confidence is None
|
||||
dimensions[name] = {
|
||||
"score": estimate.score,
|
||||
"confidence": estimate.confidence,
|
||||
"confidence_missing": confidence_missing,
|
||||
"below_confidence_threshold": (
|
||||
True if confidence_missing else estimate.confidence < confidence_threshold
|
||||
),
|
||||
"probabilities": estimate.probabilities,
|
||||
}
|
||||
report["metrics"]["success_count"] += 1
|
||||
report["metrics"]["input_tokens_total"] += appraisal.input_tokens
|
||||
report["metrics"]["output_tokens_total"] += appraisal.output_tokens
|
||||
latencies.append(appraisal.latency_ms)
|
||||
models.add(appraisal.model)
|
||||
if appraisal.cost_usd is not None:
|
||||
known_costs.append(appraisal.cost_usd)
|
||||
report["results"].append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"repeat": repeat,
|
||||
"status": "collected",
|
||||
"provider": appraisal.provider,
|
||||
"actual_model": appraisal.model,
|
||||
"appraisal_latency_ms": appraisal.latency_ms,
|
||||
"input_tokens": appraisal.input_tokens,
|
||||
"output_tokens": appraisal.output_tokens,
|
||||
"dimensions": dimensions,
|
||||
}
|
||||
)
|
||||
except JevError as exc:
|
||||
report["metrics"]["failure_count"] += 1
|
||||
report["results"].append(
|
||||
{"case_id": "runner", "repeat": 0, "status": "failed", "error_code": exc.code}
|
||||
)
|
||||
except Exception:
|
||||
report["metrics"]["failure_count"] += 1
|
||||
report["results"].append(
|
||||
{"case_id": "runner", "repeat": 0, "status": "failed", "error_code": "unexpected_error"}
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
await jev_client.shutdown()
|
||||
except Exception:
|
||||
report["metrics"]["failure_count"] += 1
|
||||
report["results"].append(
|
||||
{"case_id": "runner", "repeat": 0, "status": "failed", "error_code": "shutdown_error"}
|
||||
)
|
||||
|
||||
report["metrics"]["actual_models"] = sorted(models)
|
||||
if known_costs:
|
||||
report["metrics"]["known_success_cost_usd"] = sum(known_costs)
|
||||
report["metrics"]["appraisal_latency_ms"] = {
|
||||
"sample_count": len(latencies),
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
}
|
||||
success_count = report["metrics"]["success_count"]
|
||||
failure_count = report["metrics"]["failure_count"]
|
||||
report["metrics"]["failed_calls_cost_known"] = failure_count == 0
|
||||
if (
|
||||
failure_count == 0
|
||||
and success_count > 0
|
||||
and success_count == len(known_costs)
|
||||
):
|
||||
report["metrics"]["cost_usd_total"] = report["metrics"]["known_success_cost_usd"]
|
||||
if failure_count == 0:
|
||||
report["status"] = "collected"
|
||||
elif success_count == 0:
|
||||
report["status"] = "failed"
|
||||
else:
|
||||
report["status"] = "partial"
|
||||
|
||||
|
||||
def load_settings() -> Any:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
original_cwd = Path.cwd()
|
||||
try:
|
||||
os.chdir(API_ROOT)
|
||||
from app.config import settings
|
||||
|
||||
return settings
|
||||
finally:
|
||||
os.chdir(original_cwd)
|
||||
|
||||
|
||||
def provider_key_present(settings: Any) -> bool:
|
||||
if settings.jev_provider == "openrouter":
|
||||
return bool(settings.openrouter_api_key.get_secret_value().strip())
|
||||
if settings.jev_provider == "typesafe":
|
||||
return bool(settings.typesafe_api_key.get_secret_value().strip())
|
||||
return False
|
||||
|
||||
|
||||
def invalid_configuration_report(*, fixture_path: Path, case_count: int, repeats: int) -> dict[str, Any]:
|
||||
report = base_report(
|
||||
fixture_path=fixture_path,
|
||||
case_count=case_count,
|
||||
repeats=repeats,
|
||||
provider="unavailable",
|
||||
requested_model="unavailable",
|
||||
timeout_seconds=0.0,
|
||||
confidence_threshold=0.0,
|
||||
)
|
||||
report["blocking_reason"] = "configuration_invalid"
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
if not 1 <= args.repeats <= 20:
|
||||
raise SystemExit("--repeats must be between 1 and 20")
|
||||
try:
|
||||
cases = load_cases(args.fixtures)
|
||||
except ValueError as exc:
|
||||
report = {
|
||||
"status": "failed",
|
||||
"failure_reason": str(exc),
|
||||
"measurement_started_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
write_report(Path(args.output), report)
|
||||
return 1
|
||||
|
||||
try:
|
||||
settings = load_settings()
|
||||
except ValidationError:
|
||||
write_report(
|
||||
Path(args.output),
|
||||
invalid_configuration_report(
|
||||
fixture_path=args.fixtures,
|
||||
case_count=len(cases),
|
||||
repeats=args.repeats,
|
||||
),
|
||||
)
|
||||
return 2
|
||||
report = base_report(
|
||||
fixture_path=args.fixtures,
|
||||
case_count=len(cases),
|
||||
repeats=args.repeats,
|
||||
provider=settings.jev_provider,
|
||||
requested_model=settings.jev_model,
|
||||
timeout_seconds=settings.jev_timeout_seconds,
|
||||
confidence_threshold=settings.jev_min_confidence,
|
||||
)
|
||||
if not provider_key_present(settings):
|
||||
report["blocking_reason"] = f"{settings.jev_provider}_api_key_missing"
|
||||
write_report(Path(args.output), report)
|
||||
return 2
|
||||
|
||||
asyncio.run(collect(report, cases, args.repeats))
|
||||
write_report(Path(args.output), report)
|
||||
return 0 if report["status"] == "collected" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue