운영 지표와 지원 요청 저장소 추가

apps/api/app/services/phase3_kpi_export.py

@@ -0,0 +1,276 @@
+"""Phase 3 KPI evidence export helpers.
+
+This module turns persisted pre/post aggregate scores into the Phase 3 evidence
+shape checked by scripts/check-phase3-artifacts.py. It does not claim clinical
+effectiveness; it only produces pilot evidence files for operator review.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from collections import defaultdict
+from datetime import UTC, date, datetime
+from decimal import Decimal
+from pathlib import Path
+from typing import Any, Iterable, Mapping, Sequence
+from uuid import UUID
+
+PREPOST_MEASURE_NAMES = (
+    "self_efficacy",
+    "skill_proficiency",
+    "training_satisfaction",
+)
+PREPOST_TIMEPOINTS = ("pre", "post")
+PHASE3_KPI_METRICS = (
+    "embedding_consistency",
+    "hallucination_rate",
+    "icc",
+    "inter_rater_kappa",
+    "pilot_completion",
+    "self_efficacy_prepost",
+    "session_completion",
+    "sus",
+    "top1",
+)
+PREPOST_CSV_PATH = "02-measures/prepost_measures.csv"
+KPI_REPORT_PATH = "02-measures/kpi_report.json"
+
+
+class ParticipantKeys:
+    def __init__(self) -> None:
+        self._keys: dict[str, str] = {}
+
+    def key(self, raw_id: Any) -> str:
+        value = str(raw_id or "unknown-participant")
+        if value not in self._keys:
+            self._keys[value] = f"P3-{len(self._keys) + 1:03d}"
+        return self._keys[value]
+
+    def __len__(self) -> int:
+        return len(self._keys)
+
+
+def json_safe(value: Any) -> Any:
+    if isinstance(value, datetime):
+        if value.tzinfo is None:
+            value = value.replace(tzinfo=UTC)
+        return value.isoformat().replace("+00:00", "Z")
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, (Decimal, UUID)):
+        return str(value)
+    if isinstance(value, Mapping):
+        return {str(key): json_safe(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [json_safe(item) for item in value]
+    if isinstance(value, tuple):
+        return [json_safe(item) for item in value]
+    return value
+
+
+def iso_timestamp(value: Any) -> str:
+    safe = json_safe(value)
+    return safe if isinstance(safe, str) else str(safe or "")
+
+
+def normalized_score(raw_score: float, min_score: float, max_score: float) -> float:
+    if max_score <= min_score:
+        return 0.0
+    return round(((raw_score - min_score) / (max_score - min_score)) * 100.0, 3)
+
+
+def latest_prepost_rows(rows: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    latest: dict[tuple[str, str, str], dict[str, Any]] = {}
+    for row in rows:
+        learner_id = str(row.get("learner_id") or row.get("participant_id") or "")
+        measure_name = str(row.get("measure_name") or "")
+        timepoint = str(row.get("timepoint") or "")
+        if measure_name not in PREPOST_MEASURE_NAMES or timepoint not in PREPOST_TIMEPOINTS:
+            continue
+        key = (learner_id, measure_name, timepoint)
+        current = dict(row)
+        current_order = iso_timestamp(current.get("updated_at") or current.get("collected_at"))
+        previous = latest.get(key)
+        previous_order = iso_timestamp(previous.get("updated_at") or previous.get("collected_at")) if previous else ""
+        if previous is None or current_order >= previous_order:
+            latest[key] = current
+    return sorted(
+        latest.values(),
+        key=lambda item: (
+            str(item.get("learner_id") or item.get("participant_id") or ""),
+            str(item.get("measure_name") or ""),
+            str(item.get("timepoint") or ""),
+        ),
+    )
+
+
+def build_prepost_csv_rows(
+    rows: Iterable[Mapping[str, Any]],
+    *,
+    participant_keys: ParticipantKeys | None = None,
+) -> list[dict[str, str]]:
+    keys = participant_keys if participant_keys is not None else ParticipantKeys()
+    output: list[dict[str, str]] = []
+    for row in latest_prepost_rows(rows):
+        raw_score = float(row.get("raw_score") or row.get("score") or 0.0)
+        output.append(
+            {
+                "participant_id": keys.key(row.get("learner_id") or row.get("participant_id")),
+                "measure_name": str(row.get("measure_name") or ""),
+                "timepoint": str(row.get("timepoint") or ""),
+                "score": _format_number(raw_score),
+                "collected_at": iso_timestamp(row.get("collected_at") or row.get("updated_at")),
+            }
+        )
+    return output
+
+
+def paired_prepost_summary(rows: Iterable[Mapping[str, Any]], measure_name: str) -> dict[str, Any]:
+    by_participant: dict[str, dict[str, float]] = defaultdict(dict)
+    for row in latest_prepost_rows(rows):
+        if str(row.get("measure_name") or "") != measure_name:
+            continue
+        participant_id = str(row.get("learner_id") or row.get("participant_id") or "")
+        raw_score = float(row.get("raw_score") or row.get("score") or 0.0)
+        min_score = float(row.get("min_score") or 1.0)
+        max_score = float(row.get("max_score") or 5.0)
+        by_participant[participant_id][str(row.get("timepoint") or "")] = normalized_score(
+            raw_score,
+            min_score,
+            max_score,
+        )
+
+    deltas: list[float] = []
+    pre_values: list[float] = []
+    post_values: list[float] = []
+    missing_pairs = 0
+    for values in by_participant.values():
+        if "pre" not in values or "post" not in values:
+            missing_pairs += 1
+            continue
+        pre_values.append(values["pre"])
+        post_values.append(values["post"])
+        deltas.append(values["post"] - values["pre"])
+
+    return {
+        "participants_with_any_measure": len(by_participant),
+        "complete_pairs": len(deltas),
+        "missing_pairs": missing_pairs,
+        "mean_pre": _mean(pre_values),
+        "mean_post": _mean(post_values),
+        "mean_delta": _mean(deltas),
+    }
+
+
+def build_kpi_report(
+    rows: Sequence[Mapping[str, Any]],
+    *,
+    pilot_id: str,
+    generated_at: str,
+    source_window: Mapping[str, Any] | None = None,
+    review_operator: str = "",
+) -> dict[str, Any]:
+    latest_rows = latest_prepost_rows(rows)
+    participants = {str(row.get("learner_id") or row.get("participant_id") or "") for row in latest_rows}
+    metrics = {name: _placeholder_metric(name) for name in PHASE3_KPI_METRICS}
+
+    self_efficacy = paired_prepost_summary(latest_rows, "self_efficacy")
+    metrics["self_efficacy_prepost"] = {
+        "value": self_efficacy["mean_delta"],
+        "threshold": 0.0,
+        "pass": False,
+        "numerator": self_efficacy["complete_pairs"],
+        "denominator": max(self_efficacy["participants_with_any_measure"], 0),
+        "method": "paired normalized post-pre delta for pilot review; no official pass/fail gate",
+        "source_files": [PREPOST_CSV_PATH],
+        "mean_pre": self_efficacy["mean_pre"],
+        "mean_post": self_efficacy["mean_post"],
+        "mean_delta": self_efficacy["mean_delta"],
+        "complete_pairs": self_efficacy["complete_pairs"],
+        "missing_pairs": self_efficacy["missing_pairs"],
+    }
+
+    for measure_name in ("skill_proficiency", "training_satisfaction"):
+        summary = paired_prepost_summary(latest_rows, measure_name)
+        metrics[f"{measure_name}_prepost"] = {
+            **_placeholder_metric(f"{measure_name}_prepost"),
+            **summary,
+            "value": summary["mean_delta"],
+            "numerator": summary["complete_pairs"],
+            "denominator": summary["participants_with_any_measure"],
+            "method": "paired normalized post-pre delta for pilot review; not a required KPI gate yet",
+            "source_files": [PREPOST_CSV_PATH],
+        }
+
+    return {
+        "pilot_id": pilot_id,
+        "generated_at": generated_at,
+        "source_window": dict(source_window or _source_window(latest_rows)),
+        "cohort_size": len(participants),
+        "metrics": metrics,
+        "exclusions": [],
+        "open_schema_gaps": [
+            "official item text and validated scoring rules are not encoded here",
+            "experimental/control assignment and statistical testing require owner/evaluation-design approval",
+        ],
+        "review": {
+            "operator": review_operator,
+            "reviewed_at": "",
+            "decision": "pending",
+        },
+    }
+
+
+def write_prepost_csv(rows: Sequence[Mapping[str, str]], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=("participant_id", "measure_name", "timepoint", "score", "collected_at"),
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def write_kpi_report(report: Mapping[str, Any], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(json_safe(report), ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _placeholder_metric(name: str) -> dict[str, Any]:
+    return {
+        "value": 0.0,
+        "threshold": 0.0,
+        "pass": False,
+        "numerator": 0,
+        "denominator": 0,
+        "method": f"not computed by prepost export scaffold: {name}",
+        "source_files": [],
+    }
+
+
+def _source_window(rows: Sequence[Mapping[str, Any]]) -> dict[str, str]:
+    timestamps = [
+        iso_timestamp(row.get("collected_at") or row.get("updated_at"))
+        for row in rows
+        if iso_timestamp(row.get("collected_at") or row.get("updated_at"))
+    ]
+    if not timestamps:
+        return {"started_at": "", "ended_at": ""}
+    return {"started_at": min(timestamps), "ended_at": max(timestamps)}
+
+
+def _mean(values: Sequence[float]) -> float:
+    if not values:
+        return 0.0
+    return round(sum(values) / len(values), 3)
+
+
+def _format_number(value: float) -> str:
+    if value.is_integer():
+        return str(int(value))
+    return f"{value:.3f}".rstrip("0").rstrip(".")