diff --git a/apps/api/app/services/phase3_kpi_export.py b/apps/api/app/services/phase3_kpi_export.py index b396817..c152ebc 100644 --- a/apps/api/app/services/phase3_kpi_export.py +++ b/apps/api/app/services/phase3_kpi_export.py @@ -16,25 +16,13 @@ from pathlib import Path from typing import Any, Iterable, Mapping, Sequence from uuid import UUID -PREPOST_MEASURE_NAMES = ( - "self_efficacy", - "skill_proficiency", - "training_satisfaction", +from .phase3_kpi_contract import ( + KPI_REPORT_PATH, + PHASE3_KPI_METRICS, + PREPOST_CSV_PATH, + PREPOST_MEASURE_NAMES, + PREPOST_TIMEPOINTS, ) -PREPOST_TIMEPOINTS = ("pre", "post") -PHASE3_KPI_METRICS = ( - "embedding_consistency", - "hallucination_rate", - "icc", - "inter_rater_kappa", - "pilot_completion", - "self_efficacy_prepost", - "session_completion", - "sus", - "top1", -) -PREPOST_CSV_PATH = "02-measures/prepost_measures.csv" -KPI_REPORT_PATH = "02-measures/kpi_report.json" class ParticipantKeys: @@ -173,7 +161,7 @@ def build_kpi_report( ) -> dict[str, Any]: latest_rows = latest_prepost_rows(rows) participants = {str(row.get("learner_id") or row.get("participant_id") or "") for row in latest_rows} - metrics = {name: _placeholder_metric(name) for name in PHASE3_KPI_METRICS} + metrics = {name: _design_pending_metric(name) for name in PHASE3_KPI_METRICS} self_efficacy = paired_prepost_summary(latest_rows, "self_efficacy") metrics["self_efficacy_prepost"] = { @@ -183,6 +171,7 @@ def build_kpi_report( "numerator": self_efficacy["complete_pairs"], "denominator": max(self_efficacy["participants_with_any_measure"], 0), "method": "paired normalized post-pre delta for pilot review; no official pass/fail gate", + "status": "computed_prepost", "source_files": [PREPOST_CSV_PATH], "mean_pre": self_efficacy["mean_pre"], "mean_post": self_efficacy["mean_post"], @@ -194,12 +183,13 @@ def build_kpi_report( for measure_name in ("skill_proficiency", "training_satisfaction"): summary = paired_prepost_summary(latest_rows, measure_name) metrics[f"{measure_name}_prepost"] = { - **_placeholder_metric(f"{measure_name}_prepost"), + **_design_pending_metric(f"{measure_name}_prepost"), **summary, "value": summary["mean_delta"], "numerator": summary["complete_pairs"], "denominator": summary["participants_with_any_measure"], "method": "paired normalized post-pre delta for pilot review; not a required KPI gate yet", + "status": "computed_prepost", "source_files": [PREPOST_CSV_PATH], } @@ -241,13 +231,14 @@ def write_kpi_report(report: Mapping[str, Any], path: Path) -> None: ) -def _placeholder_metric(name: str) -> dict[str, Any]: +def _design_pending_metric(name: str) -> dict[str, Any]: return { "value": 0.0, "threshold": 0.0, "pass": False, "numerator": 0, "denominator": 0, + "status": "design_pending", "method": f"not computed by prepost export scaffold: {name}", "source_files": [], } diff --git a/apps/api/app/test_phase3_artifact_checker.py b/apps/api/app/test_phase3_artifact_checker.py index 681d73c..6ee4189 100644 --- a/apps/api/app/test_phase3_artifact_checker.py +++ b/apps/api/app/test_phase3_artifact_checker.py @@ -75,6 +75,7 @@ class Phase3ArtifactCheckerTests(unittest.TestCase): "pass": True, "numerator": 1, "denominator": 1, + "status": "computed_prepost", "method": "fixture", "source_files": ["fixture"], } @@ -195,6 +196,20 @@ class Phase3ArtifactCheckerTests(unittest.TestCase): report.errors, ) + def test_kpi_metric_status_must_be_known(self) -> None: + root = self.make_root() + report_path = root / "02-measures" / "kpi_report.json" + data = json.loads(report_path.read_text(encoding="utf-8")) + data["metrics"]["sus"]["status"] = "maybe_later" + report_path.write_text(json.dumps(data), encoding="utf-8") + + report = checker.validate(root, max_scan_rows=100) + + self.assertTrue( + any("metric 'sus' status 'maybe_later' is invalid" in error for error in report.errors), + report.errors, + ) + def test_csv_enum_values_are_validated(self) -> None: root = self.make_root() write_text( diff --git a/apps/api/app/test_phase3_kpi_export.py b/apps/api/app/test_phase3_kpi_export.py index f60f283..757b766 100644 --- a/apps/api/app/test_phase3_kpi_export.py +++ b/apps/api/app/test_phase3_kpi_export.py @@ -23,6 +23,7 @@ REQUIRED_METRIC_KEYS = { "numerator", "pass", "source_files", + "status", "threshold", "value", } @@ -136,8 +137,10 @@ class Phase3KpiExportTests(unittest.TestCase): self.assertTrue(REQUIRED_METRIC_KEYS.issubset(metric)) self_efficacy = report["metrics"]["self_efficacy_prepost"] self.assertFalse(self_efficacy["pass"]) + self.assertEqual(self_efficacy["status"], "computed_prepost") self.assertEqual(self_efficacy["value"], 25.0) self.assertEqual(self_efficacy["source_files"], [PREPOST_CSV_PATH]) + self.assertEqual(report["metrics"]["icc"]["status"], "design_pending") def test_writers_create_phase3_evidence_files(self) -> None: rows = build_prepost_csv_rows(fixture_rows(), participant_keys=ParticipantKeys()) diff --git a/scripts/check-phase3-artifacts.py b/scripts/check-phase3-artifacts.py index 6472915..e4ea084 100644 --- a/scripts/check-phase3-artifacts.py +++ b/scripts/check-phase3-artifacts.py @@ -160,9 +160,14 @@ KPI_METRIC_REQUIRED_KEYS = { "numerator", "pass", "source_files", + "status", "threshold", "value", } +KPI_METRIC_STATUSES = { + "computed_prepost", + "design_pending", +} MANIFEST_KEYS = { "agreement", @@ -342,6 +347,13 @@ def validate_kpi_report(root: Path, report: Report) -> None: report.error(f"{rel_path}: metric '{metric_name}' missing '{key}'") if "source_files" in metric and not isinstance(metric["source_files"], list): report.error(f"{rel_path}: metric '{metric_name}' source_files must be a list") + status = metric.get("status") + if status is not None and status not in KPI_METRIC_STATUSES: + expected = ", ".join(sorted(KPI_METRIC_STATUSES)) + report.error( + f"{rel_path}: metric '{metric_name}' status '{status}' is invalid " + f"(expected one of: {expected})" + ) def sha256_file(path: Path) -> str: