주기 실회기 검증과 G7 종료계약 보강

apps/api/app/contracts/g7_external_evidence.py

@@ -32,7 +32,7 @@ class G7EvidenceProvenance(BaseModel):
 
     @model_validator(mode="after")
     def require_preregistered_analysis(self) -> "G7EvidenceProvenance":
-        if self.registered_at > self.held_out_labels_opened_at:
+        if self.registered_at >= self.held_out_labels_opened_at:
             raise ValueError("analysis protocol must precede held-out label access")
         return self
 
@@ -62,7 +62,7 @@ class G7PowerPlan(BaseModel):
     required_held_out_participants: int = Field(ge=1)
     required_held_out_sessions: int = Field(ge=1)
     required_paired_axis_observations: int = Field(ge=3)
-    alpha: float = Field(gt=0.0, le=0.05)
+    alpha: Literal[0.05] = 0.05
     target_power: float = Field(ge=0.8, lt=1.0)
     minimally_detectable_gain: float = Field(gt=0.0, le=1.0)
     planned_bootstrap_samples: Literal[10000] = 10000
@@ -91,10 +91,7 @@ class G7HumanAxisLabel(BaseModel):
 
     labeler_key: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,127}$")
     score: float = Field(ge=0.0, le=1.0)
-    category: str | None = Field(
-        default=None,
-        pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$",
-    )
+    category: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$")
 
 
 class G7ReliabilityClaim(BaseModel):
@@ -103,11 +100,7 @@ class G7ReliabilityClaim(BaseModel):
     method: Literal["ICC(A,1)"] = "ICC(A,1)"
     labeler_keys: tuple[str, ...] = Field(min_length=2)
     reported_icc: float = Field(ge=-1.0, le=1.0)
-    reported_categorical_kappa: float | None = Field(
-        default=None,
-        ge=-1.0,
-        le=1.0,
-    )
+    reported_categorical_kappa: float = Field(ge=-1.0, le=1.0)
     report_sha256: Sha256 = Field(pattern=r"^[a-f0-9]{64}$")
 
     @model_validator(mode="after")
@@ -146,9 +139,6 @@ class G7PairedAxisObservation(BaseModel):
         labeler_keys = [item.labeler_key for item in self.labels]
         if len(set(labeler_keys)) != len(labeler_keys):
             raise ValueError("observation labeler keys must be unique")
-        has_category = [item.category is not None for item in self.labels]
-        if any(has_category) and not all(has_category):
-            raise ValueError("categorical labels must be complete within an observation")
         return self
 
 
@@ -227,7 +217,6 @@ class G7HumanVoiceGainEvidencePack(BaseModel):
 
         participant_by_session: dict[str, str] = {}
         axes_by_session: dict[str, set[AllianceAxis]] = {}
-        categorical_modes: set[bool] = set()
         for observation in self.observations:
             if split_by_participant.get(observation.participant_key) != "held_out":
                 raise ValueError("evaluation observations must use held-out participants")
@@ -243,16 +232,8 @@ class G7HumanVoiceGainEvidencePack(BaseModel):
             row_labelers = {item.labeler_key for item in observation.labels}
             if row_labelers != reliability_panel:
                 raise ValueError("every row must use the declared reliability panel")
-            categorical_modes.add(observation.labels[0].category is not None)
 
         required_axes: set[AllianceAxis] = {"goal", "task", "bond"}
         if any(axes != required_axes for axes in axes_by_session.values()):
             raise ValueError("every held-out session must cover goal, task, and bond")
-        if len(categorical_modes) != 1:
-            raise ValueError("categorical labels must be all-present or all-absent")
-        has_categories = True in categorical_modes
-        if has_categories != (
-            self.reliability.reported_categorical_kappa is not None
-        ):
-            raise ValueError("categorical labels and reported kappa must appear together")
         return self

apps/api/app/services/g7_voice_gain_evidence.py

@@ -51,7 +51,9 @@ class VoiceGainEvidenceResult(BaseModel):
     passed: bool
     clinical_claim_allowed: Literal[False] = False
     held_out_participants: int
+    total_held_out_sessions: int
     held_out_sessions: int
+    total_axis_observations: int
     paired_axis_observations: int
     intention_to_evaluate_imputations: int
     text_only_one_minus_mae: float
@@ -62,7 +64,7 @@ class VoiceGainEvidenceResult(BaseModel):
     confidence_level: float
     bootstrap_samples: int
     recomputed_icc: float
-    recomputed_categorical_kappa: float | None
+    recomputed_categorical_kappa: float
     checks: tuple[VoiceGainEvidenceCheck, ...]
     failure_reasons: tuple[str, ...]
 
@@ -221,22 +223,23 @@ def evaluate_human_voice_gain(
         labels_by_key = {item.labeler_key: item for item in observation.labels}
         ordered_labels = [labels_by_key[labeler_key] for labeler_key in panel]
         ratings.append([item.score for item in ordered_labels])
-        if ordered_labels[0].category is not None:
-            categories.append([str(item.category) for item in ordered_labels])
+        categories.append([item.category for item in ordered_labels])
         reference = sum(item.score for item in ordered_labels) / len(ordered_labels)
 
-        if observation.text_only_status == "observed":
+        text_observed = observation.text_only_status == "observed"
+        voice_observed = observation.voice_enabled_status == "observed"
+        if text_observed and voice_observed:
             assert observation.text_only_score is not None
             text_error = abs(observation.text_only_score - reference)
-        else:
-            text_error = 1.0
-            imputation_count += 1
-        if observation.voice_enabled_status == "observed":
             assert observation.voice_enabled_score is not None
             voice_error = abs(observation.voice_enabled_score - reference)
         else:
+            # 한 조건만 결측이어도 두 조건을 모두 최대 오류로 대치한다. baseline-only
+            # 결측이 candidate gain을 인위적으로 키우는 비대칭을 차단하면서도
+            # intention-to-evaluate 행은 분석에서 유지한다.
+            text_error = 1.0
             voice_error = 1.0
-            imputation_count += 1
+            imputation_count += int(not text_observed) + int(not voice_observed)
         errors_by_participant[observation.participant_key].append(
             (text_error, voice_error)
         )
@@ -258,11 +261,28 @@ def evaluate_human_voice_gain(
         seed=thresholds.seed,
     )
     recomputed_icc = _icc_absolute_agreement_single(ratings)
-    recomputed_kappa = _fleiss_kappa(categories) if categories else None
+    recomputed_kappa = _fleiss_kappa(categories)
 
     held_out_participants = len(errors_by_participant)
-    held_out_sessions = len({item.session_key for item in pack.observations})
-    observation_count = len(pack.observations)
+    all_session_keys = {item.session_key for item in pack.observations}
+    complete_axes_by_session: dict[str, set[str]] = defaultdict(set)
+    for observation in pack.observations:
+        if (
+            observation.text_only_status == "observed"
+            and observation.voice_enabled_status == "observed"
+        ):
+            complete_axes_by_session[observation.session_key].add(observation.axis)
+    required_axes = {"goal", "task", "bond"}
+    held_out_sessions = sum(
+        axes == required_axes for axes in complete_axes_by_session.values()
+    )
+    total_observation_count = len(pack.observations)
+    paired_observation_count = sum(
+        1
+        for observation in pack.observations
+        if observation.text_only_status == "observed"
+        and observation.voice_enabled_status == "observed"
+    )
     checks: list[VoiceGainEvidenceCheck] = []
     _check(
         checks,
@@ -281,8 +301,8 @@ def evaluate_human_voice_gain(
     _check(
         checks,
         "production_observation_floor",
-        observation_count >= thresholds.min_paired_axis_observations,
-        observation_count,
+        paired_observation_count >= thresholds.min_paired_axis_observations,
+        paired_observation_count,
         thresholds.min_paired_axis_observations,
     )
     _check(
@@ -318,8 +338,8 @@ def evaluate_human_voice_gain(
     _check(
         checks,
         "power_plan_observations_achieved",
-        observation_count >= pack.power_plan.required_paired_axis_observations,
-        observation_count,
+        paired_observation_count >= pack.power_plan.required_paired_axis_observations,
+        paired_observation_count,
         pack.power_plan.required_paired_axis_observations,
     )
     _check(
@@ -355,26 +375,24 @@ def evaluate_human_voice_gain(
         recomputed_icc,
         pack.reliability.reported_icc,
     )
-    if recomputed_kappa is not None:
-        assert pack.reliability.reported_categorical_kappa is not None
-        _check(
-            checks,
-            "recomputed_categorical_kappa",
-            recomputed_kappa >= thresholds.min_categorical_kappa,
-            recomputed_kappa,
-            thresholds.min_categorical_kappa,
-        )
-        _check(
-            checks,
-            "reported_kappa_matches_rows",
-            math.isclose(
-                recomputed_kappa,
-                pack.reliability.reported_categorical_kappa,
-                abs_tol=0.0005,
-            ),
+    _check(
+        checks,
+        "recomputed_categorical_kappa",
+        recomputed_kappa >= thresholds.min_categorical_kappa,
+        recomputed_kappa,
+        thresholds.min_categorical_kappa,
+    )
+    _check(
+        checks,
+        "reported_kappa_matches_rows",
+        math.isclose(
             recomputed_kappa,
             pack.reliability.reported_categorical_kappa,
-        )
+            abs_tol=0.0005,
+        ),
+        recomputed_kappa,
+        pack.reliability.reported_categorical_kappa,
+    )
     _check(checks, "minimum_paired_gain", gain >= thresholds.min_gain, gain, thresholds.min_gain)
     _check(checks, "bootstrap_ci_excludes_zero", ci_lower > 0.0, ci_lower, "> 0")
     _check(
@@ -390,8 +408,10 @@ def evaluate_human_voice_gain(
         passed=not failures,
         clinical_claim_allowed=False,
         held_out_participants=held_out_participants,
+        total_held_out_sessions=len(all_session_keys),
         held_out_sessions=held_out_sessions,
-        paired_axis_observations=observation_count,
+        total_axis_observations=total_observation_count,
+        paired_axis_observations=paired_observation_count,
         intention_to_evaluate_imputations=imputation_count,
         text_only_one_minus_mae=text_accuracy,
         voice_enabled_one_minus_mae=voice_accuracy,

apps/api/app/test_g7_voice_gain_evidence.py

@@ -179,8 +179,34 @@ class G7HumanVoiceGainEvidenceTests(unittest.TestCase):
         self.assertEqual(result.intention_to_evaluate_imputations, 1)
         expected_voice_accuracy = 1.0 - ((1.0 + (0.05 * 5)) / 6)
         self.assertAlmostEqual(result.voice_enabled_one_minus_mae, expected_voice_accuracy)
+        self.assertEqual(result.paired_axis_observations, 5)
+        self.assertEqual(result.held_out_sessions, 1)
+        self.assertIn("production_observation_floor", result.failure_reasons)
+        self.assertIn("production_session_floor", result.failure_reasons)
         self.assertFalse(result.passed)
 
+    def test_baseline_only_missing_cannot_inflate_candidate_gain(self) -> None:
+        complete = G7HumanVoiceGainEvidencePack.model_validate(_valid_payload())
+        complete_result = evaluate_human_voice_gain(
+            complete,
+            thresholds=_test_thresholds(),
+        )
+        payload = _valid_payload()
+        observations = payload["observations"]
+        assert isinstance(observations, list)
+        observations[0]["text_only_status"] = "missing"
+        observations[0]["text_only_score"] = None
+        missing = G7HumanVoiceGainEvidencePack.model_validate(payload)
+
+        missing_result = evaluate_human_voice_gain(
+            missing,
+            thresholds=_test_thresholds(),
+        )
+
+        self.assertLess(missing_result.paired_gain, complete_result.paired_gain)
+        self.assertEqual(missing_result.intention_to_evaluate_imputations, 1)
+        self.assertEqual(missing_result.paired_axis_observations, 5)
+
     def test_duplicate_or_calibration_observation_is_rejected(self) -> None:
         duplicate = _valid_payload()
         duplicate_rows = duplicate["observations"]
@@ -236,6 +262,64 @@ class G7HumanVoiceGainEvidenceTests(unittest.TestCase):
         self.assertFalse(result.passed)
         self.assertIn("reported_icc_matches_rows", result.failure_reasons)
 
+    def test_categorical_kappa_is_mandatory_and_recomputed(self) -> None:
+        missing_report = _valid_payload()
+        reliability = missing_report["reliability"]
+        assert isinstance(reliability, dict)
+        reliability.pop("reported_categorical_kappa")
+        with self.assertRaises(ValidationError):
+            G7HumanVoiceGainEvidencePack.model_validate(missing_report)
+
+        missing_category = _valid_payload()
+        observations = missing_category["observations"]
+        assert isinstance(observations, list)
+        labels = observations[0]["labels"]
+        assert isinstance(labels, list)
+        labels[0].pop("category")
+        with self.assertRaises(ValidationError):
+            G7HumanVoiceGainEvidencePack.model_validate(missing_category)
+
+        mismatched = _valid_payload()
+        reliability = mismatched["reliability"]
+        assert isinstance(reliability, dict)
+        reliability["reported_categorical_kappa"] = 0.8
+        pack = G7HumanVoiceGainEvidencePack.model_validate(mismatched)
+        result = evaluate_human_voice_gain(pack, thresholds=_test_thresholds())
+        self.assertIn("reported_kappa_matches_rows", result.failure_reasons)
+
+        weak = _valid_payload()
+        observations = weak["observations"]
+        assert isinstance(observations, list)
+        for observation in observations:
+            labels = observation["labels"]
+            assert isinstance(labels, list)
+            labels[0]["category"] = "low"
+            labels[1]["category"] = "high"
+        reliability = weak["reliability"]
+        assert isinstance(reliability, dict)
+        reliability["reported_categorical_kappa"] = -1.0
+        weak_pack = G7HumanVoiceGainEvidencePack.model_validate(weak)
+        weak_result = evaluate_human_voice_gain(
+            weak_pack,
+            thresholds=_test_thresholds(),
+        )
+        self.assertIn("recomputed_categorical_kappa", weak_result.failure_reasons)
+
+    def test_analysis_registration_must_strictly_precede_held_out_access(self) -> None:
+        payload = _valid_payload()
+        provenance = payload["provenance"]
+        assert isinstance(provenance, dict)
+        provenance["registered_at"] = provenance["held_out_labels_opened_at"]
+        with self.assertRaisesRegex(ValidationError, "must precede"):
+            G7HumanVoiceGainEvidencePack.model_validate(payload)
+
+        wrong_alpha = _valid_payload()
+        power_plan = wrong_alpha["power_plan"]
+        assert isinstance(power_plan, dict)
+        power_plan["alpha"] = 0.01
+        with self.assertRaises(ValidationError):
+            G7HumanVoiceGainEvidencePack.model_validate(wrong_alpha)
+
     def test_custom_thresholds_require_explicit_test_factory(self) -> None:
         pack = G7HumanVoiceGainEvidencePack.model_validate(_valid_payload())
         with self.assertRaisesRegex(ValueError, "test-only"):