주기 실회기 검증과 G7 종료계약 보강
This commit is contained in:
parent
83590e9ef7
commit
7b4955c3fc
23 changed files with 2916 additions and 117 deletions
|
|
@ -44,6 +44,7 @@ def human_pack() -> dict[str, object]:
|
|||
)
|
||||
observations = []
|
||||
axes = ("goal", "task", "bond")
|
||||
categories = ("low", "medium", "high")
|
||||
for session_index in range(50):
|
||||
participant = f"held-{session_index % 30:03d}"
|
||||
for axis_index, axis in enumerate(axes):
|
||||
|
|
@ -60,8 +61,16 @@ def human_pack() -> dict[str, object]:
|
|||
"voice_enabled_status": "observed",
|
||||
"voice_enabled_score": target,
|
||||
"labels": [
|
||||
{"labeler_key": "labeler-001", "score": target},
|
||||
{"labeler_key": "labeler-002", "score": target},
|
||||
{
|
||||
"labeler_key": "labeler-001",
|
||||
"score": target,
|
||||
"category": categories[axis_index],
|
||||
},
|
||||
{
|
||||
"labeler_key": "labeler-002",
|
||||
"score": target,
|
||||
"category": categories[axis_index],
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
|
@ -108,6 +117,7 @@ def human_pack() -> dict[str, object]:
|
|||
"reliability": {
|
||||
"labeler_keys": ["labeler-001", "labeler-002"],
|
||||
"reported_icc": 1.0,
|
||||
"reported_categorical_kappa": 1.0,
|
||||
"report_sha256": _sha(112),
|
||||
},
|
||||
"observations": observations,
|
||||
|
|
@ -432,6 +442,25 @@ class G7ExternalProofTests(unittest.TestCase):
|
|||
self.assertEqual([], errors)
|
||||
self.assertTrue(gain["passed"])
|
||||
self.assertGreaterEqual(gain["held_out_participants"], 30)
|
||||
self.assertGreaterEqual(gain["recomputed_categorical_kappa"], 0.70)
|
||||
|
||||
def test_human_pack_cannot_bypass_the_required_categorical_kappa(self) -> None:
|
||||
errors: list[str] = []
|
||||
payload = human_pack()
|
||||
reliability = payload["reliability"]
|
||||
assert isinstance(reliability, dict)
|
||||
reliability.pop("reported_categorical_kappa")
|
||||
observations = payload["observations"]
|
||||
assert isinstance(observations, list)
|
||||
for observation in observations:
|
||||
labels = observation["labels"]
|
||||
for label in labels:
|
||||
label.pop("category")
|
||||
|
||||
gain = self.checker.validate_human_gain(payload, errors)
|
||||
|
||||
self.assertEqual({}, gain)
|
||||
self.assertIn("human_gain:invalid:ValidationError", errors)
|
||||
|
||||
def test_windows_host_topology_passes_without_weakening_compose(self) -> None:
|
||||
errors: list[str] = []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue