주기 실회기 검증과 G7 종료계약 보강
This commit is contained in:
Yun Chan
parent
83590e9ef7
commit
7b4955c3fc
23 changed files with 2916 additions and 117 deletions
|
|
@ -51,7 +51,9 @@ class VoiceGainEvidenceResult(BaseModel):
|
|||
passed: bool
|
||||
clinical_claim_allowed: Literal[False] = False
|
||||
held_out_participants: int
|
||||
total_held_out_sessions: int
|
||||
held_out_sessions: int
|
||||
total_axis_observations: int
|
||||
paired_axis_observations: int
|
||||
intention_to_evaluate_imputations: int
|
||||
text_only_one_minus_mae: float
|
||||
|
|
@ -62,7 +64,7 @@ class VoiceGainEvidenceResult(BaseModel):
|
|||
confidence_level: float
|
||||
bootstrap_samples: int
|
||||
recomputed_icc: float
|
||||
recomputed_categorical_kappa: float | None
|
||||
recomputed_categorical_kappa: float
|
||||
checks: tuple[VoiceGainEvidenceCheck, ...]
|
||||
failure_reasons: tuple[str, ...]
|
||||
|
||||
|
|
@ -221,22 +223,23 @@ def evaluate_human_voice_gain(
|
|||
labels_by_key = {item.labeler_key: item for item in observation.labels}
|
||||
ordered_labels = [labels_by_key[labeler_key] for labeler_key in panel]
|
||||
ratings.append([item.score for item in ordered_labels])
|
||||
if ordered_labels[0].category is not None:
|
||||
categories.append([str(item.category) for item in ordered_labels])
|
||||
categories.append([item.category for item in ordered_labels])
|
||||
reference = sum(item.score for item in ordered_labels) / len(ordered_labels)
|
||||
|
||||
if observation.text_only_status == "observed":
|
||||
text_observed = observation.text_only_status == "observed"
|
||||
voice_observed = observation.voice_enabled_status == "observed"
|
||||
if text_observed and voice_observed:
|
||||
assert observation.text_only_score is not None
|
||||
text_error = abs(observation.text_only_score - reference)
|
||||
else:
|
||||
text_error = 1.0
|
||||
imputation_count += 1
|
||||
if observation.voice_enabled_status == "observed":
|
||||
assert observation.voice_enabled_score is not None
|
||||
voice_error = abs(observation.voice_enabled_score - reference)
|
||||
else:
|
||||
# 한 조건만 결측이어도 두 조건을 모두 최대 오류로 대치한다. baseline-only
|
||||
# 결측이 candidate gain을 인위적으로 키우는 비대칭을 차단하면서도
|
||||
# intention-to-evaluate 행은 분석에서 유지한다.
|
||||
text_error = 1.0
|
||||
voice_error = 1.0
|
||||
imputation_count += 1
|
||||
imputation_count += int(not text_observed) + int(not voice_observed)
|
||||
errors_by_participant[observation.participant_key].append(
|
||||
(text_error, voice_error)
|
||||
)
|
||||
|
|
@ -258,11 +261,28 @@ def evaluate_human_voice_gain(
|
|||
seed=thresholds.seed,
|
||||
)
|
||||
recomputed_icc = _icc_absolute_agreement_single(ratings)
|
||||
recomputed_kappa = _fleiss_kappa(categories) if categories else None
|
||||
recomputed_kappa = _fleiss_kappa(categories)
|
||||
|
||||
held_out_participants = len(errors_by_participant)
|
||||
held_out_sessions = len({item.session_key for item in pack.observations})
|
||||
observation_count = len(pack.observations)
|
||||
all_session_keys = {item.session_key for item in pack.observations}
|
||||
complete_axes_by_session: dict[str, set[str]] = defaultdict(set)
|
||||
for observation in pack.observations:
|
||||
if (
|
||||
observation.text_only_status == "observed"
|
||||
and observation.voice_enabled_status == "observed"
|
||||
):
|
||||
complete_axes_by_session[observation.session_key].add(observation.axis)
|
||||
required_axes = {"goal", "task", "bond"}
|
||||
held_out_sessions = sum(
|
||||
axes == required_axes for axes in complete_axes_by_session.values()
|
||||
)
|
||||
total_observation_count = len(pack.observations)
|
||||
paired_observation_count = sum(
|
||||
1
|
||||
for observation in pack.observations
|
||||
if observation.text_only_status == "observed"
|
||||
and observation.voice_enabled_status == "observed"
|
||||
)
|
||||
checks: list[VoiceGainEvidenceCheck] = []
|
||||
_check(
|
||||
checks,
|
||||
|
|
@ -281,8 +301,8 @@ def evaluate_human_voice_gain(
|
|||
_check(
|
||||
checks,
|
||||
"production_observation_floor",
|
||||
observation_count >= thresholds.min_paired_axis_observations,
|
||||
observation_count,
|
||||
paired_observation_count >= thresholds.min_paired_axis_observations,
|
||||
paired_observation_count,
|
||||
thresholds.min_paired_axis_observations,
|
||||
)
|
||||
_check(
|
||||
|
|
@ -318,8 +338,8 @@ def evaluate_human_voice_gain(
|
|||
_check(
|
||||
checks,
|
||||
"power_plan_observations_achieved",
|
||||
observation_count >= pack.power_plan.required_paired_axis_observations,
|
||||
observation_count,
|
||||
paired_observation_count >= pack.power_plan.required_paired_axis_observations,
|
||||
paired_observation_count,
|
||||
pack.power_plan.required_paired_axis_observations,
|
||||
)
|
||||
_check(
|
||||
|
|
@ -355,26 +375,24 @@ def evaluate_human_voice_gain(
|
|||
recomputed_icc,
|
||||
pack.reliability.reported_icc,
|
||||
)
|
||||
if recomputed_kappa is not None:
|
||||
assert pack.reliability.reported_categorical_kappa is not None
|
||||
_check(
|
||||
checks,
|
||||
"recomputed_categorical_kappa",
|
||||
recomputed_kappa >= thresholds.min_categorical_kappa,
|
||||
recomputed_kappa,
|
||||
thresholds.min_categorical_kappa,
|
||||
)
|
||||
_check(
|
||||
checks,
|
||||
"reported_kappa_matches_rows",
|
||||
math.isclose(
|
||||
recomputed_kappa,
|
||||
pack.reliability.reported_categorical_kappa,
|
||||
abs_tol=0.0005,
|
||||
),
|
||||
_check(
|
||||
checks,
|
||||
"recomputed_categorical_kappa",
|
||||
recomputed_kappa >= thresholds.min_categorical_kappa,
|
||||
recomputed_kappa,
|
||||
thresholds.min_categorical_kappa,
|
||||
)
|
||||
_check(
|
||||
checks,
|
||||
"reported_kappa_matches_rows",
|
||||
math.isclose(
|
||||
recomputed_kappa,
|
||||
pack.reliability.reported_categorical_kappa,
|
||||
)
|
||||
abs_tol=0.0005,
|
||||
),
|
||||
recomputed_kappa,
|
||||
pack.reliability.reported_categorical_kappa,
|
||||
)
|
||||
_check(checks, "minimum_paired_gain", gain >= thresholds.min_gain, gain, thresholds.min_gain)
|
||||
_check(checks, "bootstrap_ci_excludes_zero", ci_lower > 0.0, ci_lower, "> 0")
|
||||
_check(
|
||||
|
|
@ -390,8 +408,10 @@ def evaluate_human_voice_gain(
|
|||
passed=not failures,
|
||||
clinical_claim_allowed=False,
|
||||
held_out_participants=held_out_participants,
|
||||
total_held_out_sessions=len(all_session_keys),
|
||||
held_out_sessions=held_out_sessions,
|
||||
paired_axis_observations=observation_count,
|
||||
total_axis_observations=total_observation_count,
|
||||
paired_axis_observations=paired_observation_count,
|
||||
intention_to_evaluate_imputations=imputation_count,
|
||||
text_only_one_minus_mae=text_accuracy,
|
||||
voice_enabled_one_minus_mae=voice_accuracy,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue