801 lines
32 KiB
Python
801 lines
32 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
from uuid import uuid4
|
|
|
|
from fastapi import HTTPException
|
|
from pydantic import SecretStr, ValidationError
|
|
|
|
from .contracts.continuous_improvement import (
|
|
ContentBenchmarkQualification,
|
|
ContentSourceArtifact,
|
|
GeneratedContentDraft,
|
|
IndependentRedTeamReview,
|
|
)
|
|
from .routes import continuous_improvement
|
|
from .services import continuous_improvement_store
|
|
|
|
|
|
def _source() -> ContentSourceArtifact:
|
|
return ContentSourceArtifact(
|
|
source_id="oas-g8-source-smoke",
|
|
version="1.0.0",
|
|
content_sha256="a" * 64,
|
|
provenance_uri="repo://synthetic/g8/source",
|
|
usage_status="approved",
|
|
citation_label="synthetic replay source",
|
|
)
|
|
|
|
|
|
def _draft() -> GeneratedContentDraft:
|
|
return GeneratedContentDraft(
|
|
draft_id="oas-g8-draft-smoke",
|
|
content_kind="case",
|
|
source_refs=("oas-g8-source-smoke",),
|
|
generation_model="content-agent",
|
|
prompt_version="1.0.0",
|
|
prompt_sha256="b" * 64,
|
|
payload_sha256="c" * 64,
|
|
synthetic_identity_id="synthetic-identity-smoke",
|
|
difficulty_level=4,
|
|
hidden_answer_fingerprint="d" * 64,
|
|
visible_answer_overlap_tokens=0,
|
|
pii_findings=0,
|
|
unsupported_clinical_claims=0,
|
|
)
|
|
|
|
|
|
def _reviews() -> tuple[IndependentRedTeamReview, IndependentRedTeamReview]:
|
|
return (
|
|
IndependentRedTeamReview(
|
|
review_id="oas-g8-review-smoke-a",
|
|
draft_id="oas-g8-draft-smoke",
|
|
reviewer_agent_id="red-team-a",
|
|
dimensions=("safety", "identity", "pii", "grounding"),
|
|
findings=(),
|
|
reviewed_payload_sha256="c" * 64,
|
|
),
|
|
IndependentRedTeamReview(
|
|
review_id="oas-g8-review-smoke-b",
|
|
draft_id="oas-g8-draft-smoke",
|
|
reviewer_agent_id="red-team-b",
|
|
dimensions=("answer_leakage", "cultural_bias", "difficulty"),
|
|
findings=(),
|
|
reviewed_payload_sha256="c" * 64,
|
|
),
|
|
)
|
|
|
|
|
|
def _benchmark() -> ContentBenchmarkQualification:
|
|
return ContentBenchmarkQualification(
|
|
benchmark_id="oas-g8-benchmark-smoke",
|
|
draft_id="oas-g8-draft-smoke",
|
|
variant_count=8,
|
|
variant_pass_rate=0.875,
|
|
answer_leakage_count=0,
|
|
pii_finding_count=0,
|
|
unsupported_claim_count=0,
|
|
safety_failure_count=0,
|
|
reward_hacking_count=0,
|
|
evidence_refs=("audit://synthetic/g8/benchmark",),
|
|
)
|
|
|
|
|
|
def _artifact() -> dict[str, object]:
|
|
return {
|
|
"artifact_record_id": uuid4(),
|
|
"artifact_id": "artifact",
|
|
"content_sha256": "e" * 64,
|
|
"provenance_uri": "audit://synthetic/g8/artifact",
|
|
}
|
|
|
|
|
|
def _visible_catalog_payload() -> dict[str, object]:
|
|
return {
|
|
"title": "합성 관계 균열 수선 연습",
|
|
"synthetic_profile": "실존 인물과 무관한 합성 내담자",
|
|
"scenario": "주제가 너무 빨리 바뀌어 합성 내담자가 서두른다고 느낀 상황",
|
|
"rupture_or_challenge": "상호작용을 명명하고 내담자의 정정을 초대한다.",
|
|
"learner_task": "영향을 방어하지 않고 인정한 뒤 다음 초점을 공동 결정한다.",
|
|
"success_criteria": ["상호작용 명명", "정정 초대"],
|
|
"source_refs": ["oas-g8-source-repo-synthetic-rupture-v2"],
|
|
"grounded_claims": [
|
|
{
|
|
"claim": "합성 수련 시나리오",
|
|
"source_ref": "oas-g8-source-repo-synthetic-rupture-v2",
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
class ContinuousImprovementPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_agentic_replay_lookup_returns_durable_pipeline_fingerprint(self) -> None:
|
|
pipeline_id = uuid4()
|
|
benchmark_record_id = uuid4()
|
|
qualification_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.return_value = {
|
|
"operation_kind": "content_pipeline",
|
|
"result_id": qualification_id,
|
|
"pipeline_id": pipeline_id,
|
|
"draft_id": f"oas-g8-draft-{pipeline_id.hex}",
|
|
"prompt_sha256": "a" * 64,
|
|
"qualification_id": qualification_id,
|
|
"candidate_catalog_entry_id": f"oas-g8-catalog-{pipeline_id.hex}",
|
|
"benchmark_record_id": benchmark_record_id,
|
|
"benchmark_id": f"oas-g8-benchmark-{pipeline_id.hex}",
|
|
"benchmark_variant_count": 3,
|
|
"red_team_review_count": 2,
|
|
}
|
|
replay = await continuous_improvement_store.find_content_pipeline_submission(
|
|
conn, submission_id=uuid4()
|
|
)
|
|
self.assertEqual(replay["pipeline_id"], pipeline_id)
|
|
self.assertEqual(replay["prompt_sha256"], "a" * 64)
|
|
self.assertEqual(replay["red_team_review_count"], 2)
|
|
|
|
async def test_incident_metadata_can_feed_adversarial_pipeline(self) -> None:
|
|
conn = AsyncMock()
|
|
conn.fetchrow.return_value = {
|
|
"incident_id": "oas-g8-incident-runtime-drift",
|
|
"error_fingerprint": "e" * 64,
|
|
"affected_contract": "evaluation.runtime",
|
|
"evidence_refs": ["audit://incidents/runtime-drift"],
|
|
"pii_included": False,
|
|
}
|
|
incident = await continuous_improvement_store.read_operational_incident(
|
|
conn, incident_record_id=uuid4()
|
|
)
|
|
self.assertEqual(incident.affected_contract, "evaluation.runtime")
|
|
self.assertFalse(incident.pii_included)
|
|
|
|
async def test_admin_projection_is_metadata_only_and_fail_closed(self) -> None:
|
|
conn = AsyncMock()
|
|
conn.fetch.side_effect = [[] for _ in range(9)]
|
|
|
|
result = await continuous_improvement_store.read_continuous_improvement_view(
|
|
conn
|
|
)
|
|
|
|
self.assertEqual(conn.fetch.await_count, 9)
|
|
lifecycle_query = next(
|
|
str(call.args[0])
|
|
for call in conn.fetch.call_args_list
|
|
if "audit.ci_lifecycle_event" in str(call.args[0])
|
|
)
|
|
self.assertIn("approval_event_id", lifecycle_query)
|
|
self.assertIn("artifact_record_id", lifecycle_query)
|
|
self.assertIn("executor_receipt_id", lifecycle_query)
|
|
self.assertIn("executor_evidence_refs", lifecycle_query)
|
|
self.assertFalse(result["silent_auto_promotion_allowed"])
|
|
self.assertFalse(result["raw_transcript_included"])
|
|
self.assertFalse(result["pii_included"])
|
|
self.assertFalse(result["clinical_claim_allowed"])
|
|
self.assertEqual(result["gate_artifacts"], [])
|
|
self.assertEqual(result["incidents"], [])
|
|
self.assertEqual(result["regression_dag_nodes"], [])
|
|
qualification_query = str(conn.fetch.await_args_list[0].args[0])
|
|
self.assertIn("jsonb_build_object", qualification_query)
|
|
self.assertNotIn("hidden_answer", qualification_query)
|
|
|
|
async def test_catalog_projection_reads_only_approved_visible_payloads(self) -> None:
|
|
conn = AsyncMock()
|
|
qualification_id = uuid4()
|
|
conn.fetch.return_value = [
|
|
{
|
|
"catalog_record_id": uuid4(),
|
|
"qualification_id": qualification_id,
|
|
"catalog_entry_id": "oas-g8-catalog-repo-synthetic-rupture-v2",
|
|
"payload_sha256": "a" * 64,
|
|
"status": "approved",
|
|
"clinical_claim_allowed": False,
|
|
"approved_at": "2026-08-07T00:00:00Z",
|
|
"content_kind": "rupture",
|
|
"difficulty_level": 3,
|
|
"synthetic_identity_id": "synthetic-identity-repo-v2",
|
|
"source_provenance_uris": [
|
|
"repo://apps/api/app/data/continuous_improvement/synthetic_source_pack.v2.json"
|
|
],
|
|
"payload": _visible_catalog_payload(),
|
|
}
|
|
]
|
|
|
|
result = await continuous_improvement_store.read_approved_catalog_entries(conn)
|
|
|
|
self.assertEqual(len(result), 1)
|
|
self.assertEqual(result[0]["qualification_id"], qualification_id)
|
|
self.assertNotIn("hidden_answer", result[0]["payload"])
|
|
query = str(conn.fetch.await_args.args[0])
|
|
self.assertIn("WHERE c.status = 'approved'", query)
|
|
self.assertIn("p.draft_payload IS NOT NULL", query)
|
|
self.assertNotIn("hidden_answer", query)
|
|
|
|
async def test_eligible_content_stays_pending_until_human_approval(self) -> None:
|
|
conn = AsyncMock()
|
|
conn.fetchrow.return_value = None
|
|
source_record_id = uuid4()
|
|
with patch.object(
|
|
continuous_improvement_store,
|
|
"_ensure_source",
|
|
AsyncMock(return_value=source_record_id),
|
|
):
|
|
result = await continuous_improvement_store.submit_content_pipeline(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
pipeline_id=uuid4(),
|
|
benchmark_record_id=uuid4(),
|
|
qualification_id=uuid4(),
|
|
draft=_draft(),
|
|
sources=[_source()],
|
|
reviews=list(_reviews()),
|
|
benchmark=_benchmark(),
|
|
)
|
|
self.assertEqual(result["state"], "pending_human_approval")
|
|
self.assertTrue(result["human_approval_required"])
|
|
self.assertFalse(result["catalog_promoted"])
|
|
executed_sql = "\n".join(
|
|
str(call.args[0]) for call in conn.execute.await_args_list
|
|
)
|
|
self.assertNotIn("INSERT INTO app.ci_catalog_entry", executed_sql)
|
|
|
|
async def test_stable_submission_replays_same_result(self) -> None:
|
|
result_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.return_value = {
|
|
"content_hash": "a" * 64,
|
|
"operation_kind": "release_gate",
|
|
"result_id": result_id,
|
|
}
|
|
with patch.object(
|
|
continuous_improvement_store, "_canonical_hash", return_value="a" * 64
|
|
):
|
|
replay = await continuous_improvement_store._begin_submission(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
operation_kind="release_gate",
|
|
result_id=result_id,
|
|
payload={"stable": True},
|
|
)
|
|
self.assertTrue(replay)
|
|
self.assertEqual(conn.execute.await_count, 1)
|
|
|
|
async def test_changed_submission_is_conflict(self) -> None:
|
|
conn = AsyncMock()
|
|
conn.fetchrow.return_value = {
|
|
"content_hash": "b" * 64,
|
|
"operation_kind": "release_gate",
|
|
"result_id": uuid4(),
|
|
}
|
|
with patch.object(
|
|
continuous_improvement_store, "_canonical_hash", return_value="a" * 64
|
|
):
|
|
with self.assertRaises(
|
|
continuous_improvement_store.ContinuousImprovementConflictError
|
|
):
|
|
await continuous_improvement_store._begin_submission(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
operation_kind="release_gate",
|
|
result_id=uuid4(),
|
|
payload={"changed": True},
|
|
)
|
|
|
|
async def test_unconfigured_rollback_stays_requested(self) -> None:
|
|
artifact_record_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.side_effect = [
|
|
None,
|
|
{
|
|
"rollback_artifact_id": artifact_record_id,
|
|
"artifact_id": "oas-g8-model-rollback-baseline",
|
|
"content_sha256": "e" * 64,
|
|
"provenance_uri": "repo://synthetic/g8/model-rollback",
|
|
"subject_id": "oas-g8-model-snapshot-candidate",
|
|
"rollback_target_id": "oas-g8-model-snapshot-baseline",
|
|
},
|
|
]
|
|
|
|
result = await continuous_improvement_store.append_human_approval(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
approval_event_id=uuid4(),
|
|
effect_record_id=uuid4(),
|
|
target_kind="model_change_gate",
|
|
target_id=uuid4(),
|
|
decision="authorize_rollback",
|
|
actor_uid=uuid4(),
|
|
reason_code="approved-by-owner",
|
|
evidence_refs=["audit://synthetic/g8/rollback-approval"],
|
|
)
|
|
|
|
self.assertEqual(result["lifecycle_status"], "requested")
|
|
lifecycle_call = next(
|
|
call
|
|
for call in conn.execute.await_args_list
|
|
if "INSERT INTO audit.ci_lifecycle_event" in str(call.args[0])
|
|
)
|
|
self.assertEqual(lifecycle_call.args[6], "requested")
|
|
self.assertIsNone(lifecycle_call.args[11])
|
|
self.assertIsNone(lifecycle_call.args[12])
|
|
|
|
async def test_model_rollback_is_executed_only_with_matching_receipt(self) -> None:
|
|
artifact_record_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.side_effect = [
|
|
None,
|
|
{
|
|
"rollback_artifact_id": artifact_record_id,
|
|
"artifact_id": "oas-g8-model-rollback-baseline",
|
|
"content_sha256": "e" * 64,
|
|
"provenance_uri": "repo://synthetic/g8/model-rollback",
|
|
"subject_id": "oas-g8-model-snapshot-candidate",
|
|
"rollback_target_id": "oas-g8-model-snapshot-baseline",
|
|
},
|
|
]
|
|
|
|
class RecordingExecutor:
|
|
def __init__(self) -> None:
|
|
self.requests: list[
|
|
continuous_improvement_store.RollbackExecutionRequest
|
|
] = []
|
|
|
|
async def execute(
|
|
self,
|
|
request: continuous_improvement_store.RollbackExecutionRequest,
|
|
) -> continuous_improvement_store.RollbackExecutionReceipt:
|
|
self.requests.append(request)
|
|
return continuous_improvement_store.RollbackExecutionReceipt(
|
|
execution_id="model-rollback-execution-001",
|
|
idempotency_key=request.idempotency_key,
|
|
rollback_scope=request.rollback_scope,
|
|
target_kind=request.target_kind,
|
|
target_id=request.target_id,
|
|
artifact_record_id=request.artifact_record_id,
|
|
artifact_sha256=request.artifact_sha256,
|
|
evidence_refs=("audit://rollback-executor/model/execution-001",),
|
|
)
|
|
|
|
executor = RecordingExecutor()
|
|
effect_record_id = uuid4()
|
|
result = await continuous_improvement_store.append_human_approval(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
approval_event_id=uuid4(),
|
|
effect_record_id=effect_record_id,
|
|
target_kind="model_change_gate",
|
|
target_id=uuid4(),
|
|
decision="authorize_rollback",
|
|
actor_uid=uuid4(),
|
|
reason_code="approved-by-owner",
|
|
evidence_refs=["audit://synthetic/g8/rollback-approval"],
|
|
rollback_executor=executor,
|
|
)
|
|
|
|
self.assertEqual(result["lifecycle_status"], "executed")
|
|
self.assertEqual(len(executor.requests), 1)
|
|
self.assertEqual(executor.requests[0].rollback_scope, "model")
|
|
self.assertEqual(executor.requests[0].idempotency_key, effect_record_id)
|
|
lifecycle_call = next(
|
|
call
|
|
for call in conn.execute.await_args_list
|
|
if "INSERT INTO audit.ci_lifecycle_event" in str(call.args[0])
|
|
)
|
|
self.assertEqual(lifecycle_call.args[6], "executed")
|
|
self.assertEqual(lifecycle_call.args[11], "model-rollback-execution-001")
|
|
self.assertEqual(
|
|
lifecycle_call.args[12],
|
|
["audit://rollback-executor/model/execution-001"],
|
|
)
|
|
|
|
async def test_runtime_rollback_executor_failure_is_recorded_failed(self) -> None:
|
|
artifact_record_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.side_effect = [
|
|
None,
|
|
{
|
|
"rollback_artifact_id": artifact_record_id,
|
|
"artifact_id": "oas-g8-runtime-rollback-baseline",
|
|
"content_sha256": "f" * 64,
|
|
"provenance_uri": "repo://synthetic/g8/runtime-rollback",
|
|
"subject_id": "oas-g8-release-candidate",
|
|
"rollback_target_id": "oas-g8-runtime-rollback-baseline",
|
|
},
|
|
]
|
|
executor = SimpleNamespace(
|
|
execute=AsyncMock(side_effect=RuntimeError("secret-bearing failure"))
|
|
)
|
|
|
|
result = await continuous_improvement_store.append_human_approval(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
approval_event_id=uuid4(),
|
|
effect_record_id=uuid4(),
|
|
target_kind="release_gate",
|
|
target_id=uuid4(),
|
|
decision="authorize_rollback",
|
|
actor_uid=uuid4(),
|
|
reason_code="approved-by-owner",
|
|
evidence_refs=["audit://synthetic/g8/runtime-rollback-approval"],
|
|
rollback_executor=executor,
|
|
)
|
|
|
|
self.assertEqual(result["lifecycle_status"], "failed")
|
|
request = executor.execute.await_args.args[0]
|
|
self.assertEqual(request.rollback_scope, "runtime")
|
|
lifecycle_call = next(
|
|
call
|
|
for call in conn.execute.await_args_list
|
|
if "INSERT INTO audit.ci_lifecycle_event" in str(call.args[0])
|
|
)
|
|
self.assertEqual(lifecycle_call.args[6], "failed")
|
|
self.assertNotIn("secret-bearing", str(lifecycle_call.args))
|
|
|
|
async def test_runtime_rollback_success_requires_runtime_bound_receipt(self) -> None:
|
|
artifact_record_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.side_effect = [
|
|
None,
|
|
{
|
|
"rollback_artifact_id": artifact_record_id,
|
|
"artifact_id": "oas-g8-runtime-rollback-baseline",
|
|
"content_sha256": "f" * 64,
|
|
"provenance_uri": "repo://synthetic/g8/runtime-rollback",
|
|
"subject_id": "oas-g8-release-candidate",
|
|
"rollback_target_id": "oas-g8-runtime-rollback-baseline",
|
|
},
|
|
]
|
|
|
|
async def execute(request):
|
|
return continuous_improvement_store.RollbackExecutionReceipt(
|
|
execution_id="runtime-rollback-execution-001",
|
|
idempotency_key=request.idempotency_key,
|
|
rollback_scope="runtime",
|
|
target_kind="release_gate",
|
|
target_id=request.target_id,
|
|
artifact_record_id=request.artifact_record_id,
|
|
artifact_sha256=request.artifact_sha256,
|
|
evidence_refs=("audit://rollback-executor/runtime/execution-001",),
|
|
)
|
|
|
|
executor = SimpleNamespace(execute=AsyncMock(side_effect=execute))
|
|
result = await continuous_improvement_store.append_human_approval(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
approval_event_id=uuid4(),
|
|
effect_record_id=uuid4(),
|
|
target_kind="release_gate",
|
|
target_id=uuid4(),
|
|
decision="authorize_rollback",
|
|
actor_uid=uuid4(),
|
|
reason_code="approved-by-owner",
|
|
evidence_refs=["audit://synthetic/g8/runtime-rollback-approval"],
|
|
rollback_executor=executor,
|
|
)
|
|
|
|
self.assertEqual(result["lifecycle_status"], "executed")
|
|
request = executor.execute.await_args.args[0]
|
|
self.assertEqual(request.rollback_scope, "runtime")
|
|
self.assertEqual(request.target_kind, "release_gate")
|
|
|
|
async def test_mismatched_success_receipt_is_recorded_failed(self) -> None:
|
|
artifact_record_id = uuid4()
|
|
conn = AsyncMock()
|
|
conn.fetchrow.side_effect = [
|
|
None,
|
|
{
|
|
"rollback_artifact_id": artifact_record_id,
|
|
"artifact_id": "oas-g8-model-rollback-baseline",
|
|
"content_sha256": "e" * 64,
|
|
"provenance_uri": "repo://synthetic/g8/model-rollback",
|
|
"subject_id": "oas-g8-model-snapshot-candidate",
|
|
"rollback_target_id": "oas-g8-model-snapshot-baseline",
|
|
},
|
|
]
|
|
|
|
async def execute(request):
|
|
return continuous_improvement_store.RollbackExecutionReceipt(
|
|
execution_id="mismatched-model-execution-001",
|
|
idempotency_key=request.idempotency_key,
|
|
rollback_scope=request.rollback_scope,
|
|
target_kind=request.target_kind,
|
|
target_id=request.target_id,
|
|
artifact_record_id=request.artifact_record_id,
|
|
artifact_sha256="0" * 64,
|
|
evidence_refs=("audit://rollback-executor/model/mismatch-001",),
|
|
)
|
|
|
|
result = await continuous_improvement_store.append_human_approval(
|
|
conn,
|
|
submission_id=uuid4(),
|
|
approval_event_id=uuid4(),
|
|
effect_record_id=uuid4(),
|
|
target_kind="model_change_gate",
|
|
target_id=uuid4(),
|
|
decision="authorize_rollback",
|
|
actor_uid=uuid4(),
|
|
reason_code="approved-by-owner",
|
|
evidence_refs=["audit://synthetic/g8/model-rollback-approval"],
|
|
rollback_executor=SimpleNamespace(execute=execute),
|
|
)
|
|
|
|
self.assertEqual(result["lifecycle_status"], "failed")
|
|
lifecycle_call = next(
|
|
call
|
|
for call in conn.execute.await_args_list
|
|
if "INSERT INTO audit.ci_lifecycle_event" in str(call.args[0])
|
|
)
|
|
self.assertIsNone(lifecycle_call.args[11])
|
|
self.assertIsNone(lifecycle_call.args[12])
|
|
|
|
|
|
class ContinuousImprovementBoundaryTests(unittest.TestCase):
|
|
def test_lifecycle_view_accepts_honest_rollback_statuses(self) -> None:
|
|
for event_status in ("requested", "failed", "executed"):
|
|
executor_receipt_id = None
|
|
executor_evidence_refs = None
|
|
evidence_refs = ["audit://synthetic/g8/rollback"]
|
|
if event_status == "executed":
|
|
executor_receipt_id = "rollback-execution-001"
|
|
executor_evidence_refs = [
|
|
"audit://rollback-executor/model/execution-001"
|
|
]
|
|
evidence_refs.extend(executor_evidence_refs)
|
|
event = continuous_improvement.LifecycleEventView(
|
|
lifecycle_event_id=uuid4(),
|
|
target_kind="release_gate",
|
|
target_id=uuid4(),
|
|
event_type="rollback",
|
|
event_status=event_status,
|
|
approval_event_id=uuid4(),
|
|
artifact_record_id=uuid4(),
|
|
evidence_refs=evidence_refs,
|
|
executor_receipt_id=executor_receipt_id,
|
|
executor_evidence_refs=executor_evidence_refs,
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
self.assertEqual(event.event_status, event_status)
|
|
|
|
def test_lifecycle_view_rejects_unbound_executed_receipt(self) -> None:
|
|
with self.assertRaisesRegex(ValidationError, "included in lifecycle"):
|
|
continuous_improvement.LifecycleEventView(
|
|
lifecycle_event_id=uuid4(),
|
|
target_kind="model_change_gate",
|
|
target_id=uuid4(),
|
|
event_type="rollback",
|
|
event_status="executed",
|
|
approval_event_id=uuid4(),
|
|
artifact_record_id=uuid4(),
|
|
evidence_refs=["audit://synthetic/g8/rollback"],
|
|
executor_receipt_id="rollback-execution-001",
|
|
executor_evidence_refs=[
|
|
"audit://rollback-executor/model/execution-001"
|
|
],
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
|
|
def test_schema_requires_executed_rollback_before_verification(self) -> None:
|
|
schema_path = (
|
|
Path(__file__).resolve().parents[3]
|
|
/ "infra"
|
|
/ "db"
|
|
/ "init"
|
|
/ "14_continuous_improvement.sql"
|
|
)
|
|
schema = schema_path.read_text(encoding="utf-8")
|
|
self.assertIn("e.event_status = 'executed'", schema)
|
|
self.assertIn("executor_receipt_id", schema)
|
|
self.assertIn("executor_evidence_refs", schema)
|
|
|
|
def test_schema_keeps_reject_and_quarantine_fail_closed_for_every_target(self) -> None:
|
|
schema_path = (
|
|
Path(__file__).resolve().parents[3]
|
|
/ "infra"
|
|
/ "db"
|
|
/ "init"
|
|
/ "14_continuous_improvement.sql"
|
|
)
|
|
schema = schema_path.read_text(encoding="utf-8")
|
|
self.assertIn(
|
|
"IF NEW.decision IN ('reject', 'keep_quarantine') THEN",
|
|
schema,
|
|
)
|
|
self.assertIn("IF NEW.decision <> expected_decision", schema)
|
|
smoke = (
|
|
Path(__file__).resolve().parents[3]
|
|
/ "scripts"
|
|
/ "smoke-continuous-improvement-ledger.sql"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn("'content_qualification'", smoke)
|
|
self.assertIn("'keep_quarantine'", smoke)
|
|
self.assertIn("'release_gate'", smoke)
|
|
self.assertIn("'reject'", smoke)
|
|
self.assertIn("fail-closed human decision created a side effect", smoke)
|
|
|
|
def test_admin_projection_rejects_raw_transcript_claim(self) -> None:
|
|
payload = {
|
|
"content_qualifications": [],
|
|
"model_change_gates": [],
|
|
"release_gates": [],
|
|
"gate_artifacts": [],
|
|
"approvals": [],
|
|
"catalog_entries": [],
|
|
"lifecycle_events": [],
|
|
"incidents": [],
|
|
"regression_dag_nodes": [],
|
|
"data_classification": "synthetic_replay_red_team_coverage_drift",
|
|
"silent_auto_promotion_allowed": False,
|
|
"raw_transcript_included": True,
|
|
"pii_included": False,
|
|
"clinical_claim_allowed": False,
|
|
}
|
|
with self.assertRaises(ValidationError):
|
|
continuous_improvement.ContinuousImprovementViewResponse.model_validate(
|
|
payload
|
|
)
|
|
|
|
def test_gate_artifacts_are_all_mandatory(self) -> None:
|
|
artifact = _artifact()
|
|
with self.assertRaises(ValidationError):
|
|
continuous_improvement.CompleteGateArtifacts.model_validate(
|
|
{
|
|
"baseline": artifact,
|
|
"threshold": _artifact(),
|
|
"provenance": [_artifact()],
|
|
}
|
|
)
|
|
|
|
def test_empty_provenance_is_rejected(self) -> None:
|
|
with self.assertRaises(ValidationError):
|
|
continuous_improvement.CompleteGateArtifacts(
|
|
baseline=continuous_improvement.GateArtifact.model_validate(
|
|
_artifact()
|
|
),
|
|
threshold=continuous_improvement.GateArtifact.model_validate(
|
|
_artifact()
|
|
),
|
|
provenance=[],
|
|
rollback=continuous_improvement.GateArtifact.model_validate(
|
|
_artifact()
|
|
),
|
|
)
|
|
|
|
def test_non_synthetic_input_classification_is_rejected(self) -> None:
|
|
payload = {
|
|
"submission_id": str(uuid4()),
|
|
"incident_record_id": str(uuid4()),
|
|
"data_classification": "production_transcript",
|
|
"incident": {
|
|
"incident_id": "oas-g8-incident-smoke",
|
|
"error_fingerprint": "f" * 64,
|
|
"affected_contract": "synthetic.replay",
|
|
"evidence_refs": ["audit://synthetic/g8/incident"],
|
|
},
|
|
}
|
|
with self.assertRaises(ValidationError):
|
|
continuous_improvement.IncidentDagRequest.model_validate(payload)
|
|
|
|
def test_conflict_maps_to_http_409(self) -> None:
|
|
with self.assertRaises(HTTPException) as captured:
|
|
continuous_improvement._raise_store_error(
|
|
continuous_improvement_store.ContinuousImprovementConflictError(
|
|
"changed content"
|
|
)
|
|
)
|
|
self.assertEqual(captured.exception.status_code, 409)
|
|
|
|
def test_qualification_response_cannot_claim_automatic_promotion(self) -> None:
|
|
response = continuous_improvement.ContentPipelineResponse(
|
|
submission_id=uuid4(),
|
|
pipeline_id=uuid4(),
|
|
qualification_id=uuid4(),
|
|
candidate_catalog_entry_id="oas-g8-catalog-smoke",
|
|
state="pending_human_approval",
|
|
human_approval_required=True,
|
|
catalog_promoted=False,
|
|
idempotent_replay=False,
|
|
)
|
|
self.assertTrue(response.human_approval_required)
|
|
self.assertFalse(response.catalog_promoted)
|
|
|
|
def test_catalog_consumer_rejects_hidden_answer_or_raw_transcript(self) -> None:
|
|
payload = _visible_catalog_payload()
|
|
payload["hidden_answer"] = "노출되면 안 되는 정답"
|
|
payload["raw_transcript"] = "원문"
|
|
with self.assertRaises(ValidationError):
|
|
continuous_improvement.CatalogVisiblePayload.model_validate(payload)
|
|
|
|
def test_catalog_consumer_contract_is_fail_closed(self) -> None:
|
|
entry = {
|
|
"catalog_record_id": str(uuid4()),
|
|
"qualification_id": str(uuid4()),
|
|
"catalog_entry_id": "oas-g8-catalog-repo-synthetic-rupture-v2",
|
|
"payload_sha256": "a" * 64,
|
|
"content_kind": "rupture",
|
|
"difficulty_level": 3,
|
|
"synthetic_identity_id": "synthetic-identity-repo-v2",
|
|
"source_provenance_uris": [
|
|
"repo://apps/api/app/data/continuous_improvement/synthetic_source_pack.v2.json"
|
|
],
|
|
"payload": _visible_catalog_payload(),
|
|
"status": "approved",
|
|
"clinical_claim_allowed": False,
|
|
"approved_at": "2026-08-07T00:00:00Z",
|
|
}
|
|
response = continuous_improvement.ApprovedCatalogConsumerResponse(
|
|
entries=[continuous_improvement.ApprovedCatalogConsumerEntry.model_validate(entry)],
|
|
data_classification="synthetic_replay_red_team_coverage_drift",
|
|
)
|
|
self.assertTrue(response.human_approval_required)
|
|
self.assertFalse(response.raw_transcript_included)
|
|
self.assertFalse(response.pii_included)
|
|
self.assertFalse(response.clinical_claim_allowed)
|
|
|
|
|
|
class ContinuousImprovementAuthenticationTests(unittest.IsolatedAsyncioTestCase):
|
|
TOKEN = "g8-continuous-improvement-token-at-least-32-characters"
|
|
|
|
async def _assert_rejected(
|
|
self, configured: str | None, presented: str | None, status_code: int
|
|
) -> None:
|
|
reached = False
|
|
|
|
async def fake_provider():
|
|
nonlocal reached
|
|
reached = True
|
|
yield AsyncMock()
|
|
|
|
settings = SimpleNamespace(
|
|
continuous_improvement_internal_token=SecretStr(configured or "")
|
|
)
|
|
with (
|
|
patch.object(
|
|
continuous_improvement, "_research_db_provider", fake_provider
|
|
),
|
|
):
|
|
dependency = continuous_improvement.continuous_improvement_internal_db(
|
|
settings=settings, presented_token=presented
|
|
)
|
|
with self.assertRaises(HTTPException) as captured:
|
|
await anext(dependency)
|
|
self.assertEqual(captured.exception.status_code, status_code)
|
|
self.assertFalse(reached)
|
|
|
|
async def test_unconfigured_token_is_503_before_db(self) -> None:
|
|
await self._assert_rejected(None, None, 503)
|
|
|
|
async def test_short_token_is_503_before_db(self) -> None:
|
|
await self._assert_rejected("short", None, 503)
|
|
|
|
async def test_missing_header_is_401_before_db(self) -> None:
|
|
await self._assert_rejected(self.TOKEN, None, 401)
|
|
|
|
async def test_wrong_header_is_403_before_db(self) -> None:
|
|
await self._assert_rejected(self.TOKEN, "wrong", 403)
|
|
|
|
async def test_valid_header_reaches_research_view_provider(self) -> None:
|
|
conn = AsyncMock()
|
|
|
|
async def fake_provider():
|
|
yield conn
|
|
|
|
with (
|
|
patch.object(
|
|
continuous_improvement, "_research_db_provider", fake_provider
|
|
),
|
|
):
|
|
dependency = continuous_improvement.continuous_improvement_internal_db(
|
|
settings=SimpleNamespace(
|
|
continuous_improvement_internal_token=SecretStr(self.TOKEN)
|
|
),
|
|
presented_token=self.TOKEN,
|
|
)
|
|
self.assertIs(await anext(dependency), conn)
|
|
await dependency.aclose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|