from __future__ import annotations import hashlib import json import unittest from unittest.mock import AsyncMock, patch from uuid import uuid4 import httpx from .config import Settings from .contracts.continuous_improvement import ContentSourceArtifact, OperationalIncident from .contracts.engine_gateway import GenerateResponse from .routes import continuous_improvement from .services import continuous_improvement_agentic as agentic from .services import continuous_improvement_store from .services.guardrail import mask_synthetic_generated_pii SOURCE_CONTENT = ( "Approved synthetic training guidance: acknowledge mismatch, ask one open question, " "and avoid diagnosis or claims of treatment efficacy." ) def _sha(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() def _source_pack() -> agentic.AgenticSourcePack: return agentic.AgenticSourcePack( artifact=ContentSourceArtifact( source_id="oas-g8-source-agentic-test", version="1.0.0", content_sha256=_sha(SOURCE_CONTENT), provenance_uri="repo://synthetic/g8/agentic-test", usage_status="approved", citation_label="agentic synthetic test source", ), content=SOURCE_CONTENT, ) def _draft_payload() -> dict[str, object]: return { "title": "Repair practice", "synthetic_profile": "Synthetic learner-facing persona without identity", "scenario": "The client says the counselor misunderstood the concern.", "rupture_or_challenge": "Acknowledge the mismatch before asking a new question.", "learner_task": "Respond with one grounded repair turn.", "success_criteria": ["acknowledge mismatch", "ask an open question"], "hidden_answer": "hidden_anchor_zeta", "source_refs": ["oas-g8-source-agentic-test"], "grounded_claims": [ { "claim": "Repair practice should acknowledge a mismatch.", "source_ref": "oas-g8-source-agentic-test", } ], } def _variant(index: int) -> dict[str, object]: return { "visible_scenario": f"Variant {index}: the relationship context changes.", "learner_task": "Offer a concise repair response.", "success_criteria": ["acknowledge mismatch", "invite correction"], "hidden_answer": f"variant_secret_{index}", "source_refs": ["oas-g8-source-agentic-test"], } class FakeEngine: def __init__(self, overrides: dict[str, object] | None = None) -> None: self.requests = [] self.overrides = overrides or {} self.override_offsets: dict[str, int] = {} def _payload(self, stage: str) -> object: if stage in self.overrides: override = self.overrides[stage] if isinstance(override, list): offset = self.override_offsets.get(stage, 0) self.override_offsets[stage] = offset + 1 if offset >= len(override): raise AssertionError(f"override sequence exhausted: {stage}") return override[offset] return override if stage == "draft_generation": return _draft_payload() if stage == "red_team_safety-identity": return { "dimensions": ["safety", "identity", "pii", "grounding"], "findings": [], } if stage == "red_team_leakage-bias": return { "dimensions": ["answer_leakage", "cultural_bias", "difficulty"], "findings": [], } if stage == "benchmark_variant_generation": return {"variants": [_variant(1), _variant(2), _variant(3)]} if stage.startswith("benchmark_judge_"): return { "passed": True, "answer_leakage": False, "pii_found": False, "unsupported_claim": False, "safety_failure": False, "reward_hacking": False, "evidence_summary": "all fail-closed gates passed", } raise AssertionError(f"unexpected stage: {stage}") async def generate(self, req): self.requests.append(req) stage = str(req.metadata["agentic_stage"]) payload = self._payload(stage) if isinstance(payload, Exception): raise payload if payload is None: return GenerateResponse( text="", model="fake-model", provider="fake-provider", structured=None ) return GenerateResponse( text="", model="fake-model", provider="fake-provider", structured=payload, ) class AgenticContinuousImprovementTests(unittest.IsolatedAsyncioTestCase): def test_answer_overlap_requires_marker_or_repeated_phrase(self) -> None: self.assertEqual( agentic._answer_overlap_count( "상담자는 영향을 인정하고 다음 초점을 함께 정한다.", {"scenario": "상담자는 화제를 바꾼 영향을 살핀다."}, ), 0, ) self.assertGreater( agentic._answer_overlap_count( "hidden_anchor_zeta", {"scenario": "visible hidden_anchor_zeta"}, ), 0, ) self.assertGreater( agentic._answer_overlap_count( "invite a correction before choosing the next focus together", {"scenario": "Please invite a correction before choosing the next step."}, ), 0, ) def test_generated_pii_gate_avoids_contextless_korean_false_positive(self) -> None: ordinary = mask_synthetic_generated_pii( "fictional client가 서운함을 느끼고 learner가 상호작용을 고쳐나간다." ) explicit_name = mask_synthetic_generated_pii("내담자 김서연은 말을 멈췄다.") explicit_phone = mask_synthetic_generated_pii("연락처는 010-1234-5678입니다.") self.assertNotIn("NAME", ordinary.entities) self.assertIn("NAME", explicit_name.entities) self.assertIn("PHONE", explicit_phone.entities) def setUp(self) -> None: self.submission_id = uuid4() self.pipeline_id = uuid4() self.benchmark_record_id = uuid4() self.qualification_id = uuid4() self.conn = AsyncMock() def _stored(self) -> dict[str, object]: return { "submission_id": self.submission_id, "pipeline_id": self.pipeline_id, "qualification_id": self.qualification_id, "candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}", "state": "pending_human_approval", "human_approval_required": True, "catalog_promoted": False, "idempotent_replay": False, "clinical_claim_allowed": False, } async def _run(self, engine: FakeEngine): return await agentic.run_agentic_content_pipeline( conn=self.conn, engine=engine, submission_id=self.submission_id, pipeline_id=self.pipeline_id, benchmark_record_id=self.benchmark_record_id, qualification_id=self.qualification_id, source_packs=[_source_pack()], content_kind="case", difficulty_level=4, variant_count=3, ) async def test_calls_independent_agents_and_stores_only_pending_candidate(self) -> None: engine = FakeEngine() submit = AsyncMock(return_value=self._stored()) with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): result = await self._run(engine) self.assertEqual(result.agent_calls_executed, 7) self.assertEqual(result.red_team_review_count, 2) self.assertEqual(result.benchmark_variant_count, 3) self.assertEqual(result.state, "pending_human_approval") self.assertTrue(result.human_approval_required) self.assertFalse(result.catalog_promoted) self.assertFalse(result.clinical_claim_allowed) stages = [str(req.metadata["agentic_stage"]) for req in engine.requests] self.assertEqual(stages.count("draft_generation"), 1) review_requests = [ req for req in engine.requests if str(req.metadata["agentic_stage"]).startswith("red_team_") ] self.assertEqual(len(review_requests), 2) self.assertEqual( len({str(req.metadata["agent_id"]) for req in review_requests}), 2 ) judge_requests = [ req for req in engine.requests if str(req.metadata["agentic_stage"]).startswith("benchmark_judge_") ] self.assertEqual(len(judge_requests), 3) self.assertEqual(len({str(req.metadata["agent_id"]) for req in judge_requests}), 3) kwargs = submit.await_args.kwargs draft = kwargs["draft"] reviews = kwargs["reviews"] benchmark = kwargs["benchmark"] persisted_payload = kwargs["draft_payload"] self.assertEqual(draft.generation_model, "fake-provider/fake-model") self.assertEqual(draft.source_refs, ("oas-g8-source-agentic-test",)) self.assertEqual(len(draft.prompt_sha256), 64) self.assertEqual(len(draft.payload_sha256), 64) self.assertEqual( draft.payload_sha256, agentic._sha256_text(agentic._canonical_json(persisted_payload)), ) self.assertEqual(persisted_payload["hidden_answer"], "hidden_anchor_zeta") self.assertTrue( all(item.reviewed_payload_sha256 == draft.payload_sha256 for item in reviews) ) self.assertEqual(len({item.reviewer_agent_id for item in reviews}), 2) self.assertTrue(benchmark.qualified) self.assertEqual(benchmark.variant_pass_rate, 1.0) async def test_sequential_replay_skips_every_model_call(self) -> None: source_context = agentic._source_context([_source_pack()]) prompt = agentic._generation_prompt_payload( source_context=source_context, content_kind="case", difficulty_level=4, prompt_version="1.0.0", trigger_kind="source_pack", ) replay = { "pipeline_id": self.pipeline_id, "benchmark_record_id": self.benchmark_record_id, "qualification_id": self.qualification_id, "prompt_sha256": agentic._sha256_text(agentic._canonical_json(prompt)), "candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}", "draft_id": f"oas-g8-draft-{self.pipeline_id.hex}", "benchmark_id": f"oas-g8-benchmark-{self.pipeline_id.hex}", "red_team_review_count": 2, "benchmark_variant_count": 3, } engine = FakeEngine() submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=replay), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): result = await self._run(engine) self.assertTrue(result.idempotent_replay) self.assertEqual(result.agent_calls_executed, 0) self.assertEqual(engine.requests, []) submit.assert_not_awaited() async def test_model_failure_never_reaches_store(self) -> None: engine = FakeEngine({"draft_generation": None}) submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): with self.assertRaisesRegex( agentic.AgenticPipelineExecutionError, "no structured output" ): await self._run(engine) submit.assert_not_awaited() async def test_invalid_structured_output_gets_one_schema_repair_attempt(self) -> None: engine = FakeEngine( {"draft_generation": [{"title": "incomplete"}, _draft_payload()]} ) submit = AsyncMock(return_value=self._stored()) with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): result = await self._run(engine) draft_requests = [ request for request in engine.requests if request.metadata["agentic_stage"] == "draft_generation" ] self.assertEqual(result.agent_calls_executed, 8) self.assertEqual(len(draft_requests), 2) self.assertEqual(draft_requests[0].metadata["structured_attempt"], 1) self.assertFalse(draft_requests[0].metadata["structured_repair"]) self.assertEqual(draft_requests[1].metadata["structured_attempt"], 2) self.assertTrue(draft_requests[1].metadata["structured_repair"]) self.assertEqual(draft_requests[1].temperature, 0.0) self.assertEqual( draft_requests[1].structured_schema, draft_requests[0].structured_schema ) self.assertIn("JSON schema", draft_requests[1].messages[-1].content) async def test_repeated_invalid_structured_output_fails_closed(self) -> None: engine = FakeEngine( { "draft_generation": [ {"title": "still incomplete"}, {"title": "still incomplete after repair"}, ] } ) submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): with self.assertRaisesRegex( agentic.AgenticPipelineExecutionError, "invalid structured output", ): await self._run(engine) self.assertEqual(len(engine.requests), 2) submit.assert_not_awaited() async def test_pii_and_answer_leakage_each_fail_before_red_team(self) -> None: cases = { "pii": {**_draft_payload(), "scenario": "Call 010-1234-5678."}, "leakage": { **_draft_payload(), "scenario": "The visible answer is hidden_anchor_zeta.", }, } for name, payload in cases.items(): with self.subTest(name=name): engine = FakeEngine({"draft_generation": payload}) submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit, ), ): with self.assertRaises(agentic.AgenticPipelineRejectedError): await self._run(engine) self.assertEqual(len(engine.requests), 1) submit.assert_not_awaited() async def test_unsupported_claim_from_independent_red_team_blocks_storage(self) -> None: unsafe_review = { "dimensions": ["safety", "identity", "pii", "grounding"], "findings": [ { "dimension": "grounding", "severity": "high", "evidence_summary": "clinical efficacy claim has no source support", } ], } engine = FakeEngine({"red_team_safety-identity": unsafe_review}) submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): with self.assertRaisesRegex( agentic.AgenticPipelineRejectedError, "red-team" ): await self._run(engine) self.assertEqual( len( [ req for req in engine.requests if str(req.metadata["agentic_stage"]).startswith("red_team_") ] ), 2, ) self.assertFalse( any( str(req.metadata["agentic_stage"]).startswith("benchmark_") for req in engine.requests ) ) submit.assert_not_awaited() async def test_failed_variant_judge_blocks_storage(self) -> None: failed = { "passed": False, "answer_leakage": False, "pii_found": False, "unsupported_claim": False, "safety_failure": False, "reward_hacking": True, "evidence_summary": "variant rewards memorized wording", } engine = FakeEngine({"benchmark_judge_2": failed}) submit = AsyncMock() with ( patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=None), ), patch.object( continuous_improvement_store, "submit_content_pipeline", submit ), ): with self.assertRaisesRegex( agentic.AgenticPipelineRejectedError, "qualification failed" ): await self._run(engine) submit.assert_not_awaited() async def test_changed_replay_input_conflicts_without_model_call(self) -> None: replay = { "pipeline_id": self.pipeline_id, "benchmark_record_id": self.benchmark_record_id, "qualification_id": self.qualification_id, "prompt_sha256": "f" * 64, "candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}", "draft_id": f"oas-g8-draft-{self.pipeline_id.hex}", "benchmark_id": f"oas-g8-benchmark-{self.pipeline_id.hex}", "red_team_review_count": 2, "benchmark_variant_count": 3, } engine = FakeEngine() with patch.object( continuous_improvement_store, "find_content_pipeline_submission", AsyncMock(return_value=replay), ): with self.assertRaises( continuous_improvement_store.ContinuousImprovementConflictError ): await self._run(engine) self.assertEqual(engine.requests, []) async def test_operational_incident_becomes_adversarial_source_and_pipeline(self) -> None: incident_record_id = uuid4() incident = OperationalIncident( incident_id="oas-g8-incident-runtime-drift", error_fingerprint="e" * 64, affected_contract="evaluation.runtime", evidence_refs=("audit://incidents/runtime-drift",), ) source_pack = agentic.source_pack_from_operational_incident(incident) self.assertEqual(source_pack.artifact.usage_status, "approved") self.assertEqual(source_pack.artifact.content_sha256, _sha(source_pack.content)) self.assertIn("runtime-drift", source_pack.content) self.assertFalse(incident.pii_included) result = agentic.AgenticPipelineResult( **self._stored(), draft_id=f"oas-g8-draft-{self.pipeline_id.hex}", benchmark_id=f"oas-g8-benchmark-{self.pipeline_id.hex}", red_team_review_count=2, benchmark_variant_count=3, agent_calls_executed=7, trigger_kind="operational_incident", ) request = continuous_improvement.IncidentAdversarialPipelineRequest( submission_id=self.submission_id, pipeline_id=self.pipeline_id, benchmark_record_id=self.benchmark_record_id, qualification_id=self.qualification_id, data_classification="synthetic_replay_red_team_coverage_drift", ) runner = AsyncMock(return_value=result) with ( patch.object( continuous_improvement_store, "read_operational_incident", AsyncMock(return_value=incident), ), patch.object(agentic, "run_agentic_content_pipeline", runner), ): response = ( await continuous_improvement.create_incident_adversarial_content_pipeline( incident_record_id=incident_record_id, request=request, conn=self.conn, ) ) self.assertEqual(response.trigger_kind, "operational_incident") self.assertEqual(runner.await_args.kwargs["content_kind"], "benchmark") generated_source = runner.await_args.kwargs["source_packs"][0] self.assertEqual(generated_source.artifact.content_sha256, _sha(generated_source.content)) class RollbackExecutorAdapterTests(unittest.IsolatedAsyncioTestCase): async def test_http_executor_posts_pinned_command_and_returns_receipt(self) -> None: seen: list[dict[str, object]] = [] def handler(request: httpx.Request) -> httpx.Response: payload = json.loads(request.content) seen.append(payload) self.assertEqual( request.headers[agentic.HttpRollbackExecutor.TOKEN_HEADER], "rollback-control-token-with-at-least-32-characters", ) return httpx.Response( 200, json={ "schema_version": "oas.rollback-executor.v1", "status": "executed", "execution_id": "model-control-plane-execution-001", "idempotency_key": payload["idempotency_key"], "rollback_scope": payload["rollback_scope"], "target_kind": payload["target_kind"], "target_id": payload["target_id"], "artifact_record_id": payload["artifact_record_id"], "artifact_sha256": payload["artifact_sha256"], "evidence_refs": [ "audit://rollback-executor/model/control-plane-execution-001" ], }, ) request = continuous_improvement_store.RollbackExecutionRequest( idempotency_key=uuid4(), approval_event_id=uuid4(), rollback_scope="model", target_kind="model_change_gate", target_id=uuid4(), subject_id="oas-g8-model-snapshot-candidate", rollback_target_id="oas-g8-model-snapshot-baseline", artifact_record_id=uuid4(), artifact_id="oas-g8-model-rollback-baseline", artifact_sha256="a" * 64, artifact_provenance_uri="repo://synthetic/g8/model-rollback", authorization_evidence_refs=("audit://synthetic/g8/approval",), ) executor = agentic.HttpRollbackExecutor( endpoint="http://127.0.0.1:8099/internal/rollback", token="rollback-control-token-with-at-least-32-characters", timeout_seconds=5, transport=httpx.MockTransport(handler), ) receipt = await executor.execute(request) self.assertEqual(receipt.status, "executed") self.assertEqual(receipt.artifact_sha256, request.artifact_sha256) self.assertEqual(len(seen), 1) self.assertNotIn("actor_uid", seen[0]) def test_rollback_executor_is_disabled_by_default(self) -> None: settings = Settings(_env_file=None) self.assertFalse(settings.continuous_improvement_rollback_executor_enabled) self.assertIsNone(agentic.build_configured_rollback_executor(settings)) def test_enabled_rollback_executor_requires_endpoint_and_secret(self) -> None: with self.assertRaises(ValueError) as captured: Settings( _env_file=None, continuous_improvement_rollback_executor_enabled=True, ) self.assertIn("rollback executor", str(captured.exception)) if __name__ == "__main__": unittest.main()