G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
637
apps/api/app/test_continuous_improvement_agentic.py
Normal file
637
apps/api/app/test_continuous_improvement_agentic.py
Normal file
|
|
@ -0,0 +1,637 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import Settings
|
||||
from .contracts.continuous_improvement import ContentSourceArtifact, OperationalIncident
|
||||
from .contracts.engine_gateway import GenerateResponse
|
||||
from .routes import continuous_improvement
|
||||
from .services import continuous_improvement_agentic as agentic
|
||||
from .services import continuous_improvement_store
|
||||
from .services.guardrail import mask_synthetic_generated_pii
|
||||
|
||||
|
||||
SOURCE_CONTENT = (
|
||||
"Approved synthetic training guidance: acknowledge mismatch, ask one open question, "
|
||||
"and avoid diagnosis or claims of treatment efficacy."
|
||||
)
|
||||
|
||||
|
||||
def _sha(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _source_pack() -> agentic.AgenticSourcePack:
|
||||
return agentic.AgenticSourcePack(
|
||||
artifact=ContentSourceArtifact(
|
||||
source_id="oas-g8-source-agentic-test",
|
||||
version="1.0.0",
|
||||
content_sha256=_sha(SOURCE_CONTENT),
|
||||
provenance_uri="repo://synthetic/g8/agentic-test",
|
||||
usage_status="approved",
|
||||
citation_label="agentic synthetic test source",
|
||||
),
|
||||
content=SOURCE_CONTENT,
|
||||
)
|
||||
|
||||
|
||||
def _draft_payload() -> dict[str, object]:
|
||||
return {
|
||||
"title": "Repair practice",
|
||||
"synthetic_profile": "Synthetic learner-facing persona without identity",
|
||||
"scenario": "The client says the counselor misunderstood the concern.",
|
||||
"rupture_or_challenge": "Acknowledge the mismatch before asking a new question.",
|
||||
"learner_task": "Respond with one grounded repair turn.",
|
||||
"success_criteria": ["acknowledge mismatch", "ask an open question"],
|
||||
"hidden_answer": "hidden_anchor_zeta",
|
||||
"source_refs": ["oas-g8-source-agentic-test"],
|
||||
"grounded_claims": [
|
||||
{
|
||||
"claim": "Repair practice should acknowledge a mismatch.",
|
||||
"source_ref": "oas-g8-source-agentic-test",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _variant(index: int) -> dict[str, object]:
|
||||
return {
|
||||
"visible_scenario": f"Variant {index}: the relationship context changes.",
|
||||
"learner_task": "Offer a concise repair response.",
|
||||
"success_criteria": ["acknowledge mismatch", "invite correction"],
|
||||
"hidden_answer": f"variant_secret_{index}",
|
||||
"source_refs": ["oas-g8-source-agentic-test"],
|
||||
}
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
def __init__(self, overrides: dict[str, object] | None = None) -> None:
|
||||
self.requests = []
|
||||
self.overrides = overrides or {}
|
||||
self.override_offsets: dict[str, int] = {}
|
||||
|
||||
def _payload(self, stage: str) -> object:
|
||||
if stage in self.overrides:
|
||||
override = self.overrides[stage]
|
||||
if isinstance(override, list):
|
||||
offset = self.override_offsets.get(stage, 0)
|
||||
self.override_offsets[stage] = offset + 1
|
||||
if offset >= len(override):
|
||||
raise AssertionError(f"override sequence exhausted: {stage}")
|
||||
return override[offset]
|
||||
return override
|
||||
if stage == "draft_generation":
|
||||
return _draft_payload()
|
||||
if stage == "red_team_safety-identity":
|
||||
return {
|
||||
"dimensions": ["safety", "identity", "pii", "grounding"],
|
||||
"findings": [],
|
||||
}
|
||||
if stage == "red_team_leakage-bias":
|
||||
return {
|
||||
"dimensions": ["answer_leakage", "cultural_bias", "difficulty"],
|
||||
"findings": [],
|
||||
}
|
||||
if stage == "benchmark_variant_generation":
|
||||
return {"variants": [_variant(1), _variant(2), _variant(3)]}
|
||||
if stage.startswith("benchmark_judge_"):
|
||||
return {
|
||||
"passed": True,
|
||||
"answer_leakage": False,
|
||||
"pii_found": False,
|
||||
"unsupported_claim": False,
|
||||
"safety_failure": False,
|
||||
"reward_hacking": False,
|
||||
"evidence_summary": "all fail-closed gates passed",
|
||||
}
|
||||
raise AssertionError(f"unexpected stage: {stage}")
|
||||
|
||||
async def generate(self, req):
|
||||
self.requests.append(req)
|
||||
stage = str(req.metadata["agentic_stage"])
|
||||
payload = self._payload(stage)
|
||||
if isinstance(payload, Exception):
|
||||
raise payload
|
||||
if payload is None:
|
||||
return GenerateResponse(
|
||||
text="", model="fake-model", provider="fake-provider", structured=None
|
||||
)
|
||||
return GenerateResponse(
|
||||
text="",
|
||||
model="fake-model",
|
||||
provider="fake-provider",
|
||||
structured=payload,
|
||||
)
|
||||
|
||||
|
||||
class AgenticContinuousImprovementTests(unittest.IsolatedAsyncioTestCase):
|
||||
def test_answer_overlap_requires_marker_or_repeated_phrase(self) -> None:
|
||||
self.assertEqual(
|
||||
agentic._answer_overlap_count(
|
||||
"상담자는 영향을 인정하고 다음 초점을 함께 정한다.",
|
||||
{"scenario": "상담자는 화제를 바꾼 영향을 살핀다."},
|
||||
),
|
||||
0,
|
||||
)
|
||||
self.assertGreater(
|
||||
agentic._answer_overlap_count(
|
||||
"hidden_anchor_zeta",
|
||||
{"scenario": "visible hidden_anchor_zeta"},
|
||||
),
|
||||
0,
|
||||
)
|
||||
self.assertGreater(
|
||||
agentic._answer_overlap_count(
|
||||
"invite a correction before choosing the next focus together",
|
||||
{"scenario": "Please invite a correction before choosing the next step."},
|
||||
),
|
||||
0,
|
||||
)
|
||||
|
||||
def test_generated_pii_gate_avoids_contextless_korean_false_positive(self) -> None:
|
||||
ordinary = mask_synthetic_generated_pii(
|
||||
"fictional client가 서운함을 느끼고 learner가 상호작용을 고쳐나간다."
|
||||
)
|
||||
explicit_name = mask_synthetic_generated_pii("내담자 김서연은 말을 멈췄다.")
|
||||
explicit_phone = mask_synthetic_generated_pii("연락처는 010-1234-5678입니다.")
|
||||
|
||||
self.assertNotIn("NAME", ordinary.entities)
|
||||
self.assertIn("NAME", explicit_name.entities)
|
||||
self.assertIn("PHONE", explicit_phone.entities)
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.submission_id = uuid4()
|
||||
self.pipeline_id = uuid4()
|
||||
self.benchmark_record_id = uuid4()
|
||||
self.qualification_id = uuid4()
|
||||
self.conn = AsyncMock()
|
||||
|
||||
def _stored(self) -> dict[str, object]:
|
||||
return {
|
||||
"submission_id": self.submission_id,
|
||||
"pipeline_id": self.pipeline_id,
|
||||
"qualification_id": self.qualification_id,
|
||||
"candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}",
|
||||
"state": "pending_human_approval",
|
||||
"human_approval_required": True,
|
||||
"catalog_promoted": False,
|
||||
"idempotent_replay": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
async def _run(self, engine: FakeEngine):
|
||||
return await agentic.run_agentic_content_pipeline(
|
||||
conn=self.conn,
|
||||
engine=engine,
|
||||
submission_id=self.submission_id,
|
||||
pipeline_id=self.pipeline_id,
|
||||
benchmark_record_id=self.benchmark_record_id,
|
||||
qualification_id=self.qualification_id,
|
||||
source_packs=[_source_pack()],
|
||||
content_kind="case",
|
||||
difficulty_level=4,
|
||||
variant_count=3,
|
||||
)
|
||||
|
||||
async def test_calls_independent_agents_and_stores_only_pending_candidate(self) -> None:
|
||||
engine = FakeEngine()
|
||||
submit = AsyncMock(return_value=self._stored())
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
result = await self._run(engine)
|
||||
|
||||
self.assertEqual(result.agent_calls_executed, 7)
|
||||
self.assertEqual(result.red_team_review_count, 2)
|
||||
self.assertEqual(result.benchmark_variant_count, 3)
|
||||
self.assertEqual(result.state, "pending_human_approval")
|
||||
self.assertTrue(result.human_approval_required)
|
||||
self.assertFalse(result.catalog_promoted)
|
||||
self.assertFalse(result.clinical_claim_allowed)
|
||||
|
||||
stages = [str(req.metadata["agentic_stage"]) for req in engine.requests]
|
||||
self.assertEqual(stages.count("draft_generation"), 1)
|
||||
review_requests = [
|
||||
req
|
||||
for req in engine.requests
|
||||
if str(req.metadata["agentic_stage"]).startswith("red_team_")
|
||||
]
|
||||
self.assertEqual(len(review_requests), 2)
|
||||
self.assertEqual(
|
||||
len({str(req.metadata["agent_id"]) for req in review_requests}), 2
|
||||
)
|
||||
judge_requests = [
|
||||
req
|
||||
for req in engine.requests
|
||||
if str(req.metadata["agentic_stage"]).startswith("benchmark_judge_")
|
||||
]
|
||||
self.assertEqual(len(judge_requests), 3)
|
||||
self.assertEqual(len({str(req.metadata["agent_id"]) for req in judge_requests}), 3)
|
||||
|
||||
kwargs = submit.await_args.kwargs
|
||||
draft = kwargs["draft"]
|
||||
reviews = kwargs["reviews"]
|
||||
benchmark = kwargs["benchmark"]
|
||||
persisted_payload = kwargs["draft_payload"]
|
||||
self.assertEqual(draft.generation_model, "fake-provider/fake-model")
|
||||
self.assertEqual(draft.source_refs, ("oas-g8-source-agentic-test",))
|
||||
self.assertEqual(len(draft.prompt_sha256), 64)
|
||||
self.assertEqual(len(draft.payload_sha256), 64)
|
||||
self.assertEqual(
|
||||
draft.payload_sha256,
|
||||
agentic._sha256_text(agentic._canonical_json(persisted_payload)),
|
||||
)
|
||||
self.assertEqual(persisted_payload["hidden_answer"], "hidden_anchor_zeta")
|
||||
self.assertTrue(
|
||||
all(item.reviewed_payload_sha256 == draft.payload_sha256 for item in reviews)
|
||||
)
|
||||
self.assertEqual(len({item.reviewer_agent_id for item in reviews}), 2)
|
||||
self.assertTrue(benchmark.qualified)
|
||||
self.assertEqual(benchmark.variant_pass_rate, 1.0)
|
||||
|
||||
async def test_sequential_replay_skips_every_model_call(self) -> None:
|
||||
source_context = agentic._source_context([_source_pack()])
|
||||
prompt = agentic._generation_prompt_payload(
|
||||
source_context=source_context,
|
||||
content_kind="case",
|
||||
difficulty_level=4,
|
||||
prompt_version="1.0.0",
|
||||
trigger_kind="source_pack",
|
||||
)
|
||||
replay = {
|
||||
"pipeline_id": self.pipeline_id,
|
||||
"benchmark_record_id": self.benchmark_record_id,
|
||||
"qualification_id": self.qualification_id,
|
||||
"prompt_sha256": agentic._sha256_text(agentic._canonical_json(prompt)),
|
||||
"candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}",
|
||||
"draft_id": f"oas-g8-draft-{self.pipeline_id.hex}",
|
||||
"benchmark_id": f"oas-g8-benchmark-{self.pipeline_id.hex}",
|
||||
"red_team_review_count": 2,
|
||||
"benchmark_variant_count": 3,
|
||||
}
|
||||
engine = FakeEngine()
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=replay),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
result = await self._run(engine)
|
||||
self.assertTrue(result.idempotent_replay)
|
||||
self.assertEqual(result.agent_calls_executed, 0)
|
||||
self.assertEqual(engine.requests, [])
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_model_failure_never_reaches_store(self) -> None:
|
||||
engine = FakeEngine({"draft_generation": None})
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
agentic.AgenticPipelineExecutionError, "no structured output"
|
||||
):
|
||||
await self._run(engine)
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_invalid_structured_output_gets_one_schema_repair_attempt(self) -> None:
|
||||
engine = FakeEngine(
|
||||
{"draft_generation": [{"title": "incomplete"}, _draft_payload()]}
|
||||
)
|
||||
submit = AsyncMock(return_value=self._stored())
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
result = await self._run(engine)
|
||||
|
||||
draft_requests = [
|
||||
request
|
||||
for request in engine.requests
|
||||
if request.metadata["agentic_stage"] == "draft_generation"
|
||||
]
|
||||
self.assertEqual(result.agent_calls_executed, 8)
|
||||
self.assertEqual(len(draft_requests), 2)
|
||||
self.assertEqual(draft_requests[0].metadata["structured_attempt"], 1)
|
||||
self.assertFalse(draft_requests[0].metadata["structured_repair"])
|
||||
self.assertEqual(draft_requests[1].metadata["structured_attempt"], 2)
|
||||
self.assertTrue(draft_requests[1].metadata["structured_repair"])
|
||||
self.assertEqual(draft_requests[1].temperature, 0.0)
|
||||
self.assertEqual(
|
||||
draft_requests[1].structured_schema, draft_requests[0].structured_schema
|
||||
)
|
||||
self.assertIn("JSON schema", draft_requests[1].messages[-1].content)
|
||||
|
||||
async def test_repeated_invalid_structured_output_fails_closed(self) -> None:
|
||||
engine = FakeEngine(
|
||||
{
|
||||
"draft_generation": [
|
||||
{"title": "still incomplete"},
|
||||
{"title": "still incomplete after repair"},
|
||||
]
|
||||
}
|
||||
)
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
agentic.AgenticPipelineExecutionError,
|
||||
"invalid structured output",
|
||||
):
|
||||
await self._run(engine)
|
||||
|
||||
self.assertEqual(len(engine.requests), 2)
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_pii_and_answer_leakage_each_fail_before_red_team(self) -> None:
|
||||
cases = {
|
||||
"pii": {**_draft_payload(), "scenario": "Call 010-1234-5678."},
|
||||
"leakage": {
|
||||
**_draft_payload(),
|
||||
"scenario": "The visible answer is hidden_anchor_zeta.",
|
||||
},
|
||||
}
|
||||
for name, payload in cases.items():
|
||||
with self.subTest(name=name):
|
||||
engine = FakeEngine({"draft_generation": payload})
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"submit_content_pipeline",
|
||||
submit,
|
||||
),
|
||||
):
|
||||
with self.assertRaises(agentic.AgenticPipelineRejectedError):
|
||||
await self._run(engine)
|
||||
self.assertEqual(len(engine.requests), 1)
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_unsupported_claim_from_independent_red_team_blocks_storage(self) -> None:
|
||||
unsafe_review = {
|
||||
"dimensions": ["safety", "identity", "pii", "grounding"],
|
||||
"findings": [
|
||||
{
|
||||
"dimension": "grounding",
|
||||
"severity": "high",
|
||||
"evidence_summary": "clinical efficacy claim has no source support",
|
||||
}
|
||||
],
|
||||
}
|
||||
engine = FakeEngine({"red_team_safety-identity": unsafe_review})
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
agentic.AgenticPipelineRejectedError, "red-team"
|
||||
):
|
||||
await self._run(engine)
|
||||
self.assertEqual(
|
||||
len(
|
||||
[
|
||||
req
|
||||
for req in engine.requests
|
||||
if str(req.metadata["agentic_stage"]).startswith("red_team_")
|
||||
]
|
||||
),
|
||||
2,
|
||||
)
|
||||
self.assertFalse(
|
||||
any(
|
||||
str(req.metadata["agentic_stage"]).startswith("benchmark_")
|
||||
for req in engine.requests
|
||||
)
|
||||
)
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_failed_variant_judge_blocks_storage(self) -> None:
|
||||
failed = {
|
||||
"passed": False,
|
||||
"answer_leakage": False,
|
||||
"pii_found": False,
|
||||
"unsupported_claim": False,
|
||||
"safety_failure": False,
|
||||
"reward_hacking": True,
|
||||
"evidence_summary": "variant rewards memorized wording",
|
||||
}
|
||||
engine = FakeEngine({"benchmark_judge_2": failed})
|
||||
submit = AsyncMock()
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=None),
|
||||
),
|
||||
patch.object(
|
||||
continuous_improvement_store, "submit_content_pipeline", submit
|
||||
),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
agentic.AgenticPipelineRejectedError, "qualification failed"
|
||||
):
|
||||
await self._run(engine)
|
||||
submit.assert_not_awaited()
|
||||
|
||||
async def test_changed_replay_input_conflicts_without_model_call(self) -> None:
|
||||
replay = {
|
||||
"pipeline_id": self.pipeline_id,
|
||||
"benchmark_record_id": self.benchmark_record_id,
|
||||
"qualification_id": self.qualification_id,
|
||||
"prompt_sha256": "f" * 64,
|
||||
"candidate_catalog_entry_id": f"oas-g8-catalog-{self.pipeline_id.hex}",
|
||||
"draft_id": f"oas-g8-draft-{self.pipeline_id.hex}",
|
||||
"benchmark_id": f"oas-g8-benchmark-{self.pipeline_id.hex}",
|
||||
"red_team_review_count": 2,
|
||||
"benchmark_variant_count": 3,
|
||||
}
|
||||
engine = FakeEngine()
|
||||
with patch.object(
|
||||
continuous_improvement_store,
|
||||
"find_content_pipeline_submission",
|
||||
AsyncMock(return_value=replay),
|
||||
):
|
||||
with self.assertRaises(
|
||||
continuous_improvement_store.ContinuousImprovementConflictError
|
||||
):
|
||||
await self._run(engine)
|
||||
self.assertEqual(engine.requests, [])
|
||||
|
||||
async def test_operational_incident_becomes_adversarial_source_and_pipeline(self) -> None:
|
||||
incident_record_id = uuid4()
|
||||
incident = OperationalIncident(
|
||||
incident_id="oas-g8-incident-runtime-drift",
|
||||
error_fingerprint="e" * 64,
|
||||
affected_contract="evaluation.runtime",
|
||||
evidence_refs=("audit://incidents/runtime-drift",),
|
||||
)
|
||||
source_pack = agentic.source_pack_from_operational_incident(incident)
|
||||
self.assertEqual(source_pack.artifact.usage_status, "approved")
|
||||
self.assertEqual(source_pack.artifact.content_sha256, _sha(source_pack.content))
|
||||
self.assertIn("runtime-drift", source_pack.content)
|
||||
self.assertFalse(incident.pii_included)
|
||||
|
||||
result = agentic.AgenticPipelineResult(
|
||||
**self._stored(),
|
||||
draft_id=f"oas-g8-draft-{self.pipeline_id.hex}",
|
||||
benchmark_id=f"oas-g8-benchmark-{self.pipeline_id.hex}",
|
||||
red_team_review_count=2,
|
||||
benchmark_variant_count=3,
|
||||
agent_calls_executed=7,
|
||||
trigger_kind="operational_incident",
|
||||
)
|
||||
request = continuous_improvement.IncidentAdversarialPipelineRequest(
|
||||
submission_id=self.submission_id,
|
||||
pipeline_id=self.pipeline_id,
|
||||
benchmark_record_id=self.benchmark_record_id,
|
||||
qualification_id=self.qualification_id,
|
||||
data_classification="synthetic_replay_red_team_coverage_drift",
|
||||
)
|
||||
runner = AsyncMock(return_value=result)
|
||||
with (
|
||||
patch.object(
|
||||
continuous_improvement_store,
|
||||
"read_operational_incident",
|
||||
AsyncMock(return_value=incident),
|
||||
),
|
||||
patch.object(agentic, "run_agentic_content_pipeline", runner),
|
||||
):
|
||||
response = (
|
||||
await continuous_improvement.create_incident_adversarial_content_pipeline(
|
||||
incident_record_id=incident_record_id,
|
||||
request=request,
|
||||
conn=self.conn,
|
||||
)
|
||||
)
|
||||
self.assertEqual(response.trigger_kind, "operational_incident")
|
||||
self.assertEqual(runner.await_args.kwargs["content_kind"], "benchmark")
|
||||
generated_source = runner.await_args.kwargs["source_packs"][0]
|
||||
self.assertEqual(generated_source.artifact.content_sha256, _sha(generated_source.content))
|
||||
|
||||
|
||||
class RollbackExecutorAdapterTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_http_executor_posts_pinned_command_and_returns_receipt(self) -> None:
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content)
|
||||
seen.append(payload)
|
||||
self.assertEqual(
|
||||
request.headers[agentic.HttpRollbackExecutor.TOKEN_HEADER],
|
||||
"rollback-control-token-with-at-least-32-characters",
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"schema_version": "oas.rollback-executor.v1",
|
||||
"status": "executed",
|
||||
"execution_id": "model-control-plane-execution-001",
|
||||
"idempotency_key": payload["idempotency_key"],
|
||||
"rollback_scope": payload["rollback_scope"],
|
||||
"target_kind": payload["target_kind"],
|
||||
"target_id": payload["target_id"],
|
||||
"artifact_record_id": payload["artifact_record_id"],
|
||||
"artifact_sha256": payload["artifact_sha256"],
|
||||
"evidence_refs": [
|
||||
"audit://rollback-executor/model/control-plane-execution-001"
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
request = continuous_improvement_store.RollbackExecutionRequest(
|
||||
idempotency_key=uuid4(),
|
||||
approval_event_id=uuid4(),
|
||||
rollback_scope="model",
|
||||
target_kind="model_change_gate",
|
||||
target_id=uuid4(),
|
||||
subject_id="oas-g8-model-snapshot-candidate",
|
||||
rollback_target_id="oas-g8-model-snapshot-baseline",
|
||||
artifact_record_id=uuid4(),
|
||||
artifact_id="oas-g8-model-rollback-baseline",
|
||||
artifact_sha256="a" * 64,
|
||||
artifact_provenance_uri="repo://synthetic/g8/model-rollback",
|
||||
authorization_evidence_refs=("audit://synthetic/g8/approval",),
|
||||
)
|
||||
executor = agentic.HttpRollbackExecutor(
|
||||
endpoint="http://127.0.0.1:8099/internal/rollback",
|
||||
token="rollback-control-token-with-at-least-32-characters",
|
||||
timeout_seconds=5,
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
|
||||
receipt = await executor.execute(request)
|
||||
|
||||
self.assertEqual(receipt.status, "executed")
|
||||
self.assertEqual(receipt.artifact_sha256, request.artifact_sha256)
|
||||
self.assertEqual(len(seen), 1)
|
||||
self.assertNotIn("actor_uid", seen[0])
|
||||
|
||||
def test_rollback_executor_is_disabled_by_default(self) -> None:
|
||||
settings = Settings(_env_file=None)
|
||||
|
||||
self.assertFalse(settings.continuous_improvement_rollback_executor_enabled)
|
||||
self.assertIsNone(agentic.build_configured_rollback_executor(settings))
|
||||
|
||||
def test_enabled_rollback_executor_requires_endpoint_and_secret(self) -> None:
|
||||
with self.assertRaises(ValueError) as captured:
|
||||
Settings(
|
||||
_env_file=None,
|
||||
continuous_improvement_rollback_executor_enabled=True,
|
||||
)
|
||||
|
||||
self.assertIn("rollback executor", str(captured.exception))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue