G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
224
scripts/smoke-continuous-improvement-agentic-producer.py
Normal file
224
scripts/smoke-continuous-improvement-agentic-producer.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
"""Run the real G8 scheduled worker against the configured dev engine and DB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps" / "api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
os.chdir(API_ROOT)
|
||||
|
||||
from app import db # noqa: E402
|
||||
from app.engine_client import engine_client # noqa: E402
|
||||
from app.services import continuous_improvement_agentic as agentic # noqa: E402
|
||||
from app.services import continuous_improvement_producer as producer # noqa: E402
|
||||
|
||||
|
||||
class SmokeFailure(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class CountingEngine:
|
||||
def __init__(self) -> None:
|
||||
self.call_count = 0
|
||||
self.stages: list[str] = []
|
||||
self.responses: list[dict[str, str]] = []
|
||||
|
||||
async def generate(self, request: Any) -> Any:
|
||||
self.call_count += 1
|
||||
self.stages.append(str(request.metadata.get("agentic_stage", "unknown")))
|
||||
response = await engine_client.generate(
|
||||
request,
|
||||
timeout=producer.settings.continuous_improvement_producer_engine_timeout_seconds,
|
||||
)
|
||||
self.responses.append(
|
||||
{"provider": str(response.provider), "model": str(response.model)}
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _sha256_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def _read_evidence_row(job_id: Any) -> dict[str, Any]:
|
||||
async with db.acquire(ai_view="research", ai_context=True) as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT j.job_id, j.job_key, j.status, j.attempt_count,
|
||||
j.data_classification, j.source_fingerprint,
|
||||
j.last_error_code, j.last_error_message,
|
||||
j.result_submission_id, j.result_qualification_id,
|
||||
p.pipeline_id, p.generation_model, p.prompt_version,
|
||||
p.prompt_sha256, p.payload_sha256, p.state AS pipeline_state,
|
||||
p.clinical_claim_allowed,
|
||||
q.gate_state, q.catalog_entry_id,
|
||||
b.variant_count, b.variant_pass_rate, b.qualified,
|
||||
(SELECT count(*) FROM app.ci_red_team_review r
|
||||
WHERE r.pipeline_id = p.pipeline_id) AS reviewer_count,
|
||||
(SELECT count(*) FROM app.ci_catalog_entry c
|
||||
WHERE c.qualification_id = q.qualification_id) AS catalog_count,
|
||||
(SELECT count(*) FROM audit.ci_human_approval_event a
|
||||
WHERE a.target_kind = 'content_qualification'
|
||||
AND a.target_id = q.qualification_id) AS approval_count,
|
||||
(SELECT jsonb_agg(jsonb_build_object(
|
||||
'source_id', s.source_id,
|
||||
'version', s.source_version,
|
||||
'content_sha256', s.content_sha256,
|
||||
'provenance_uri', s.provenance_uri,
|
||||
'usage_status', s.usage_status
|
||||
) ORDER BY s.source_id)
|
||||
FROM app.ci_source_artifact s
|
||||
WHERE s.source_record_id = ANY(p.source_record_ids)) AS sources
|
||||
FROM app.ci_agentic_job j
|
||||
LEFT JOIN app.ci_content_pipeline p
|
||||
ON p.submission_id = j.result_submission_id
|
||||
LEFT JOIN app.ci_content_qualification q
|
||||
ON q.qualification_id = j.result_qualification_id
|
||||
LEFT JOIN app.ci_content_benchmark b
|
||||
ON b.pipeline_id = p.pipeline_id
|
||||
WHERE j.job_id = $1
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
if row is None:
|
||||
raise SmokeFailure("scheduled agentic job is not visible")
|
||||
return dict(row)
|
||||
|
||||
|
||||
async def _run(out_path: Path) -> dict[str, Any]:
|
||||
await db.init_pool()
|
||||
await engine_client.startup()
|
||||
try:
|
||||
readiness = await engine_client.health_detail()
|
||||
if not readiness.get("ok"):
|
||||
raise SmokeFailure(f"engine is not ready: {readiness.get('detail')}")
|
||||
|
||||
spec = producer.load_repo_approved_job()
|
||||
job_id = await producer.ensure_repo_approved_job()
|
||||
claimed = await producer.claim_next_agentic_job(job_id=job_id)
|
||||
if claimed is None:
|
||||
raise SmokeFailure("repo-approved job was not available for a fresh model run")
|
||||
|
||||
first_engine = CountingEngine()
|
||||
first = await producer.execute_claimed_agentic_job(
|
||||
claimed,
|
||||
engine=first_engine,
|
||||
)
|
||||
row = await _read_evidence_row(job_id)
|
||||
|
||||
evidence: dict[str, Any] = {
|
||||
"evidence_version": "g8-agentic-producer-live-v1",
|
||||
"environment": "dev",
|
||||
"engine_ready": {
|
||||
"ok": bool(readiness.get("ok")),
|
||||
"detail": str(readiness.get("detail", "")),
|
||||
},
|
||||
"repo_source": {
|
||||
"job_key": spec.job_key,
|
||||
"data_classification": spec.data_classification,
|
||||
"source_ids": [item.artifact.source_id for item in spec.source_packs],
|
||||
"usage_statuses": [
|
||||
item.artifact.usage_status for item in spec.source_packs
|
||||
],
|
||||
"content_hashes_verified": all(
|
||||
_sha256_text(item.content) == item.artifact.content_sha256
|
||||
for item in spec.source_packs
|
||||
),
|
||||
"raw_transcript_included": False,
|
||||
"pii_included": False,
|
||||
},
|
||||
"first_run": {
|
||||
"status": first.status,
|
||||
"engine_call_count": first_engine.call_count,
|
||||
"stages": sorted(first_engine.stages),
|
||||
"provider_models": first_engine.responses,
|
||||
"error_code": first.error_code,
|
||||
},
|
||||
"durable_db": row,
|
||||
"safety_boundary": {
|
||||
"human_approval_required": row.get("gate_state")
|
||||
== "pending_human_approval",
|
||||
"catalog_promoted": int(row.get("catalog_count") or 0) > 0,
|
||||
"human_approval_event_count": int(row.get("approval_count") or 0),
|
||||
"clinical_claim_allowed": bool(row.get("clinical_claim_allowed")),
|
||||
},
|
||||
}
|
||||
|
||||
if first.status == "completed":
|
||||
ids = producer._job_ids(job_id)
|
||||
replay_engine = CountingEngine()
|
||||
async with db.acquire(ai_view="research", ai_context=True) as conn:
|
||||
replay = await agentic.run_agentic_content_pipeline(
|
||||
conn=conn,
|
||||
engine=replay_engine,
|
||||
submission_id=ids["submission"],
|
||||
pipeline_id=ids["pipeline"],
|
||||
benchmark_record_id=ids["benchmark_record"],
|
||||
qualification_id=ids["qualification"],
|
||||
source_packs=spec.source_packs,
|
||||
content_kind=spec.content_kind,
|
||||
difficulty_level=spec.difficulty_level,
|
||||
variant_count=spec.variant_count,
|
||||
prompt_version=spec.prompt_version,
|
||||
trigger_kind=spec.trigger_kind,
|
||||
)
|
||||
evidence["idempotent_replay"] = {
|
||||
"reported": replay.idempotent_replay,
|
||||
"engine_call_count": replay_engine.call_count,
|
||||
"same_submission_id": str(replay.submission_id)
|
||||
== str(row.get("result_submission_id")),
|
||||
"same_qualification_id": str(replay.qualification_id)
|
||||
== str(row.get("result_qualification_id")),
|
||||
}
|
||||
else:
|
||||
evidence["storage_rejected"] = {
|
||||
"pipeline_absent": row.get("pipeline_id") is None,
|
||||
"qualification_absent": row.get("result_qualification_id") is None,
|
||||
"catalog_count": int(row.get("catalog_count") or 0),
|
||||
}
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(
|
||||
json.dumps(evidence, ensure_ascii=False, indent=2, default=str) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return evidence
|
||||
finally:
|
||||
await engine_client.shutdown()
|
||||
await db.close_pool()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="JSON evidence path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
requested_out = args.out
|
||||
out_path = (
|
||||
requested_out
|
||||
if requested_out.is_absolute()
|
||||
else REPO_ROOT / requested_out
|
||||
).resolve()
|
||||
evidence = asyncio.run(_run(out_path))
|
||||
print(json.dumps(evidence, ensure_ascii=False, indent=2, default=str))
|
||||
first_status = evidence["first_run"]["status"]
|
||||
return 0 if first_status == "completed" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue