G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
171
scripts/run-alliance-calibration.py
Normal file
171
scripts/run-alliance-calibration.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Run the Alliance agents on the versioned G0 gold scene pack.
|
||||
|
||||
The runner writes a raw, provenance-rich snapshot only. Use
|
||||
``compare-alliance-calibration.py`` to evaluate it against another version.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps" / "api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.contracts.measurement import ( # noqa: E402
|
||||
ALLIANCE_DIMENSIONS,
|
||||
BenchmarkCase,
|
||||
)
|
||||
from app.engine_client import engine_client # noqa: E402
|
||||
from app.services.alliance_calibration import ( # noqa: E402
|
||||
AllianceCalibrationPrediction,
|
||||
AllianceCalibrationSnapshot,
|
||||
)
|
||||
from app.services.alliance_measurement import ( # noqa: E402
|
||||
TranscriptTurn,
|
||||
run_agent_assessment,
|
||||
)
|
||||
|
||||
|
||||
def _needed_perspectives(case: BenchmarkCase) -> tuple[str, ...]:
|
||||
perspectives = {
|
||||
expectation.perspective
|
||||
for expectation in case.expected
|
||||
if expectation.construct_key == "working_alliance"
|
||||
and expectation.dimension in ALLIANCE_DIMENSIONS
|
||||
and expectation.perspective
|
||||
in {"client_agent_report", "independent_observer"}
|
||||
}
|
||||
return tuple(sorted(perspectives))
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> int:
|
||||
payload = json.loads(args.benchmark.read_text(encoding="utf-8"))
|
||||
cases = tuple(BenchmarkCase.model_validate(item) for item in payload["cases"])
|
||||
predictions: list[AllianceCalibrationPrediction] = []
|
||||
run_models: set[str] = set()
|
||||
run_providers: set[str] = set()
|
||||
prompt_versions: set[str] = set()
|
||||
await engine_client.startup()
|
||||
try:
|
||||
for case in cases:
|
||||
turns = tuple(
|
||||
TranscriptTurn(
|
||||
turn_id=uuid5(NAMESPACE_URL, f"{case.case_id}:turn:{index}"),
|
||||
seq=index + 1,
|
||||
speaker=turn.speaker,
|
||||
text=turn.text,
|
||||
)
|
||||
for index, turn in enumerate(case.turns)
|
||||
)
|
||||
for perspective in _needed_perspectives(case):
|
||||
session_id = uuid5(
|
||||
NAMESPACE_URL,
|
||||
f"{args.run_id}:{case.case_id}:session",
|
||||
)
|
||||
pulse_id = uuid5(
|
||||
NAMESPACE_URL,
|
||||
f"{args.run_id}:{case.case_id}:{perspective}:pulse",
|
||||
)
|
||||
result = await run_agent_assessment(
|
||||
pulse_id=pulse_id,
|
||||
session_id=session_id,
|
||||
checkpoint="post",
|
||||
perspective=perspective,
|
||||
turns=turns,
|
||||
engine=engine_client,
|
||||
)
|
||||
run_models.add(result.model_run.model)
|
||||
run_providers.add(result.model_run.provider)
|
||||
prompt_versions.add(result.model_run.prompt_bundle_version)
|
||||
assessment = result.assessment.by_dimension() if result.assessment else {}
|
||||
for dimension in ALLIANCE_DIMENSIONS:
|
||||
item = assessment.get(dimension)
|
||||
predictions.append(
|
||||
AllianceCalibrationPrediction(
|
||||
case_id=case.case_id,
|
||||
perspective=perspective,
|
||||
dimension=dimension,
|
||||
value=item.score if item is not None else None,
|
||||
confidence=item.confidence if item is not None else None,
|
||||
evidence_turn_indices=(
|
||||
item.evidence_turn_indices if item is not None else ()
|
||||
),
|
||||
status=result.measurement_status,
|
||||
error_code=result.error_code,
|
||||
provider=result.model_run.provider,
|
||||
model=result.model_run.model,
|
||||
prompt_bundle_version=result.model_run.prompt_bundle_version,
|
||||
model_run_id=str(result.model_run.model_run_id),
|
||||
attempt_count=len(result.all_model_runs),
|
||||
prior_model_run_ids=tuple(
|
||||
str(run.model_run_id)
|
||||
for run in result.prior_model_runs
|
||||
),
|
||||
prior_error_codes=tuple(
|
||||
run.error_code or "unknown_error"
|
||||
for run in result.prior_model_runs
|
||||
),
|
||||
prompt_bundle_hash=result.model_run.prompt_bundle_hash,
|
||||
input_evidence_hash=result.model_run.input_evidence_hash,
|
||||
)
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"case_id": case.case_id,
|
||||
"perspective": perspective,
|
||||
"status": result.measurement_status,
|
||||
"provider": result.model_run.provider,
|
||||
"model": result.model_run.model,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
await engine_client.shutdown()
|
||||
|
||||
snapshot = AllianceCalibrationSnapshot(
|
||||
run_id=args.run_id,
|
||||
provider=next(iter(run_providers)) if len(run_providers) == 1 else "mixed",
|
||||
model=next(iter(run_models)) if len(run_models) == 1 else "mixed",
|
||||
prompt_bundle_version=(
|
||||
next(iter(prompt_versions)) if len(prompt_versions) == 1 else "mixed"
|
||||
),
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
predictions=tuple(predictions),
|
||||
)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(
|
||||
json.dumps(snapshot.model_dump(mode="json"), ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(str(args.out))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument(
|
||||
"--benchmark",
|
||||
type=Path,
|
||||
default=API_ROOT / "app" / "data" / "outcome_alliance_benchmark_g0.v1.json",
|
||||
)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
return asyncio.run(_run(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue