G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
317
scripts/smoke-supervision-version-comparison-producer.py
Normal file
317
scripts/smoke-supervision-version-comparison-producer.py
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
"""Prove the scheduled G6 repo benchmark comparison on a real dev PostgreSQL.
|
||||
|
||||
The bootstrap creates metadata-only synthetic measurement anchors through the
|
||||
owner connection inside the local dev DB container. The actual comparison is
|
||||
then written twice through the runtime app role to prove first append,
|
||||
idempotent replay, hydrated subgroup drift, and zero duplicate rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps" / "api"
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from app.services import ( # noqa: E402
|
||||
supervision_research_store,
|
||||
supervision_research_version_evaluator as evaluator,
|
||||
)
|
||||
|
||||
|
||||
FIXTURE_NAMESPACE = UUID("46838644-191a-42c6-9737-dc1b9fbe6e63")
|
||||
FIXTURE_COHORT = "g6-repo-benchmark-v1"
|
||||
FIXTURE_LEARNER_ID = uuid5(FIXTURE_NAMESPACE, "learner")
|
||||
FIXTURE_SESSION_ID = uuid5(FIXTURE_NAMESPACE, "session")
|
||||
FIXTURE_INSTRUMENT_ID = "g6-repo-version-comparison"
|
||||
FIXTURE_INSTRUMENT_VERSION = "1.0.0"
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _parse_env(path: Path) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for raw in path.read_text(encoding="utf-8-sig").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip('"').strip("'")
|
||||
return values
|
||||
|
||||
|
||||
def _sql_literal(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _bootstrap_sql() -> str:
|
||||
benchmark = evaluator.load_validated_repository_benchmark()
|
||||
statements = [
|
||||
"\\set ON_ERROR_STOP on",
|
||||
"INSERT INTO app.app_user "
|
||||
"(user_id, external_id, email, display_name, role, cohort) VALUES ("
|
||||
f"'{FIXTURE_LEARNER_ID}',"
|
||||
"'dev:g6-repo-benchmark-v1',"
|
||||
"'dev.g6.repo.benchmark.v1@example.invalid',"
|
||||
"'G6 Repo Benchmark V1','learner',"
|
||||
f"'{FIXTURE_COHORT}') ON CONFLICT (user_id) DO NOTHING;",
|
||||
"INSERT INTO app.sessions (id, learner_id, theory_mode) VALUES ("
|
||||
f"'{FIXTURE_SESSION_ID}','{FIXTURE_LEARNER_ID}','integrative') "
|
||||
"ON CONFLICT (id) DO NOTHING;",
|
||||
"INSERT INTO app.measurement_instrument ("
|
||||
"instrument_id,instrument_version,name_ko,instrument_kind,construct,"
|
||||
"validation_basis,scoring_schema,metadata) VALUES ("
|
||||
f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}',"
|
||||
"'G6 저장소 승인 합성 평가기 비교','runtime_metric','counselor_skill',"
|
||||
"'repo-approved synthetic benchmark only',"
|
||||
'\'{"min":0,"max":1}\','
|
||||
'\'{"data_classification":"synthetic_educational",'
|
||||
'"clinical_claim_allowed":false}\') '
|
||||
"ON CONFLICT (instrument_id,instrument_version) DO NOTHING;",
|
||||
]
|
||||
for batch in (
|
||||
benchmark.pack.baseline_batch,
|
||||
benchmark.pack.candidate_batch,
|
||||
):
|
||||
for observation in batch.observations:
|
||||
measurement_id = uuid5(
|
||||
FIXTURE_NAMESPACE, f"measurement:{observation.evidence_event_id}"
|
||||
)
|
||||
metadata = evaluator.benchmark_anchor_metadata(
|
||||
benchmark, batch, observation
|
||||
)
|
||||
metadata_json = json.dumps(
|
||||
metadata,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
value = (
|
||||
1.0 if observation.predicted_label == observation.gold_label else 0.0
|
||||
)
|
||||
statements.append(
|
||||
"INSERT INTO app.measurement_event ("
|
||||
"measurement_id,session_id,construct,dimension,perspective,source_kind,"
|
||||
"instrument_id,instrument_version,value,scale_min,scale_max,status,"
|
||||
"visible_to,metadata) VALUES ("
|
||||
f"'{measurement_id}','{FIXTURE_SESSION_ID}','counselor_skill',"
|
||||
f"{_sql_literal('g6.' + observation.evidence_event_id)},"
|
||||
"'runtime_observation','observed_runtime',"
|
||||
f"'{FIXTURE_INSTRUMENT_ID}','{FIXTURE_INSTRUMENT_VERSION}',"
|
||||
f"{value},0,1,'ready','{{evaluator,supervisor,research}}',"
|
||||
f"{_sql_literal(metadata_json)}::jsonb) "
|
||||
"ON CONFLICT (measurement_id) DO NOTHING;"
|
||||
)
|
||||
return "\n".join(statements) + "\n"
|
||||
|
||||
|
||||
def _bootstrap_fixture(args: argparse.Namespace) -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-i",
|
||||
args.container,
|
||||
"psql",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-U",
|
||||
args.owner_user,
|
||||
"-d",
|
||||
args.database,
|
||||
],
|
||||
input=_bootstrap_sql().encode("utf-8"),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
detail = result.stderr.decode("utf-8", errors="replace")[-2000:]
|
||||
raise SmokeError(f"dev DB synthetic anchor bootstrap failed: {detail}")
|
||||
|
||||
|
||||
async def _set_research_context(conn: asyncpg.Connection) -> None:
|
||||
await conn.execute("SELECT set_config('app.ai_context','1',true)")
|
||||
await conn.execute("SELECT set_config('app.current_ai_view','research',true)")
|
||||
|
||||
|
||||
async def _counts(conn: asyncpg.Connection) -> dict[str, int]:
|
||||
return {
|
||||
"batches": int(
|
||||
await conn.fetchval(
|
||||
"""
|
||||
SELECT count(*) FROM app.supervision_evaluation_batch
|
||||
WHERE batch_id = ANY($1::text[])
|
||||
""",
|
||||
["oas-g6-batch-baseline", "oas-g6-batch-candidate"],
|
||||
)
|
||||
),
|
||||
"reports": int(
|
||||
await conn.fetchval(
|
||||
"SELECT count(*) FROM app.supervision_drift_report WHERE cohort_id=$1",
|
||||
FIXTURE_COHORT,
|
||||
)
|
||||
),
|
||||
"subgroup_metrics": int(
|
||||
await conn.fetchval(
|
||||
"""
|
||||
SELECT count(*)
|
||||
FROM app.supervision_drift_subgroup_metric m
|
||||
JOIN app.supervision_drift_report r
|
||||
ON r.drift_report_id=m.drift_report_id
|
||||
WHERE r.cohort_id=$1
|
||||
""",
|
||||
FIXTURE_COHORT,
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def _produce_once(
|
||||
conn: asyncpg.Connection,
|
||||
) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
async with conn.transaction():
|
||||
await _set_research_context(conn)
|
||||
result = await evaluator.produce_repository_version_comparison(conn)
|
||||
counts = await _counts(conn)
|
||||
return result, counts
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
env = _parse_env(Path(args.env_file))
|
||||
database_url = env.get("DATABASE_URL", "")
|
||||
if not database_url:
|
||||
raise SmokeError("DATABASE_URL is missing from the selected env file")
|
||||
_bootstrap_fixture(args)
|
||||
conn = await asyncpg.connect(database_url)
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await _set_research_context(conn)
|
||||
before = await _counts(conn)
|
||||
if not args.allow_existing and any(before.values()):
|
||||
raise SmokeError(f"expected first append but rows already exist: {before}")
|
||||
|
||||
first, after_first = await _produce_once(conn)
|
||||
replay, after_replay = await _produce_once(conn)
|
||||
async with conn.transaction():
|
||||
await _set_research_context(conn)
|
||||
research_view = await supervision_research_store.read_research_view(conn)
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
if not args.allow_existing and first.get("idempotent_replay") is not False:
|
||||
raise SmokeError("first comparison did not append a new report")
|
||||
if replay.get("idempotent_replay") is not True:
|
||||
raise SmokeError("second comparison was not an idempotent replay")
|
||||
if (
|
||||
first.get("status") != "drift_flagged"
|
||||
or replay.get("status") != "drift_flagged"
|
||||
):
|
||||
raise SmokeError("repository comparison lost expected drift status")
|
||||
if after_first != {"batches": 2, "reports": 1, "subgroup_metrics": 2}:
|
||||
raise SmokeError(f"unexpected first append cardinality: {after_first}")
|
||||
if after_replay != after_first:
|
||||
raise SmokeError("idempotent replay created duplicate rows")
|
||||
|
||||
drift_report_id = str(first["drift_report_id"])
|
||||
hydrated = next(
|
||||
(
|
||||
item
|
||||
for item in research_view["drift_reports"]
|
||||
if str(item["drift_report_id"]) == drift_report_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if hydrated is None:
|
||||
raise SmokeError("research read model did not hydrate the drift report")
|
||||
if hydrated.get("status") != "drift_flagged":
|
||||
raise SmokeError("hydrated drift report status changed")
|
||||
subgroup_metrics = hydrated.get("subgroup_metrics") or []
|
||||
if {item["subgroup"] for item in subgroup_metrics} != {
|
||||
"synthetic-a",
|
||||
"synthetic-b",
|
||||
}:
|
||||
raise SmokeError("hydrated drift report omitted synthetic subgroup metrics")
|
||||
required_provenance = {
|
||||
"baseline_model",
|
||||
"candidate_model",
|
||||
"baseline_prompt_version",
|
||||
"candidate_prompt_version",
|
||||
"instrument_id",
|
||||
"baseline_instrument_version",
|
||||
"candidate_instrument_version",
|
||||
}
|
||||
if any(not hydrated.get(key) for key in required_provenance):
|
||||
raise SmokeError("hydrated drift report omitted evaluator provenance")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"database": "actual dev PostgreSQL via runtime app role",
|
||||
"fixture_cohort": FIXTURE_COHORT,
|
||||
"benchmark_schema_version": first["benchmark_schema_version"],
|
||||
"benchmark_version": first["benchmark_version"],
|
||||
"benchmark_content_sha256": first["benchmark_content_sha256"],
|
||||
"first_append": {
|
||||
"idempotent_replay": first["idempotent_replay"],
|
||||
"drift_report_id": drift_report_id,
|
||||
"status": first["status"],
|
||||
"counts": after_first,
|
||||
},
|
||||
"replay": {
|
||||
"idempotent_replay": replay["idempotent_replay"],
|
||||
"counts": after_replay,
|
||||
"duplicate_rows": sum(
|
||||
after_replay[key] - after_first[key] for key in after_first
|
||||
),
|
||||
},
|
||||
"hydrated": {
|
||||
"status": hydrated["status"],
|
||||
"baseline_model": hydrated["baseline_model"],
|
||||
"candidate_model": hydrated["candidate_model"],
|
||||
"baseline_prompt_version": hydrated["baseline_prompt_version"],
|
||||
"candidate_prompt_version": hydrated["candidate_prompt_version"],
|
||||
"instrument_id": hydrated["instrument_id"],
|
||||
"baseline_instrument_version": hydrated["baseline_instrument_version"],
|
||||
"candidate_instrument_version": hydrated["candidate_instrument_version"],
|
||||
"subgroup_metrics": subgroup_metrics,
|
||||
},
|
||||
"data_classification": "synthetic_educational",
|
||||
"raw_transcript_included": False,
|
||||
"clinical_claim_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", default="apps/api/.env")
|
||||
parser.add_argument("--container", default="vignette-dev-db")
|
||||
parser.add_argument("--owner-user", default="vignette_owner")
|
||||
parser.add_argument("--database", default="vignette")
|
||||
parser.add_argument("--allow-existing", action="store_true")
|
||||
parser.add_argument("--out", default="")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(_run(args))
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
||||
if args.out:
|
||||
output = Path(args.out)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue