런타임 계약과 학습자 흐름 보강

This commit is contained in:
Yun Chan 2026-06-29 08:12:14 +09:00
parent f456b8997a
commit 206018b088
56 changed files with 4306 additions and 1008 deletions

View file

@ -0,0 +1,270 @@
#!/usr/bin/env python3
"""Validate that the dev dashboard stays aligned with current SSOT claims."""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_DASHBOARD = REPO_ROOT / "docs" / "dev_dashboard.html"
DEFAULT_SOURCE_GAPS = REPO_ROOT / "docs" / "guides" / "source-docs-and-gaps.md"
DEFAULT_BACKLOG = REPO_ROOT / "docs" / "ops" / "backlog-2026-06-26.md"
DEFAULT_TESTING = REPO_ROOT / "docs" / "guides" / "testing.md"
DEFAULT_LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md"
EXPECTED_STATUS_COUNTS = {
"done": 25,
"planned": 0,
}
EXPECTED_OWNER_COLUMN_COUNTS = {
"block": 2,
"decide": 7,
"ext": 2,
}
EXPECTED_DECISION_DETAIL_ROWS = 7
EXPECTED_EXTERNAL_GATE_ROWS = 6
REQUIRED_SHARED_PHRASES = (
"M2 digest worker + memory focused 30 passed",
"M2 주변 회귀 87 passed",
"one-shot worker/runner/default-off scheduler",
"실 provider 장시간 운영",
"임상 골든셋",
"재압축",
"블로커 2 · 결정 7 · 외부조율 2",
"decisions 7건(owner 결정)",
"결정 7개",
"MASTERPLAN_REVISIONS F-10 승인",
)
FORBIDDEN_STALE_PHRASES = (
"M2 주변 회귀 82 passed",
"one-shot digest worker + memory focused 25 passed",
"M2 one-shot digest worker + memory focused 25 passed",
"focused 26 passed",
"운영 background scheduler",
"최신 LTS",
"최종 로컬 풀스택",
"레이아웃 포커스(재설계 화면) | `session-layout`·`session-review`·`admin`·`learner`·`settings`·`teacher`, `@single-run` 제외 | desktop+mobile 병렬 | **58**",
"M2는 보수적 identity/agreement pinned_fact 자동 실적재, append-only history, 명시적 상담 약속 철회 contradiction, episodic embedding writer, 다음 턴 EngineMessage 주입 회귀까지 완료했다.",
"focused X2/backend 40 passed",
"계획·블로커",
"자유연습 기본값</b><p>수정안",
"decisions 8건(owner 결정)",
"결정 8개",
)
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str
class DashboardStructureParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.status_counts: dict[str, int] = {}
self.owner_column_counts: dict[str, int] = {}
self.decision_panel_rows = 0
self.external_gate_rows = 0
self._div_stack: list[str | None] = []
self._owner_column_stack: list[str] = []
self._decision_panel_stack: list[bool] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attrs_dict = dict(attrs)
if tag == "div":
owner_column = attrs_dict.get("data-owner-col")
in_decision_panel = attrs_dict.get("id") == "panel-decisions" or (
bool(self._decision_panel_stack) and self._decision_panel_stack[-1]
)
self._div_stack.append(owner_column)
self._decision_panel_stack.append(in_decision_panel)
if owner_column is not None:
self._owner_column_stack.append(owner_column)
if tag == "span":
classes = attrs_dict.get("class", "")
if "task-status" in classes and "s-plan" in classes:
self.external_gate_rows += 1
if (
self._decision_panel_stack
and self._decision_panel_stack[-1]
and "task-status" in classes
and "s-risk" in classes
):
self.decision_panel_rows += 1
if tag != "article":
return
status = attrs_dict.get("data-status")
if status:
self.status_counts[status] = self.status_counts.get(status, 0) + 1
if attrs_dict.get("data-owner") != "1" or not self._owner_column_stack:
return
owner_column = self._owner_column_stack[-1]
self.owner_column_counts[owner_column] = self.owner_column_counts.get(owner_column, 0) + 1
def handle_endtag(self, tag: str) -> None:
if tag != "div" or not self._div_stack:
return
owner_column = self._div_stack.pop()
if self._decision_panel_stack:
self._decision_panel_stack.pop()
if owner_column is not None and self._owner_column_stack:
self._owner_column_stack.pop()
def _read(path: Path) -> str:
return path.read_text(encoding="utf-8")
def _dashboard_structure(dashboard: str) -> DashboardStructureParser:
parser = DashboardStructureParser()
parser.feed(dashboard)
parser.close()
return parser
def _planned_cards(dashboard: str) -> list[str]:
return re.findall(
r'<article class="scard" data-status="planned"[\s\S]*?</article>',
dashboard,
)
def run_checks(paths: argparse.Namespace) -> dict[str, Any]:
dashboard = _read(paths.dashboard)
source_gaps = _read(paths.source_gaps)
backlog = _read(paths.backlog)
testing = _read(paths.testing)
local_development = _read(paths.local_development)
joined_docs = "\n".join([dashboard, source_gaps, backlog, testing, local_development])
checks: list[Check] = []
structure = _dashboard_structure(dashboard)
counts = structure.status_counts
for status, expected in EXPECTED_STATUS_COUNTS.items():
actual = counts.get(status, 0)
checks.append(
Check(
name=f"status_count_{status}",
passed=actual == expected,
detail=f"expected={expected} actual={actual}",
)
)
doing_count = counts.get("doing", 0)
checks.append(
Check(
name="no_doing_cards",
passed=doing_count == 0 and ">CHECK<" not in dashboard and "s-check" not in dashboard,
detail=f"doing={doing_count}",
)
)
planned = _planned_cards(dashboard)
checks.append(
Check(
name="planned_cards_are_gate_framed",
passed=all(("GATE" in card or "후속" in card or "남은" in card) for card in planned),
detail=f"planned_cards={len(planned)}",
)
)
for owner_column, expected in EXPECTED_OWNER_COLUMN_COUNTS.items():
actual = structure.owner_column_counts.get(owner_column, 0)
checks.append(
Check(
name=f"owner_column_count_{owner_column}",
passed=actual == expected,
detail=f"expected={expected} actual={actual}",
)
)
expected_owner_total = sum(EXPECTED_OWNER_COLUMN_COUNTS.values())
actual_owner_total = sum(structure.owner_column_counts.values())
checks.append(
Check(
name="owner_action_total",
passed=actual_owner_total == expected_owner_total,
detail=f"expected={expected_owner_total} actual={actual_owner_total}",
)
)
checks.append(
Check(
name="external_gate_rows_visible",
passed=structure.external_gate_rows == EXPECTED_EXTERNAL_GATE_ROWS,
detail=f"expected={EXPECTED_EXTERNAL_GATE_ROWS} actual={structure.external_gate_rows}",
)
)
decision_detail_rows = structure.decision_panel_rows
checks.append(
Check(
name="decision_detail_rows",
passed=decision_detail_rows == EXPECTED_DECISION_DETAIL_ROWS,
detail=f"expected={EXPECTED_DECISION_DETAIL_ROWS} actual={decision_detail_rows}",
)
)
for phrase in REQUIRED_SHARED_PHRASES:
checks.append(
Check(
name=f"required_phrase::{phrase}",
passed=phrase in joined_docs,
detail="present" if phrase in joined_docs else "missing",
)
)
for phrase in FORBIDDEN_STALE_PHRASES:
checks.append(
Check(
name=f"forbidden_phrase::{phrase}",
passed=phrase not in joined_docs,
detail="absent" if phrase not in joined_docs else "present",
)
)
return {
"schema": "vignette.dev_dashboard_ssot_check.v1",
"passed": all(check.passed for check in checks),
"status_counts": counts,
"checks": [check.__dict__ for check in checks],
}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dashboard", type=Path, default=DEFAULT_DASHBOARD)
parser.add_argument("--source-gaps", type=Path, default=DEFAULT_SOURCE_GAPS)
parser.add_argument("--backlog", type=Path, default=DEFAULT_BACKLOG)
parser.add_argument("--testing", type=Path, default=DEFAULT_TESTING)
parser.add_argument("--local-development", type=Path, default=DEFAULT_LOCAL_DEVELOPMENT)
parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.")
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
report = run_checks(args)
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
else:
status = "PASS" if report["passed"] else "FAIL"
print(f"{status} dev dashboard SSOT check")
for check in report["checks"]:
marker = "PASS" if check["passed"] else "FAIL"
print(f"{marker} {check['name']} {check['detail']}")
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -5,7 +5,6 @@ import assert from "node:assert/strict";
const DEFAULT_FIXTURE = "apps/api/engine_gateway/golden/engine_gateway_contract.v1.json";
const DEFAULT_SCHEMA = "apps/api/engine_gateway/golden/engine_gateway_schema.v1.json";
const EXPECTED_EVENTS = ["token", "done", "error"];
function usage() {
return [
@ -86,14 +85,74 @@ function assertString(value, label) {
assert.equal(typeof value, "string", `${label} must be a string`);
}
function assertBoolean(value, label) {
assert.equal(typeof value, "boolean", `${label} must be a boolean`);
}
function schemaDef(schema, name) {
const def = schema?.$defs?.[name];
assertPlainObject(def, `$defs.${name}`);
return def;
}
function assertRootShape(schema, fixture) {
assert.deepEqual(schema["x-engine-gateway-sse-events"], EXPECTED_EVENTS);
function resolveSchemaRef(schema, ref, label) {
assertString(ref, label);
const prefix = "#/$defs/";
assert.ok(ref.startsWith(prefix), `${label} must reference a schema definition`);
return schemaDef(schema, ref.slice(prefix.length));
}
function streamEventSpecs(schema) {
const packetDef = schemaDef(schema, "StreamPacket");
assert.ok(Array.isArray(packetDef.oneOf), "$defs.StreamPacket.oneOf must be an array");
const specs = new Map();
for (const [index, variant] of packetDef.oneOf.entries()) {
assertPlainObject(variant, `$defs.StreamPacket.oneOf[${index}]`);
const event = variant.properties?.event?.const;
assertString(event, `$defs.StreamPacket.oneOf[${index}].properties.event.const`);
const payloadRef = variant.properties?.payload?.$ref;
const payloadDef = resolveSchemaRef(
schema,
payloadRef,
`$defs.StreamPacket.oneOf[${index}].properties.payload.$ref`,
);
assert.ok(!specs.has(event), `duplicate stream event spec: ${event}`);
specs.set(event, { event, payloadDef });
}
return specs;
}
function defaultModelSentinel(schema) {
const value = schema["x-engine-gateway-default-model-sentinel"];
assertString(value, "x-engine-gateway-default-model-sentinel");
assert.notEqual(value.trim(), "", "x-engine-gateway-default-model-sentinel must not be blank");
return value;
}
function assertDefaultModelSentinelUsage(fixture, sentinel) {
assert.equal(
fixture.generate_request?.model,
sentinel,
"golden generate_request.model must pin the gateway default-routing sentinel",
);
assert.notEqual(
fixture.generate_response?.model,
sentinel,
"generate_response.model must report the resolved provider model, not the default-routing sentinel",
);
for (const [index, packet] of (fixture.stream_packets ?? []).entries()) {
if (packet?.event !== "done") continue;
assert.notEqual(
packet.payload?.model,
sentinel,
`stream_packets[${index}].payload.model must report the resolved provider model`,
);
}
}
function assertRootShape(schema, fixture, streamSpecs) {
assert.deepEqual(schema["x-engine-gateway-sse-events"], [...streamSpecs.keys()]);
assertDefaultModelSentinelUsage(fixture, defaultModelSentinel(schema));
assertRequired(fixture, schema.required, "fixture");
assertOnlyKeys(fixture, Object.keys(schema.properties), "fixture");
assert.equal(fixture.version, schema.properties.version.const);
@ -159,7 +218,7 @@ function assertGenerateResponse(schema, response) {
if (Object.hasOwn(response, "structured")) assertNullableObject(response.structured, "generate_response.structured");
}
function parseSseFrame(frame) {
function parseSseFrame(frame, streamSpecs) {
assertString(frame, "stream_frame");
let event = null;
const dataLines = [];
@ -174,63 +233,70 @@ function parseSseFrame(frame) {
}
assert.ok(event, "stream frame must contain an event line");
assert.ok(EXPECTED_EVENTS.includes(event), `unknown stream event: ${event}`);
assert.ok(streamSpecs.has(event), `unknown stream event: ${event}`);
assert.ok(dataLines.length > 0, `stream event ${event} must contain data`);
return packetFromEventData(event, dataLines.join("\n"));
return packetFromEventData(event, dataLines.join("\n"), streamSpecs);
}
function packetFromEventData(event, data) {
function packetFromEventData(event, data, streamSpecs) {
if (data === "[DONE]") return null;
if (event === "token") {
let payload;
try {
payload = JSON.parse(data);
} catch {
payload = { text: data };
}
assertPlainObject(payload, "token payload");
assertString(payload.text, "token payload.text");
assertOnlyKeys(payload, ["text"], "token payload");
return { event, payload };
}
let payload;
try {
payload = JSON.parse(data);
} catch (error) {
throw new Error(`stream event ${event} must contain JSON data: ${error.message}`);
if (event === "token") {
payload = { text: data };
} else {
throw new Error(`stream event ${event} must contain JSON data: ${error.message}`);
}
}
if (event === "done") {
assertPlainObject(payload, "done payload");
assertRequired(payload, ["provider", "model"], "done payload");
assertOnlyKeys(payload, ["provider", "model", "tokens_in", "tokens_out", "cost_usd", "turns"], "done payload");
assertString(payload.provider, "done payload.provider");
assertString(payload.model, "done payload.model");
if (Object.hasOwn(payload, "tokens_in")) assertInteger(payload.tokens_in, "done payload.tokens_in");
if (Object.hasOwn(payload, "tokens_out")) assertInteger(payload.tokens_out, "done payload.tokens_out");
if (Object.hasOwn(payload, "cost_usd")) assertNumber(payload.cost_usd, "done payload.cost_usd");
if (Object.hasOwn(payload, "turns")) assertInteger(payload.turns, "done payload.turns");
return { event, payload };
}
assert.equal(event, "error");
assertPlainObject(payload, "error payload");
assertRequired(payload, ["detail"], "error payload");
assertOnlyKeys(payload, ["detail"], "error payload");
assertString(payload.detail, "error payload.detail");
const spec = streamSpecs.get(event);
assert.ok(spec, `unknown stream event: ${event}`);
assertPayloadMatchesSchema(payload, spec.payloadDef, `${event} payload`);
return { event, payload };
}
function assertStreamContract(fixture) {
function assertPayloadMatchesSchema(payload, payloadDef, label) {
assertPlainObject(payload, label);
assertRequired(payload, payloadDef.required ?? [], label);
const properties = payloadDef.properties ?? {};
assertOnlyKeys(payload, Object.keys(properties), label);
for (const [key, spec] of Object.entries(properties)) {
if (!Object.hasOwn(payload, key)) continue;
assertValueMatchesSchemaType(payload[key], spec, `${label}.${key}`);
}
}
function assertValueMatchesSchemaType(value, spec, label) {
const types = Array.isArray(spec.type) ? spec.type : [spec.type];
if (value === null && types.includes("null")) return;
if (types.includes("string")) {
assertString(value, label);
} else if (types.includes("integer")) {
assertInteger(value, label);
} else if (types.includes("number")) {
assertNumber(value, label);
} else if (types.includes("boolean")) {
assertBoolean(value, label);
} else if (types.includes("object")) {
assertPlainObject(value, label);
} else if (types.includes("array")) {
assert.ok(Array.isArray(value), `${label} must be an array`);
} else {
throw new Error(`${label} has unsupported schema type: ${spec.type}`);
}
}
function assertStreamContract(fixture, streamSpecs) {
assert.ok(Array.isArray(fixture.stream_frames), "stream_frames must be an array");
assert.ok(Array.isArray(fixture.stream_packets), "stream_packets must be an array");
assert.ok(Array.isArray(fixture.compatibility_lines), "compatibility_lines must be an array");
const decodedPackets = [];
for (const frame of fixture.stream_frames) {
const packet = parseSseFrame(frame);
const packet = parseSseFrame(frame, streamSpecs);
if (packet) decodedPackets.push(packet);
}
assert.deepEqual(decodedPackets, fixture.stream_packets);
@ -243,12 +309,14 @@ function assertStreamContract(fixture) {
}
function validateContract(schema, fixture) {
assertRootShape(schema, fixture);
const streamSpecs = streamEventSpecs(schema);
assertRootShape(schema, fixture, streamSpecs);
assertGenerateRequest(schema, fixture.generate_request);
assertGenerateResponse(schema, fixture.generate_response);
assertStreamContract(fixture);
assertStreamContract(fixture, streamSpecs);
return {
version: fixture.version,
defaultModelSentinel: defaultModelSentinel(schema),
streamEvents: schema["x-engine-gateway-sse-events"],
streamFrameCount: fixture.stream_frames.length,
decodedPacketCount: fixture.stream_packets.length,

View file

@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Run one session digest compression job.
Dry-run is the default: it loads one ended session, releases the DB connection,
calls the configured engine, applies the local digest quality gate, and prints a
metadata-only result. Use --apply to replace the fallback digest after acceptance.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import sys
import time
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
API_ROOT = REPO_ROOT / "apps" / "api"
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8")
def _prepare_imports() -> None:
os.chdir(API_ROOT)
api_root = str(API_ROOT)
if api_root not in sys.path:
sys.path.insert(0, api_root)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Run the M2 session digest worker for one session. Dry-run is the "
"default; --apply writes accepted LLM digest output back to DB."
)
)
parser.add_argument("--session-id", required=True, help="app.sessions.id to compress")
parser.add_argument("--model", help="optional engine model override")
parser.add_argument(
"--forbidden-substring",
action="append",
default=[],
help="raw substring that must reject the candidate digest; repeatable",
)
parser.add_argument("--apply", action="store_true", help="write accepted digest output to DB")
parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
parser.add_argument(
"--show-digest",
action="store_true",
help="include the accepted digest text in output; default is metadata only",
)
parser.add_argument(
"--allow-rejected",
action="store_true",
help="exit 0 when the engine returned a rejected candidate",
)
return parser
async def _run(args: argparse.Namespace) -> dict[str, Any]:
_prepare_imports()
from app.db import acquire, close_pool, init_pool
from app.engine_client import engine_client
from app.routes.admin import apply_engine_config_from_store
from app.session_persistence import record_llm_call_audit
from app.services import session_digest_worker
started = time.perf_counter()
await init_pool()
try:
async with acquire(role="admin") as conn:
loaded = await session_digest_worker.load_session_digest_job(conn, args.session_id)
if loaded is None:
return {
"ok": False,
"found": False,
"applied": False,
"session_id": args.session_id,
"error": "session digest job not found",
}
await apply_engine_config_from_store()
await engine_client.startup()
worker = await session_digest_worker.run_session_digest_worker(
loaded.job,
engine_client,
existing_case_digest=loaded.existing_case_digest,
forbidden_substrings=tuple(args.forbidden_substring or ()),
model=args.model,
audit_hook=record_llm_call_audit,
)
applied = False
if args.apply and worker.apply_plan is not None:
async with acquire(role="admin") as conn:
applied = await session_digest_worker.apply_session_digest_plan(
conn,
worker.apply_plan,
learner_id=loaded.learner_id,
)
return _result_payload(
loaded=loaded,
worker=worker,
applied=applied,
apply_requested=bool(args.apply),
show_digest=bool(args.show_digest),
elapsed_ms=int((time.perf_counter() - started) * 1000),
)
finally:
await engine_client.shutdown()
await close_pool()
def _result_payload(
*,
loaded: Any,
worker: Any,
applied: bool,
apply_requested: bool,
show_digest: bool,
elapsed_ms: int,
) -> dict[str, Any]:
quality = worker.outcome.quality
result = worker.outcome.result
apply_plan = worker.apply_plan
payload: dict[str, Any] = {
"ok": bool(result is not None),
"found": True,
"accepted": bool(result is not None),
"applied": bool(applied),
"apply_requested": apply_requested,
"fallback_required": bool(worker.fallback_required),
"session_id": loaded.job.session_id,
"case_id": loaded.job.case_id,
"session_no": loaded.job.session_no,
"quality": {
"accepted": bool(quality.accepted),
"reason": quality.reason,
"retryable": bool(quality.retryable),
"details": list(quality.details),
},
"engine": {
"provider": worker.response.provider,
"model": worker.response.model,
"tokens_in": worker.response.tokens_in,
"tokens_out": worker.response.tokens_out,
"cost_usd": worker.response.cost_usd,
"inference_geo": worker.response.inference_geo,
},
"elapsed_ms": elapsed_ms,
}
if apply_plan is not None:
payload["apply_plan"] = {
"compressed_by": apply_plan.compressed_by,
"token_count": apply_plan.token_count,
"open_thread_count": len(apply_plan.result.open_threads),
"digest_chars": len(apply_plan.result.digest),
"case_digest_chars": len(apply_plan.case_digest or ""),
}
if show_digest:
payload["digest"] = apply_plan.result.digest
return payload
def _print_text(payload: dict[str, Any]) -> None:
if not payload.get("found"):
print(payload.get("error") or "session digest job not found", file=sys.stderr)
return
status = "accepted" if payload.get("accepted") else "rejected"
mode = "applied" if payload.get("applied") else "dry-run"
print(
"session digest worker: {status} ({mode}) session={session} reason={reason}".format(
status=status,
mode=mode,
session=payload.get("session_id"),
reason=(payload.get("quality") or {}).get("reason"),
)
)
plan = payload.get("apply_plan") or {}
if plan:
print(
"candidate: compressed_by={compressed_by} token_count={token_count} "
"digest_chars={digest_chars} case_digest_chars={case_digest_chars}".format(**plan)
)
engine = payload.get("engine") or {}
if engine:
print(
"engine: provider={provider} model={model} tokens={tokens_in}/{tokens_out} cost={cost_usd}".format(
**engine
)
)
if payload.get("digest"):
print("digest:")
print(payload["digest"])
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(list(argv if argv is not None else sys.argv[1:]))
try:
payload = asyncio.run(_run(args))
except Exception as exc: # noqa: BLE001 - CLI should report compactly.
payload = {
"ok": False,
"found": False,
"applied": False,
"error_type": exc.__class__.__name__,
"error": str(exc),
}
if args.json:
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
else:
_print_text(payload)
if payload.get("ok") or (args.allow_rejected and payload.get("found")):
return 0
if payload.get("found"):
return 2
return 3
if __name__ == "__main__":
raise SystemExit(main())