yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

대시보드 이슈 정리 1차

Browse source
This commit is contained in:
Yun Chan 2026-06-27 17:51:54 +09:00
parent 94cc56592f
commit f472883c31
13 changed files with 592 additions and 311 deletions
Download patch file Download diff file Expand all files Collapse all files

6
apps/api/app/routes/sessions.py
View file

@ -1410,13 +1410,12 @@ async def submit_turn(
detail=f"engine unavailable: {exc}",
) from exc
await turn_runtime.record_completed_turn(
await turn_runtime.finalize_completed_turn(
sess,
ctx,
result,
context_prefix="session",
)
await turn_runtime.record_safety_event(sess, ctx, result)
return TurnResponse(
turn_seq=result.turn_seq,
@ -1473,13 +1472,12 @@ async def stream_turn(
data = {**ev.data, "stage": _stage_label(ctx.state_after.stage)}
evaluation = await _evaluate_stream_turn(ctx, final_reply)
result = _stream_result_from_done(ctx, final_reply, data, evaluation)
await turn_runtime.record_completed_turn(
await turn_runtime.finalize_completed_turn(
sess,
ctx,
result,
context_prefix="session",
)
await turn_runtime.record_safety_event(sess, ctx, result)
yield {"event": "done", "data": json.dumps(data, ensure_ascii=False)}
else:
yield {"event": ev.event, "data": json.dumps(ev.data, ensure_ascii=False)}

3
apps/api/app/routes/voice.py
View file

@ -329,7 +329,7 @@ async def _run_turn_and_speak(
reply = result.client_reply or ""
# Persist only after the client reply has been generated. A failed AI turn
# must not leave a learner-only transcript in review or history.
await turn_runtime.record_completed_turn(
await turn_runtime.finalize_completed_turn(
sess,
ctx,
result,
@ -347,7 +347,6 @@ async def _run_turn_and_speak(
evaluation=result.evaluation,
),
)
await turn_runtime.record_safety_event(sess, ctx, result)
# Send the final client text before audio playback.
await _safe_send_json(

226
apps/api/app/test_phase3_artifact_checker.py Normal file
View file

@ -0,0 +1,226 @@
import hashlib
import importlib.util
import json
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
CHECKER_PATH = REPO_ROOT / "scripts" / "check-phase3-artifacts.py"
spec = importlib.util.spec_from_file_location("phase3_artifact_checker", CHECKER_PATH)
assert spec is not None and spec.loader is not None
checker = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = checker
spec.loader.exec_module(checker)
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class Phase3ArtifactCheckerTests(unittest.TestCase):
def make_root(self) -> Path:
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
root = Path(tmp.name)
write_text(
root / "00-intake" / "pilot_roster.csv",
"participant_id,cohort_id,consent_version,consent_signed_at,withdrawal_state,enrolled_at\n"
"P3-001,phase3,v1,2026-06-27T00:00:00Z,active,2026-06-27T00:00:00Z\n",
)
write_text(
root / "00-intake" / "consent_receipts.csv",
"participant_id,consent_version,signed_at,signer_role,receipt_id\n"
"P3-001,v1,2026-06-27T00:00:00Z,self,R-001\n",
)
write_text(
root / "01-sessions" / "session_completion.csv",
"participant_id,session_id,persona_id,started_at,ended_at,completion_state,turns_count,supervisor_reviewed_at\n"
"P3-001,S-001,P1,2026-06-27T00:00:00Z,2026-06-27T00:50:00Z,completed,12,2026-06-27T01:00:00Z\n",
)
write_text(
root / "02-measures" / "prepost_measures.csv",
"participant_id,measure_name,timepoint,score,collected_at\n"
"P3-001,self_efficacy,pre,3,2026-06-27T00:00:00Z\n"
"P3-001,self_efficacy,post,4,2026-06-27T01:00:00Z\n",
)
write_text(
root / "02-measures" / "sus_responses.csv",
"participant_id,item,response,collected_at\n"
"P3-001,1,5,2026-06-27T01:00:00Z\n",
)
write_text(
root / "04-privacy" / "withdrawal_log.csv",
"participant_id,requested_at,effective_at,scope,status,attestation_path\n",
)
write_text(
root / "04-privacy" / "privacy_audit.md",
"# Privacy audit\n\nLegal/privacy reviewer: reviewer@example.invalid\n",
)
write_text(root / "03-export" / "anonymized_dataset.jsonl", "{}\n")
metrics = {
name: {
"value": 1,
"threshold": 1,
"pass": True,
"numerator": 1,
"denominator": 1,
"method": "fixture",
"source_files": ["fixture"],
}
for name in checker.KPI_METRICS
}
write_text(
root / "02-measures" / "kpi_report.json",
json.dumps(
{
"pilot_id": "phase3-fixture",
"generated_at": "2026-06-27T00:00:00Z",
"source_window": {
"started_at": "2026-06-27T00:00:00Z",
"ended_at": "2026-06-27T01:00:00Z",
},
"cohort_size": 1,
"metrics": metrics,
"exclusions": [],
"open_schema_gaps": [],
"review": {
"operator": "test",
"reviewed_at": "2026-06-27T01:00:00Z",
"decision": "fixture",
},
},
ensure_ascii=False,
)
+ "\n",
)
dataset_path = root / "03-export" / "anonymized_dataset.jsonl"
write_text(
root / "03-export" / "export_manifest.json",
json.dumps(
{
"export_id": "phase3-fixture",
"dataset_name": "vignette_phase3_recursive_learning_seed",
"export_status": checker.APPROVED_EXPORT_STATUS,
"created_at": "2026-06-27T00:00:00Z",
"purpose": "test",
"source_window": {
"started_at": "2026-06-27T00:00:00Z",
"ended_at": "2026-06-27T01:00:00Z",
},
"source_tables": ["app.sessions"],
"selection_criteria": {
"include_withdrawn": False,
"min_completed_sessions": 2,
},
"consent_scope": {
"allowed_uses": ["education_quality_review", "recursive_learning_seed"],
"participants_included": 1,
"participants_excluded": 0,
},
"anonymization": {
"text_transform": "masked_text_only",
"direct_identifier_policy": "blocked",
},
"pii_scan": {"status": "pass"},
"agreement": {"kappa": 0.60, "icc": 0.75},
"files": [
{
"path": "03-export/anonymized_dataset.jsonl",
"rows": 1,
"sha256": sha256(dataset_path),
"schema": "phase3_dataset_item_v1",
}
],
"approvals": {
"data_steward": "steward",
"legal_or_privacy_reviewer": "privacy",
"technical_operator": "operator",
"approved_at": "2026-06-27T01:00:00Z",
},
"known_limitations": [],
},
ensure_ascii=False,
)
+ "\n",
)
return root
def test_valid_approved_fixture_passes(self) -> None:
root = self.make_root()
report = checker.validate(root, max_scan_rows=100)
self.assertEqual([], report.errors)
def test_approved_manifest_requires_privacy_and_agreement_gates(self) -> None:
root = self.make_root()
manifest_path = root / "03-export" / "export_manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
manifest["pii_scan"]["status"] = "pending"
manifest["agreement"]["kappa"] = 0.59
manifest["agreement"]["icc"] = 0.74
manifest["selection_criteria"]["include_withdrawn"] = True
manifest["consent_scope"]["allowed_uses"] = ["education_quality_review"]
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
report = checker.validate(root, max_scan_rows=100)
errors = "\n".join(report.errors)
self.assertIn("pii_scan.status='pass'", errors)
self.assertIn("agreement.kappa >= 0.60", errors)
self.assertIn("agreement.icc >= 0.75", errors)
self.assertIn("include_withdrawn=false", errors)
self.assertIn("recursive_learning_seed consent scope", errors)
def test_kpi_metric_required_fields_are_errors(self) -> None:
root = self.make_root()
report_path = root / "02-measures" / "kpi_report.json"
data = json.loads(report_path.read_text(encoding="utf-8"))
del data["metrics"]["sus"]["source_files"]
report_path.write_text(json.dumps(data), encoding="utf-8")
report = checker.validate(root, max_scan_rows=100)
self.assertTrue(
any("metric 'sus' missing 'source_files'" in error for error in report.errors),
report.errors,
)
def test_csv_enum_values_are_validated(self) -> None:
root = self.make_root()
write_text(
root / "01-sessions" / "session_completion.csv",
"participant_id,session_id,persona_id,started_at,ended_at,completion_state,turns_count,supervisor_reviewed_at\n"
"P3-001,S-001,P1,2026-06-27T00:00:00Z,2026-06-27T00:50:00Z,done,12,2026-06-27T01:00:00Z\n",
)
report = checker.validate(root, max_scan_rows=100)
self.assertTrue(
any("invalid completion_state 'done'" in error for error in report.errors),
report.errors,
)
def test_approved_manifest_hash_must_match_existing_file(self) -> None:
root = self.make_root()
manifest_path = root / "03-export" / "export_manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
manifest["files"][0]["sha256"] = "0" * 64
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
report = checker.validate(root, max_scan_rows=100)
self.assertTrue(any("sha256 mismatch" in error for error in report.errors), report.errors)
if __name__ == "__main__":
unittest.main()

20
apps/api/app/turn_runtime.py
View file

@ -182,9 +182,29 @@ async def record_safety_event(
pass
async def finalize_completed_turn(
sess: InProcSession,
ctx: orchestrator.TurnContext,
result: orchestrator.TurnResult,
*,
context_prefix: str,
counselor_turn: TurnRecord | None = None,
) -> None:
"""Persist a completed turn and emit any derived safety alert in route-safe order."""
await record_completed_turn(
sess,
ctx,
result,
context_prefix=context_prefix,
counselor_turn=counselor_turn,
)
await record_safety_event(sess, ctx, result)
__all__ = [
"SessionAccessError",
"append_completed_turn",
"finalize_completed_turn",
"load_owned_session",
"record_safety_event",
"record_completed_turn",

70
apps/api/engine_gateway/test_gateway_model.py
View file

@ -53,6 +53,32 @@ def _model_arg(args):
return args[args.index("--model") + 1]
async def _read_streaming_response(response):
chunks = []
async for chunk in response.body_iterator:
if isinstance(chunk, bytes):
chunks.append(chunk.decode("utf-8"))
else:
chunks.append(str(chunk))
return "".join(chunks)
class _FakeStreamSession:
def __init__(self, events, model="test-model"):
self.events = events
self.model = model
self.closed = False
async def turn_stream(self, content, timeout=600.0):
self.content = content
self.timeout = timeout
for event in self.events:
yield event
async def close(self):
self.closed = True
class GatewayModelTest(unittest.TestCase):
def setUp(self):
gateway.SESSIONS.clear()
@ -212,6 +238,50 @@ class GatewayModelTest(unittest.TestCase):
self.assertEqual(closed, [started[0]])
self.assertNotIn(started[0].id, gateway.SESSIONS)
def test_v1_stream_frames_token_and_done_events(self):
session = _FakeStreamSession(
[
{"type": "delta", "text": "안녕"},
{"type": "done", "cost_usd": 0.03, "turns": 2},
],
model="stream-model",
)
async def fake_resolve(req, system_prompt):
return session, True
with patch.object(gateway, "_resolve_session", fake_resolve):
response = asyncio.run(gateway.v1_stream(_request()))
body = asyncio.run(_read_streaming_response(response))
self.assertIn("event: token", body)
self.assertIn('data: {"text": "안녕"}', body)
self.assertIn("event: done", body)
self.assertIn('"provider": "claude_cli"', body)
self.assertIn('"model": "stream-model"', body)
self.assertIn('"cost_usd": 0.03', body)
self.assertEqual(session.content, "hello")
self.assertEqual(session.timeout, 600.0)
self.assertTrue(session.closed)
def test_v1_stream_frames_engine_error_event(self):
session = _FakeStreamSession(
[
{"type": "done", "is_error": True, "error": "engine failed"},
]
)
async def fake_resolve(req, system_prompt):
return session, False
with patch.object(gateway, "_resolve_session", fake_resolve):
response = asyncio.run(gateway.v1_stream(_request()))
body = asyncio.run(_read_streaming_response(response))
self.assertIn("event: error", body)
self.assertIn('data: {"detail": "engine failed"}', body)
self.assertFalse(session.closed)
if __name__ == "__main__":
unittest.main()