Jev 운영 배포 검증과 남은 확인 항목 기록
This commit is contained in:
Yun Chan
parent
969d9e2c3c
commit
20dc30c8aa
7 changed files with 291 additions and 15 deletions
|
|
@ -280,6 +280,57 @@ async def _run_turn(
|
|||
return record, context.state_after, generated_text if done_payload is not None and error_code is None else None
|
||||
|
||||
|
||||
def _is_stateless_openai_phase(runtime: dict[str, Any], phase: dict[str, Any]) -> bool:
|
||||
completed_turns = [turn for turn in phase["turns"] if turn["status"] == "done"]
|
||||
settings = runtime["settings"]
|
||||
return (
|
||||
bool(completed_turns)
|
||||
and all(turn["generation"]["provider"] == "openai" for turn in completed_turns)
|
||||
and (settings.live_client_provider == "openai" or settings.engine_mode == "openai")
|
||||
)
|
||||
|
||||
|
||||
def _mark_cleanup_failed(phase: dict[str, Any]) -> None:
|
||||
phase["session_closed"] = False
|
||||
phase["cleanup_status"] = "failed"
|
||||
phase["cleanup_error"] = "engine_close_failed"
|
||||
if phase["status"] == "done":
|
||||
phase["status"] = "error"
|
||||
phase["error"] = "engine_close_failed"
|
||||
|
||||
|
||||
async def _cleanup_synthetic_session(
|
||||
*,
|
||||
runtime: dict[str, Any],
|
||||
phase: dict[str, Any],
|
||||
session_id: str,
|
||||
) -> None:
|
||||
try:
|
||||
response = await runtime["engine_client"].client.delete(f"/session/{session_id}")
|
||||
phase["cleanup_http_status"] = response.status_code
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
_mark_cleanup_failed(phase)
|
||||
return
|
||||
|
||||
if not isinstance(payload, dict) or set(payload) != {"closed"} or type(payload["closed"]) is not bool:
|
||||
_mark_cleanup_failed(phase)
|
||||
return
|
||||
|
||||
phase["cleanup_closed"] = payload["closed"]
|
||||
if payload["closed"]:
|
||||
phase["session_closed"] = True
|
||||
phase["cleanup_status"] = "closed"
|
||||
elif _is_stateless_openai_phase(runtime, phase):
|
||||
phase["session_closed"] = False
|
||||
phase["cleanup_status"] = "not_required"
|
||||
else:
|
||||
_mark_cleanup_failed(phase)
|
||||
|
||||
|
||||
async def _run_phase(
|
||||
*,
|
||||
runtime: dict[str, Any],
|
||||
|
|
@ -337,13 +388,7 @@ async def _run_phase(
|
|||
return phase
|
||||
finally:
|
||||
settings.client_affect_provider = original_provider
|
||||
closed = await runtime["engine_client"].close_session(session_id)
|
||||
phase["session_closed"] = closed
|
||||
if not closed:
|
||||
phase["cleanup_error"] = "engine_close_failed"
|
||||
if phase["status"] == "done":
|
||||
phase["status"] = "error"
|
||||
phase["error"] = "engine_close_failed"
|
||||
await _cleanup_synthetic_session(runtime=runtime, phase=phase, session_id=session_id)
|
||||
|
||||
|
||||
def _source_sha256() -> dict[str, str]:
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import io
|
||||
import unittest
|
||||
from contextlib import redirect_stderr
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
SCRIPT_PATH = Path(__file__).with_name("probe-jev-dialogue.py")
|
||||
|
|
@ -17,6 +19,35 @@ SPEC.loader.exec_module(probe)
|
|||
|
||||
|
||||
class ProbeJevDialogueTest(unittest.TestCase):
|
||||
def _cleanup_runtime(self, response: object, *, live_provider: str = "openai", engine_mode: str = "remote"):
|
||||
requests: list[str] = []
|
||||
|
||||
class Client:
|
||||
async def delete(self, path: str) -> object:
|
||||
requests.append(path)
|
||||
return response
|
||||
|
||||
return {
|
||||
"settings": SimpleNamespace(
|
||||
live_client_provider=live_provider,
|
||||
engine_mode=engine_mode,
|
||||
),
|
||||
"engine_client": SimpleNamespace(client=Client()),
|
||||
}, requests
|
||||
|
||||
@staticmethod
|
||||
def _phase(provider: str) -> dict[str, object]:
|
||||
return {
|
||||
"status": "done",
|
||||
"turns": [
|
||||
{
|
||||
"status": "done",
|
||||
"generation": {"provider": provider},
|
||||
}
|
||||
],
|
||||
"session_closed": False,
|
||||
}
|
||||
|
||||
def test_parser_validates_phase_and_repeat_contract(self) -> None:
|
||||
args = probe.build_parser().parse_args(
|
||||
["--output", "report.json", "--phases", "jev,legacy", "--repeats", "3", "--label", "trial-a"]
|
||||
|
|
@ -85,6 +116,107 @@ class ProbeJevDialogueTest(unittest.TestCase):
|
|||
],
|
||||
)
|
||||
|
||||
def test_openai_stateless_cleanup_allows_http_200_closed_false(self) -> None:
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, bool]:
|
||||
return {"closed": False}
|
||||
|
||||
response = Response()
|
||||
runtime, requests = self._cleanup_runtime(response)
|
||||
phase = self._phase("openai")
|
||||
|
||||
asyncio.run(probe._cleanup_synthetic_session(runtime=runtime, phase=phase, session_id="session-1"))
|
||||
|
||||
self.assertEqual(requests, ["/session/session-1"])
|
||||
self.assertEqual(phase["status"], "done")
|
||||
self.assertFalse(phase["session_closed"])
|
||||
self.assertEqual(phase["cleanup_http_status"], 200)
|
||||
self.assertFalse(phase["cleanup_closed"])
|
||||
self.assertEqual(phase["cleanup_status"], "not_required")
|
||||
self.assertNotIn("cleanup_error", phase)
|
||||
|
||||
def test_non_openai_cleanup_closed_false_is_failure(self) -> None:
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, bool]:
|
||||
return {"closed": False}
|
||||
|
||||
response = Response()
|
||||
runtime, requests = self._cleanup_runtime(response)
|
||||
phase = self._phase("claude")
|
||||
|
||||
asyncio.run(probe._cleanup_synthetic_session(runtime=runtime, phase=phase, session_id="session-2"))
|
||||
|
||||
self.assertEqual(requests, ["/session/session-2"])
|
||||
self.assertEqual(phase["status"], "error")
|
||||
self.assertFalse(phase["session_closed"])
|
||||
self.assertEqual(phase["cleanup_http_status"], 200)
|
||||
self.assertFalse(phase["cleanup_closed"])
|
||||
self.assertEqual(phase["cleanup_status"], "failed")
|
||||
self.assertEqual(phase["cleanup_error"], "engine_close_failed")
|
||||
|
||||
def test_openai_cleanup_http_failure_is_failure(self) -> None:
|
||||
class Response:
|
||||
status_code = 503
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.json_called = False
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
raise RuntimeError("http failure")
|
||||
|
||||
def json(self) -> dict[str, bool]:
|
||||
self.json_called = True
|
||||
return {"closed": False}
|
||||
|
||||
response = Response()
|
||||
runtime, requests = self._cleanup_runtime(response)
|
||||
phase = self._phase("openai")
|
||||
|
||||
asyncio.run(probe._cleanup_synthetic_session(runtime=runtime, phase=phase, session_id="session-3"))
|
||||
|
||||
self.assertEqual(requests, ["/session/session-3"])
|
||||
self.assertEqual(phase["status"], "error")
|
||||
self.assertFalse(phase["session_closed"])
|
||||
self.assertEqual(phase["cleanup_http_status"], 503)
|
||||
self.assertFalse(response.json_called)
|
||||
self.assertNotIn("cleanup_closed", phase)
|
||||
self.assertEqual(phase["cleanup_status"], "failed")
|
||||
self.assertEqual(phase["cleanup_error"], "engine_close_failed")
|
||||
|
||||
def test_openai_cleanup_requires_boolean_closed_json(self) -> None:
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, int]:
|
||||
return {"closed": 0}
|
||||
|
||||
response = Response()
|
||||
runtime, requests = self._cleanup_runtime(response)
|
||||
phase = self._phase("openai")
|
||||
|
||||
asyncio.run(probe._cleanup_synthetic_session(runtime=runtime, phase=phase, session_id="session-4"))
|
||||
|
||||
self.assertEqual(requests, ["/session/session-4"])
|
||||
self.assertEqual(phase["status"], "error")
|
||||
self.assertFalse(phase["session_closed"])
|
||||
self.assertEqual(phase["cleanup_http_status"], 200)
|
||||
self.assertNotIn("cleanup_closed", phase)
|
||||
self.assertEqual(phase["cleanup_status"], "failed")
|
||||
self.assertEqual(phase["cleanup_error"], "engine_close_failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue