"""/ready 캐시의 실패 TTL 계약. 2026-08-18 공개 런타임 장애: reasoning_effort=high 콜드 프로브 1회가 20초 타임아웃을 넘기자 실패 결과가 성공과 같은 TTL(운영 1800초)로 캐시돼, 공개 API가 30분간 engine=false로 오염됐다. 워치독은 기본 /ready(캐시 200)만 보고 엔진을 healthy로 오판해 API만 재시작하는 무한 루프에 빠졌다. 실패는 짧은 TTL 뒤 재프로브해야 한다. """ import asyncio import unittest from unittest.mock import patch from fastapi.testclient import TestClient from engine_gateway import gateway class _FakeProbe: """EngineSession 대역: 호출 순서별로 미리 정한 결과를 돌려준다.""" def __init__(self, outcomes, calls): self._outcomes = outcomes self._calls = calls async def start(self): return None async def turn(self, content, timeout=None): index = min(len(self._calls), len(self._outcomes) - 1) outcome = self._outcomes[index] self._calls.append(outcome) if isinstance(outcome, Exception): raise outcome return {"text": outcome} async def failure_detail(self, reason): return reason async def close(self): return None def _probe_factory(outcomes, calls): def factory(*args, **kwargs): return _FakeProbe(outcomes, calls) return factory class ReadyFailureCacheTest(unittest.TestCase): def setUp(self): gateway._READY_CACHE.clear() self.addCleanup(gateway._READY_CACHE.clear) def test_failed_probe_is_retried_after_failure_ttl(self): calls = [] with ( patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", ""), patch.object(gateway, "READY_TTL_SECONDS", 1800.0), patch.object(gateway, "READY_FAILURE_TTL_SECONDS", 0.0), patch.object( gateway, "EngineSession", _probe_factory([asyncio.TimeoutError(), "OK"], calls), ), ): client = TestClient(gateway.app) first = client.get("/ready") second = client.get("/ready") self.assertEqual(first.status_code, 503) self.assertEqual(second.status_code, 200) self.assertEqual(len(calls), 2) self.assertFalse(second.json()["cached"]) def test_successful_probe_stays_cached_for_full_ttl(self): calls = [] with ( patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", ""), patch.object(gateway, "READY_TTL_SECONDS", 1800.0), patch.object(gateway, "READY_FAILURE_TTL_SECONDS", 0.0), patch.object( gateway, "EngineSession", _probe_factory(["OK", asyncio.TimeoutError()], calls), ), ): client = TestClient(gateway.app) first = client.get("/ready") second = client.get("/ready") self.assertEqual(first.status_code, 200) self.assertEqual(second.status_code, 200) self.assertEqual(len(calls), 1) self.assertTrue(second.json()["cached"]) def test_failed_probe_is_not_hammered_within_failure_ttl(self): calls = [] with ( patch.object(gateway, "ENGINE_GATEWAY_SHARED_SECRET", ""), patch.object(gateway, "READY_TTL_SECONDS", 1800.0), patch.object(gateway, "READY_FAILURE_TTL_SECONDS", 60.0), patch.object( gateway, "EngineSession", _probe_factory([asyncio.TimeoutError(), "OK"], calls), ), ): client = TestClient(gateway.app) first = client.get("/ready") second = client.get("/ready") self.assertEqual(first.status_code, 503) self.assertEqual(second.status_code, 503) self.assertEqual(len(calls), 1) self.assertTrue(second.json()["cached"]) def test_defaults_give_cold_high_effort_spawn_headroom(self): # reasoning_effort=high 콜드 스폰은 20초를 자주 넘긴다(2026-08-18 실측). self.assertGreaterEqual(gateway.READY_TIMEOUT_SECONDS, 45.0) # 실패 캐시는 1분 넘게 유지하면 안 된다 — 복구 지연이 곧 공개 장애 시간이 된다. self.assertLessEqual(gateway.READY_FAILURE_TTL_SECONDS, 60.0)