494 lines
20 KiB
Python
494 lines
20 KiB
Python
"""Regression tests for DB outage runtime fallback policy."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
import time
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from . import auth_sessions, session_persistence
|
|
from .config import Settings, settings
|
|
from .deps import Principal, Role
|
|
from .routes import admin as admin_routes
|
|
from .routes import eval as eval_routes
|
|
from .routes import kb as kb_routes
|
|
from .routes import users as users_routes
|
|
|
|
|
|
@contextmanager
|
|
def environment(value: str):
|
|
previous = settings.environment
|
|
settings.environment = value
|
|
try:
|
|
yield
|
|
finally:
|
|
settings.environment = previous
|
|
|
|
|
|
class RuntimeFallbackPolicyTest(unittest.IsolatedAsyncioTestCase):
|
|
async def asyncSetUp(self) -> None:
|
|
auth_sessions._sessions.clear()
|
|
auth_sessions._users.clear()
|
|
auth_sessions._email_index.clear()
|
|
auth_sessions._inactive_emails.clear()
|
|
users_routes._preferences.clear()
|
|
admin_routes._ENGINE_CONFIG = None
|
|
|
|
async def asyncTearDown(self) -> None:
|
|
admin_routes._ENGINE_CONFIG = None
|
|
|
|
async def test_dev_allows_auth_registry_fallback_when_db_pool_missing(self) -> None:
|
|
with environment("dev"):
|
|
users, durable = await auth_sessions.list_managed_users()
|
|
|
|
self.assertFalse(durable)
|
|
self.assertEqual(users, [])
|
|
|
|
async def test_staging_blocks_auth_registry_fallback_when_db_pool_missing(self) -> None:
|
|
with environment("staging"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await auth_sessions.list_managed_users()
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in staging", caught.exception.detail)
|
|
|
|
async def test_prod_blocks_browser_session_creation_when_db_pool_missing(self) -> None:
|
|
with environment("prod"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await auth_sessions.create_session(
|
|
email="learner@hs.ac.kr",
|
|
display_name="Learner",
|
|
role="learner",
|
|
)
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in prod", caught.exception.detail)
|
|
|
|
async def test_staging_blocks_session_store_fallback_when_db_pool_missing(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000001",
|
|
role=Role.LEARNER,
|
|
cohort_ids=[],
|
|
email="learner@hs.ac.kr",
|
|
display_name="Learner",
|
|
)
|
|
with environment("staging"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await session_persistence.list_sessions(principal)
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in staging", caught.exception.detail)
|
|
|
|
async def test_staging_blocks_eval_store_cache_fallback_when_db_pool_missing(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000002",
|
|
role=Role.TEACHER,
|
|
cohort_ids=[],
|
|
email="teacher@hs.ac.kr",
|
|
display_name="Teacher",
|
|
)
|
|
with environment("staging"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await eval_routes.get_session_evaluation("missing-session-id", principal)
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in staging", caught.exception.detail)
|
|
|
|
async def test_staging_kb_search_fails_closed_when_db_pool_missing(self) -> None:
|
|
with environment("staging"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await kb_routes.search(kb_routes.KBSearchRequest(query="rapport"))
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("DB not ready", caught.exception.detail)
|
|
|
|
async def test_staging_blocks_user_preferences_fallback_when_db_pool_missing(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000003",
|
|
role=Role.LEARNER,
|
|
cohort_ids=[],
|
|
email="learner@hs.ac.kr",
|
|
display_name="Learner",
|
|
)
|
|
with environment("staging"):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await users_routes.get_preferences(principal)
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in staging", caught.exception.detail)
|
|
|
|
async def test_dev_allows_user_preferences_fallback_when_db_pool_missing(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000004",
|
|
role=Role.LEARNER,
|
|
cohort_ids=[],
|
|
email="learner@hs.ac.kr",
|
|
display_name="Learner",
|
|
)
|
|
with environment("dev"):
|
|
prefs = await users_routes.get_preferences(principal)
|
|
|
|
self.assertEqual(prefs.voice_preset_id, "soft-young-fem")
|
|
|
|
async def test_prod_blocks_runtime_schema_bootstrap_ddl_when_schema_incomplete(self) -> None:
|
|
class IncompleteConn:
|
|
async def fetchrow(self, *args, **kwargs):
|
|
return {
|
|
"has_user_columns": False,
|
|
"has_persona_triggers": False,
|
|
"has_persona_voice_map": False,
|
|
"has_auth_session": False,
|
|
"has_preferences": False,
|
|
"has_engine_config": False,
|
|
"has_session_columns": False,
|
|
"has_state_columns": False,
|
|
"has_stage_defs": False,
|
|
"has_admin_health_event": False,
|
|
"has_support_ticket": False,
|
|
"has_admin_health_event_policies": False,
|
|
"has_support_ticket_policies": False,
|
|
"has_session_write_policies": False,
|
|
"removed_old_session_policy": False,
|
|
"has_turn_write_policies": False,
|
|
"removed_old_turn_policy": False,
|
|
}
|
|
|
|
async def execute(self, *args, **kwargs):
|
|
raise AssertionError("prod startup must not run owner DDL")
|
|
|
|
class IncompleteAcquire:
|
|
async def __aenter__(self):
|
|
return IncompleteConn()
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return None
|
|
|
|
class IncompletePool:
|
|
def acquire(self):
|
|
return IncompleteAcquire()
|
|
|
|
with environment("prod"), patch.object(auth_sessions, "get_pool", return_value=IncompletePool()):
|
|
with self.assertRaises(RuntimeError) as caught:
|
|
await auth_sessions.ensure_runtime_tables()
|
|
|
|
self.assertIn("runtime DB schema is incomplete", str(caught.exception))
|
|
|
|
async def test_staging_uses_env_engine_config_when_row_missing(self) -> None:
|
|
class EmptyConfigConn:
|
|
async def fetchrow(self, *args, **kwargs):
|
|
return None
|
|
|
|
class EmptyConfigAcquire:
|
|
async def __aenter__(self):
|
|
return EmptyConfigConn()
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return None
|
|
|
|
class EmptyConfigPool:
|
|
def acquire(self):
|
|
return EmptyConfigAcquire()
|
|
|
|
with environment("staging"), patch.object(admin_routes, "get_pool", return_value=EmptyConfigPool()):
|
|
config = await admin_routes._current_engine_config()
|
|
|
|
self.assertFalse(config.durable)
|
|
self.assertEqual(config.source, "runtime_default")
|
|
self.assertEqual(config.engine_mode, settings.engine_mode)
|
|
self.assertEqual(config.engine_url, settings.engine_url)
|
|
|
|
async def test_staging_blocks_admin_engine_config_when_db_unavailable(self) -> None:
|
|
class BrokenConfigPool:
|
|
def acquire(self):
|
|
raise RuntimeError("db unavailable")
|
|
|
|
with environment("staging"), patch.object(admin_routes, "get_pool", return_value=BrokenConfigPool()):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await admin_routes._current_engine_config()
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in staging", caught.exception.detail)
|
|
|
|
async def test_prod_admin_health_marks_db_down_when_persistence_unavailable(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000005",
|
|
role=Role.ADMIN,
|
|
cohort_ids=[],
|
|
email="admin@twentyoz.kr",
|
|
display_name="Admin",
|
|
)
|
|
engine_config = admin_routes.AdminEngineConfigResponse(
|
|
engine_mode="claude_cli",
|
|
engine_url="http://127.0.0.1:9099",
|
|
model="gateway-default",
|
|
durable=True,
|
|
source="database",
|
|
)
|
|
|
|
with (
|
|
environment("prod"),
|
|
patch.object(admin_routes, "_current_engine_config", AsyncMock(return_value=engine_config)),
|
|
patch.object(admin_routes, "healthcheck", AsyncMock(return_value=False)),
|
|
patch.object(admin_routes.engine_client, "health_detail", AsyncMock(return_value={"ok": True})),
|
|
patch.object(admin_routes.voice_service, "is_available", return_value=False),
|
|
):
|
|
health = await admin_routes.admin_health(principal)
|
|
|
|
db = next(service for service in health.services if service.key == "db")
|
|
self.assertEqual(db.status, "down")
|
|
self.assertEqual(db.metric, "저장소 중단")
|
|
self.assertIn("DB 저장소", db.detail)
|
|
|
|
async def test_dev_admin_health_labels_db_fallback_as_non_durable_runtime(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000006",
|
|
role=Role.ADMIN,
|
|
cohort_ids=[],
|
|
email="admin@twentyoz.kr",
|
|
display_name="Admin",
|
|
)
|
|
engine_config = admin_routes.AdminEngineConfigResponse(
|
|
engine_mode="claude_cli",
|
|
engine_url="http://127.0.0.1:9099",
|
|
model="gateway-default",
|
|
durable=False,
|
|
source="runtime_cache",
|
|
)
|
|
|
|
with (
|
|
environment("dev"),
|
|
patch.object(admin_routes, "_current_engine_config", AsyncMock(return_value=engine_config)),
|
|
patch.object(admin_routes, "healthcheck", AsyncMock(return_value=False)),
|
|
patch.object(admin_routes.engine_client, "health_detail", AsyncMock(return_value={"ok": True})),
|
|
patch.object(admin_routes.voice_service, "is_available", return_value=False),
|
|
):
|
|
health = await admin_routes.admin_health(principal)
|
|
|
|
db = next(service for service in health.services if service.key == "db")
|
|
self.assertEqual(db.status, "degraded")
|
|
self.assertEqual(db.metric, "비영구 런타임 기록")
|
|
self.assertIn("비영구 개발 런타임 기록", db.detail)
|
|
|
|
async def test_dev_admin_usage_falls_back_to_runtime_store(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000007",
|
|
role=Role.ADMIN,
|
|
cohort_ids=[],
|
|
email="admin@twentyoz.kr",
|
|
display_name="Admin",
|
|
)
|
|
now = time.time()
|
|
fake_session = SimpleNamespace(
|
|
turns=[
|
|
SimpleNamespace(
|
|
speaker="client",
|
|
created_at=now,
|
|
llm_provider="claude_cli",
|
|
model="gateway-default",
|
|
tokens_in=11,
|
|
tokens_out=13,
|
|
cost_usd=0.0042,
|
|
),
|
|
SimpleNamespace(
|
|
speaker="client",
|
|
created_at=now,
|
|
llm_provider=None,
|
|
model=None,
|
|
tokens_in=None,
|
|
tokens_out=None,
|
|
cost_usd=None,
|
|
),
|
|
SimpleNamespace(
|
|
speaker="counselor",
|
|
created_at=now,
|
|
llm_provider="ignored",
|
|
model="ignored",
|
|
tokens_in=100,
|
|
tokens_out=100,
|
|
cost_usd=9.0,
|
|
),
|
|
]
|
|
)
|
|
|
|
with (
|
|
environment("dev"),
|
|
patch.object(
|
|
admin_routes,
|
|
"_usage_from_database",
|
|
AsyncMock(side_effect=RuntimeError("db unavailable")),
|
|
),
|
|
patch.object(admin_routes.store, "list", return_value=[fake_session]),
|
|
patch.object(admin_routes.settings, "admin_usage_budget_usd", 0.005),
|
|
):
|
|
usage = await admin_routes.admin_usage(principal, window_days=7)
|
|
|
|
self.assertEqual(usage.source, "server_session_registry")
|
|
self.assertFalse(usage.durable)
|
|
self.assertEqual(usage.total_turns, 2)
|
|
self.assertEqual(usage.metered_turns, 1)
|
|
self.assertEqual(usage.tokens_in, 11)
|
|
self.assertEqual(usage.tokens_out, 13)
|
|
self.assertAlmostEqual(usage.cost_usd, 0.0042)
|
|
self.assertEqual(usage.budget.status, "warn")
|
|
self.assertAlmostEqual(usage.budget.limit_usd, 0.005)
|
|
self.assertAlmostEqual(usage.budget.used_ratio, 0.84)
|
|
self.assertEqual(usage.by_provider[0].provider, "claude_cli")
|
|
self.assertEqual(usage.by_provider[0].turns, 1)
|
|
|
|
async def test_prod_admin_usage_rejects_runtime_store_fallback(self) -> None:
|
|
principal = Principal(
|
|
user_id="00000000-0000-0000-0000-000000000008",
|
|
role=Role.ADMIN,
|
|
cohort_ids=[],
|
|
email="admin@twentyoz.kr",
|
|
display_name="Admin",
|
|
)
|
|
|
|
with (
|
|
environment("prod"),
|
|
patch.object(
|
|
admin_routes,
|
|
"_usage_from_database",
|
|
AsyncMock(side_effect=RuntimeError("db unavailable")),
|
|
),
|
|
):
|
|
with self.assertRaises(HTTPException) as caught:
|
|
await admin_routes.admin_usage(principal, window_days=7)
|
|
|
|
self.assertEqual(caught.exception.status_code, 503)
|
|
self.assertIn("runtime fallback is disabled in prod", caught.exception.detail)
|
|
|
|
def test_non_dev_rejects_fixture_runtime_flags(self) -> None:
|
|
with self.assertRaises(ValueError) as caught:
|
|
Settings(
|
|
environment="staging",
|
|
auth_dev_login_enabled=True,
|
|
auto_seed_personas=True,
|
|
allow_seed_persona_fallback=True,
|
|
voice_poc_sample_tts_enabled=True,
|
|
)
|
|
|
|
self.assertIn("AUTH_DEV_LOGIN_ENABLED", str(caught.exception))
|
|
self.assertIn("AUTO_SEED_PERSONAS", str(caught.exception))
|
|
self.assertIn("ALLOW_SEED_PERSONA_FALLBACK", str(caught.exception))
|
|
self.assertIn("VIGNETTE_VOICE_POC_SAMPLE_TTS", str(caught.exception))
|
|
|
|
def test_non_dev_rejects_missing_public_runtime_config(self) -> None:
|
|
with self.assertRaises(ValueError) as caught:
|
|
Settings(
|
|
environment="prod",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
oauth_google_client_id="",
|
|
oauth_google_client_secret="",
|
|
session_secret="dev-insecure-change-me",
|
|
frontend_base_url="http://localhost:5173",
|
|
cors_origins=["http://localhost:5173"],
|
|
)
|
|
|
|
error = str(caught.exception)
|
|
self.assertIn("OAUTH_GOOGLE_CLIENT_ID", error)
|
|
self.assertIn("OAUTH_GOOGLE_CLIENT_SECRET", error)
|
|
self.assertIn("SESSION_SECRET", error)
|
|
self.assertIn("FRONTEND_BASE_URL", error)
|
|
|
|
def test_non_dev_accepts_public_runtime_config(self) -> None:
|
|
cfg = Settings(
|
|
environment="staging",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
session_secret="staging-secret-change-me",
|
|
oauth_google_client_id="google-client-id",
|
|
oauth_google_client_secret="google-client-secret",
|
|
frontend_base_url="https://vignette.chanpaca.net",
|
|
cors_origins=["https://vignette.chanpaca.net"],
|
|
voice_poc_sample_tts_enabled=False,
|
|
)
|
|
|
|
self.assertEqual(cfg.environment, "staging")
|
|
self.assertEqual(cfg.cors_origins, ["https://vignette.chanpaca.net"])
|
|
|
|
def test_non_dev_accepts_vnet_frontend_origin_map(self) -> None:
|
|
cfg = Settings(
|
|
environment="prod",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
session_secret="prod-secret-change-me",
|
|
oauth_google_client_id="google-client-id",
|
|
oauth_google_client_secret="google-client-secret",
|
|
frontend_base_url="https://vignette.chanpaca.net",
|
|
frontend_origin_map={
|
|
"api-vignette.chanpaca.net": "https://vignette.chanpaca.net",
|
|
"api-vnet.18ka.net": "https://vnet.18ka.net",
|
|
},
|
|
cors_origins=["https://vignette.chanpaca.net", "https://vnet.18ka.net"],
|
|
voice_poc_sample_tts_enabled=False,
|
|
)
|
|
|
|
self.assertEqual(cfg.frontend_origin_map["api-vnet.18ka.net"], "https://vnet.18ka.net")
|
|
self.assertIn("https://vnet.18ka.net", cfg.cors_origins)
|
|
|
|
def test_non_dev_rejects_local_frontend_origin_map(self) -> None:
|
|
with self.assertRaises(ValueError) as caught:
|
|
Settings(
|
|
environment="prod",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
session_secret="prod-secret-change-me",
|
|
oauth_google_client_id="google-client-id",
|
|
oauth_google_client_secret="google-client-secret",
|
|
frontend_base_url="https://vignette.chanpaca.net",
|
|
frontend_origin_map={"api.example.test": "http://localhost:9999"},
|
|
cors_origins=["https://vignette.chanpaca.net"],
|
|
)
|
|
|
|
self.assertIn("FRONTEND_ORIGIN_MAP", str(caught.exception))
|
|
|
|
def test_non_dev_accepts_explicit_local_vite_cors_ports(self) -> None:
|
|
cfg = Settings(
|
|
environment="prod",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
session_secret="prod-secret-change-me",
|
|
oauth_google_client_id="google-client-id",
|
|
oauth_google_client_secret="google-client-secret",
|
|
frontend_base_url="https://vignette.chanpaca.net",
|
|
cors_origins=[
|
|
"https://vignette.chanpaca.net",
|
|
"http://localhost:5170",
|
|
"http://127.0.0.1:5180",
|
|
],
|
|
voice_poc_sample_tts_enabled=False,
|
|
)
|
|
|
|
self.assertIn("http://localhost:5170", cfg.cors_origins)
|
|
self.assertIn("http://127.0.0.1:5180", cfg.cors_origins)
|
|
|
|
def test_non_dev_rejects_unscoped_local_cors_ports(self) -> None:
|
|
with self.assertRaises(ValueError) as caught:
|
|
Settings(
|
|
environment="prod",
|
|
auth_dev_login_enabled=False,
|
|
auto_seed_personas=False,
|
|
allow_seed_persona_fallback=False,
|
|
session_secret="prod-secret-change-me",
|
|
oauth_google_client_id="google-client-id",
|
|
oauth_google_client_secret="google-client-secret",
|
|
frontend_base_url="https://vignette.chanpaca.net",
|
|
cors_origins=["http://localhost:5181"],
|
|
)
|
|
|
|
self.assertIn("CORS_ORIGINS", str(caught.exception))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|