370 lines
15 KiB
Python
370 lines
15 KiB
Python
"""Admin health sample retention/rollup tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
from .services import admin_health_maintenance as maintenance
|
|
|
|
|
|
_VALID_ROLLUP_DELETE_POLICY = {
|
|
"policy_name": "p_admin_health_daily_rollup_delete",
|
|
"command": "d",
|
|
"permissive": True,
|
|
"rls_enabled": True,
|
|
"qual": "(app.current_role_name() = 'admin'::text)",
|
|
}
|
|
|
|
|
|
class _Acquire:
|
|
def __init__(self, conn):
|
|
self.conn = conn
|
|
self.exit_error = None
|
|
|
|
async def __aenter__(self):
|
|
return self.conn
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
self.exit_error = exc
|
|
return None
|
|
|
|
|
|
class AdminHealthMaintenanceTest(unittest.IsolatedAsyncioTestCase):
|
|
async def test_dry_run_counts_rollup_and_prunable_events_without_mutation(self) -> None:
|
|
class Conn:
|
|
def __init__(self):
|
|
self.fetch_calls = []
|
|
|
|
async def fetchrow(self, query, *args, **kwargs):
|
|
self.fetch_calls.append((query, args))
|
|
if len(self.fetch_calls) == 1:
|
|
return {"event_count": 12, "bucket_count": 3}
|
|
if len(self.fetch_calls) == 2:
|
|
return {"event_count": 4}
|
|
return {"rollup_count": 2}
|
|
|
|
async def fetch(self, *args, **kwargs):
|
|
raise AssertionError("dry-run must not upsert rollups")
|
|
|
|
async def execute(self, *args, **kwargs):
|
|
raise AssertionError("dry-run must not delete raw events")
|
|
|
|
conn = Conn()
|
|
with unittest.mock.patch.object(maintenance, "acquire", return_value=_Acquire(conn)):
|
|
result = await maintenance.maintain_admin_health_events(
|
|
rollup_days=2,
|
|
apply=False,
|
|
)
|
|
|
|
self.assertFalse(result.applied)
|
|
self.assertEqual(result.rollup_event_count, 12)
|
|
self.assertEqual(result.rollup_bucket_count, 3)
|
|
self.assertEqual(result.prunable_event_count, 4)
|
|
self.assertEqual(result.prunable_rollup_count, 2)
|
|
self.assertEqual(result.rollup_retention_days, 365)
|
|
self.assertEqual(result.upserted_rollups, 0)
|
|
self.assertEqual(result.deleted_events, 0)
|
|
self.assertEqual(result.deleted_rollups, 0)
|
|
self.assertEqual(conn.fetch_calls[0][1], (2,))
|
|
self.assertEqual(result.retention_days, 90)
|
|
self.assertEqual(conn.fetch_calls[1][1], (90,))
|
|
self.assertEqual(conn.fetch_calls[2][1], (365,))
|
|
self.assertIn("observed_at::date < current_date - $1::int", conn.fetch_calls[0][0])
|
|
self.assertIn("rollup_date < current_date - $1::int", conn.fetch_calls[2][0])
|
|
|
|
async def test_apply_upserts_rollups_before_pruning_raw_events(self) -> None:
|
|
calls = []
|
|
|
|
class Conn:
|
|
async def fetchrow(self, query, *args, **kwargs):
|
|
calls.append(("fetchrow", query, args))
|
|
fetchrow_count = len([call for call in calls if call[0] == "fetchrow"])
|
|
if fetchrow_count == 1:
|
|
return {"event_count": 7, "bucket_count": 2}
|
|
if fetchrow_count == 2:
|
|
return {"event_count": 5}
|
|
return {"rollup_count": 3}
|
|
|
|
async def fetch(self, query, *args, **kwargs):
|
|
calls.append(("fetch", query, args))
|
|
if "FROM pg_policy AS policy" in query:
|
|
return [_VALID_ROLLUP_DELETE_POLICY]
|
|
return [1, 1]
|
|
|
|
async def execute(self, query, *args, **kwargs):
|
|
calls.append(("execute", query, args))
|
|
return "DELETE 5" if "admin_health_event" in query else "DELETE 3"
|
|
|
|
with unittest.mock.patch.object(maintenance, "acquire", return_value=_Acquire(Conn())):
|
|
result = await maintenance.maintain_admin_health_events(
|
|
rollup_days=3,
|
|
retention_days=30,
|
|
apply=True,
|
|
)
|
|
|
|
self.assertTrue(result.applied)
|
|
self.assertEqual(result.upserted_rollups, 2)
|
|
self.assertEqual(result.deleted_events, 5)
|
|
self.assertEqual(result.deleted_rollups, 3)
|
|
self.assertEqual(
|
|
[call[0] for call in calls],
|
|
["fetchrow", "fetchrow", "fetchrow", "fetch", "fetch", "execute", "execute"],
|
|
)
|
|
self.assertIn("pg_get_expr(policy.polqual, policy.polrelid)", calls[3][1])
|
|
self.assertNotIn("LIKE", calls[3][1])
|
|
self.assertIn("INSERT INTO app.admin_health_daily_rollup", calls[4][1])
|
|
self.assertIn("last_down_at", calls[4][1])
|
|
self.assertIn("DELETE FROM app.admin_health_event", calls[5][1])
|
|
self.assertIn("DELETE FROM app.admin_health_daily_rollup", calls[6][1])
|
|
self.assertEqual(calls[6][2], (365,))
|
|
|
|
def test_invalid_windows_are_rejected(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
maintenance._validate_windows(
|
|
rollup_days=0,
|
|
retention_days=30,
|
|
rollup_retention_days=365,
|
|
)
|
|
with self.assertRaises(ValueError):
|
|
maintenance._validate_windows(
|
|
rollup_days=7,
|
|
retention_days=2,
|
|
rollup_retention_days=365,
|
|
)
|
|
with self.assertRaises(ValueError):
|
|
maintenance._validate_windows(
|
|
rollup_days=7,
|
|
retention_days=30,
|
|
rollup_retention_days=29,
|
|
)
|
|
|
|
async def test_rollup_delete_policy_requires_exact_admin_expression(self) -> None:
|
|
class Conn:
|
|
def __init__(self, policies):
|
|
self.policies = policies
|
|
self.query = ""
|
|
|
|
async def fetch(self, query, *args, **kwargs):
|
|
self.query = query
|
|
return self.policies
|
|
|
|
valid = Conn([_VALID_ROLLUP_DELETE_POLICY])
|
|
await maintenance._require_rollup_delete_policy(valid)
|
|
self.assertIn("policy.polcmd::text AS command", valid.query)
|
|
self.assertIn("pg_get_expr(policy.polqual, policy.polrelid)", valid.query)
|
|
self.assertIn("policy.polpermissive IS TRUE", valid.query)
|
|
self.assertIn("relation.relrowsecurity AS rls_enabled", valid.query)
|
|
self.assertIn("policy.polcmd IN ('d', '*')", valid.query)
|
|
self.assertNotIn("LIKE", valid.query)
|
|
|
|
invalid_policy_sets = (
|
|
[
|
|
{
|
|
**_VALID_ROLLUP_DELETE_POLICY,
|
|
"qual": "(app.current_role_name() = 'admin'::text) OR true",
|
|
}
|
|
],
|
|
[
|
|
{
|
|
**_VALID_ROLLUP_DELETE_POLICY,
|
|
"qual": "(app.current_role_name() = 'learner'::text)",
|
|
}
|
|
],
|
|
[
|
|
{
|
|
**_VALID_ROLLUP_DELETE_POLICY,
|
|
"permissive": 1,
|
|
}
|
|
],
|
|
[
|
|
{
|
|
**_VALID_ROLLUP_DELETE_POLICY,
|
|
"rls_enabled": False,
|
|
}
|
|
],
|
|
[
|
|
_VALID_ROLLUP_DELETE_POLICY,
|
|
{
|
|
"policy_name": "p_admin_health_daily_rollup_extra_delete",
|
|
"command": "d",
|
|
"permissive": True,
|
|
"qual": "true",
|
|
},
|
|
],
|
|
[
|
|
_VALID_ROLLUP_DELETE_POLICY,
|
|
{
|
|
"policy_name": "p_admin_health_daily_rollup_all",
|
|
"command": "*",
|
|
"permissive": True,
|
|
"qual": "true",
|
|
},
|
|
],
|
|
)
|
|
for policies in invalid_policy_sets:
|
|
with self.subTest(policies=policies):
|
|
with self.assertRaisesRegex(RuntimeError, "missing, invalid, or permissive"):
|
|
await maintenance._require_rollup_delete_policy(Conn(policies))
|
|
|
|
async def test_apply_stops_before_dml_when_rollup_delete_policy_is_missing(
|
|
self,
|
|
) -> None:
|
|
class Conn:
|
|
def __init__(self):
|
|
self.fetchrow_calls = 0
|
|
|
|
async def fetchrow(self, query, *args, **kwargs):
|
|
self.fetchrow_calls += 1
|
|
if self.fetchrow_calls == 1:
|
|
return {"event_count": 1, "bucket_count": 1}
|
|
if self.fetchrow_calls == 2:
|
|
return {"event_count": 0}
|
|
if self.fetchrow_calls == 3:
|
|
return {"rollup_count": 0}
|
|
|
|
async def fetch(self, query, *args, **kwargs):
|
|
if "FROM pg_policy AS policy" in query:
|
|
return []
|
|
raise AssertionError("missing policy must stop before rollup upsert")
|
|
|
|
async def execute(self, *args, **kwargs):
|
|
raise AssertionError("missing policy must stop before deletion")
|
|
|
|
context = _Acquire(Conn())
|
|
with unittest.mock.patch.object(maintenance, "acquire", return_value=context):
|
|
with self.assertRaisesRegex(
|
|
RuntimeError,
|
|
"owner must apply migration 25_admin_health_rollup_retention.sql",
|
|
):
|
|
await maintenance.maintain_admin_health_events(
|
|
rollup_days=90,
|
|
retention_days=90,
|
|
rollup_retention_days=365,
|
|
apply=True,
|
|
)
|
|
self.assertIsInstance(context.exit_error, RuntimeError)
|
|
|
|
async def test_repeat_apply_keeps_rollup_upsert_and_cutoff_contract(self) -> None:
|
|
calls = []
|
|
|
|
class Conn:
|
|
async def fetchrow(self, query, *args, **kwargs):
|
|
calls.append(("fetchrow", query, args))
|
|
if "admin_health_daily_rollup" in query:
|
|
return {"rollup_count": 0}
|
|
if "COUNT(DISTINCT" in query:
|
|
return {"event_count": 1, "bucket_count": 1}
|
|
return {"event_count": 0}
|
|
|
|
async def fetch(self, query, *args, **kwargs):
|
|
calls.append(("fetch", query, args))
|
|
if "FROM pg_policy AS policy" in query:
|
|
return [_VALID_ROLLUP_DELETE_POLICY]
|
|
return [1]
|
|
|
|
async def execute(self, query, *args, **kwargs):
|
|
calls.append(("execute", query, args))
|
|
return "DELETE 0"
|
|
|
|
context = _Acquire(Conn())
|
|
with unittest.mock.patch.object(maintenance, "acquire", return_value=context):
|
|
first = await maintenance.maintain_admin_health_events(
|
|
rollup_days=90,
|
|
retention_days=90,
|
|
rollup_retention_days=365,
|
|
apply=True,
|
|
)
|
|
second = await maintenance.maintain_admin_health_events(
|
|
rollup_days=90,
|
|
retention_days=90,
|
|
rollup_retention_days=365,
|
|
apply=True,
|
|
)
|
|
|
|
self.assertEqual(first.upserted_rollups, second.upserted_rollups)
|
|
upserts = [
|
|
call
|
|
for call in calls
|
|
if call[0] == "fetch" and "INSERT INTO app.admin_health_daily_rollup" in call[1]
|
|
]
|
|
self.assertEqual(2, len(upserts))
|
|
self.assertTrue(
|
|
all("ON CONFLICT (rollup_date, environment, engine_mode, service_key)" in call[1] for call in upserts)
|
|
)
|
|
rollup_prunes = [
|
|
call
|
|
for call in calls
|
|
if call[0] == "execute" and "admin_health_daily_rollup" in call[1]
|
|
]
|
|
self.assertEqual(2, len(rollup_prunes))
|
|
self.assertTrue(
|
|
all("rollup_date < current_date - $1::int" in call[1] for call in rollup_prunes)
|
|
)
|
|
self.assertTrue(all(call[2] == (365,) for call in rollup_prunes))
|
|
|
|
async def test_raw_prune_failure_stops_before_rollup_prune(self) -> None:
|
|
calls = []
|
|
|
|
class Conn:
|
|
async def fetchrow(self, query, *args, **kwargs):
|
|
if "admin_health_daily_rollup" in query:
|
|
return {"rollup_count": 0}
|
|
if "COUNT(DISTINCT" in query:
|
|
return {"event_count": 1, "bucket_count": 1}
|
|
return {"event_count": 1}
|
|
|
|
async def fetch(self, query, *args, **kwargs):
|
|
calls.append(("fetch", query))
|
|
if "FROM pg_policy AS policy" in query:
|
|
return [_VALID_ROLLUP_DELETE_POLICY]
|
|
return [1]
|
|
|
|
async def execute(self, query, *args, **kwargs):
|
|
calls.append(("execute", query))
|
|
raise RuntimeError("raw prune failed")
|
|
|
|
context = _Acquire(Conn())
|
|
with unittest.mock.patch.object(maintenance, "acquire", return_value=context):
|
|
with self.assertRaisesRegex(RuntimeError, "raw prune failed"):
|
|
await maintenance.maintain_admin_health_events(
|
|
rollup_days=90,
|
|
retention_days=90,
|
|
rollup_retention_days=365,
|
|
apply=True,
|
|
)
|
|
self.assertIsInstance(context.exit_error, RuntimeError)
|
|
self.assertEqual([call[0] for call in calls], ["fetch", "fetch", "execute"])
|
|
self.assertIn("FROM pg_policy AS policy", calls[0][1])
|
|
self.assertIn("DELETE FROM app.admin_health_event", calls[2][1])
|
|
|
|
def test_schema_defines_rollup_table_and_rls(self) -> None:
|
|
from pathlib import Path
|
|
|
|
root = Path(__file__).resolve().parents[3]
|
|
schema = (root / "infra" / "db" / "init" / "05_runtime_auth.sql").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
self.assertIn("CREATE TABLE IF NOT EXISTS app.admin_health_daily_rollup", schema)
|
|
self.assertIn("PRIMARY KEY (rollup_date, environment, engine_mode, service_key)", schema)
|
|
self.assertIn("last_down_at TIMESTAMPTZ", schema)
|
|
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_select", schema)
|
|
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_update", schema)
|
|
self.assertIn("CREATE POLICY p_admin_health_event_delete", schema)
|
|
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_delete", schema)
|
|
self.assertIn("USING (app.current_role_name() = 'admin');", schema)
|
|
|
|
migration = (
|
|
root / "infra" / "db" / "init" / "25_admin_health_rollup_retention.sql"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn("DROP POLICY IF EXISTS p_admin_health_daily_rollup_delete", migration)
|
|
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_delete", migration)
|
|
self.assertIn("FOR DELETE", migration)
|
|
self.assertIn("app.current_role_name() = 'admin'", migration)
|
|
self.assertNotIn("learner", migration)
|
|
|
|
bootstrap = (root / "apps" / "api" / "app" / "auth_sessions.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_delete", bootstrap)
|