158 lines
6 KiB
Python
158 lines
6 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from .services.usage_report import REPORT_SCHEMA, build_model_cost_report
|
|
|
|
|
|
class UsageReportTest(unittest.TestCase):
|
|
def test_model_cost_report_derives_cost_efficiency_and_warnings(self) -> None:
|
|
report = build_model_cost_report(
|
|
{
|
|
"source": "database",
|
|
"durable": True,
|
|
"window_days": 7,
|
|
"total_turns": 5,
|
|
"metered_turns": 4,
|
|
"token_metered_turns": 3,
|
|
"token_unmetered_turns": 1,
|
|
"tokens_in": 300,
|
|
"tokens_out": 100,
|
|
"cost_usd": 0.4,
|
|
"budget": {
|
|
"status": "warn",
|
|
"limit_usd": 0.5,
|
|
"used_ratio": 0.8,
|
|
"remaining_usd": 0.1,
|
|
},
|
|
"evaluator_cache": {
|
|
"enabled": True,
|
|
"requests": 10,
|
|
"hits": 2,
|
|
"misses": 8,
|
|
"hit_rate": 0.2,
|
|
},
|
|
"by_provider": [
|
|
{
|
|
"provider": "cheap",
|
|
"model": "mini",
|
|
"turns": 3,
|
|
"token_metered_turns": 2,
|
|
"token_unmetered_turns": 1,
|
|
"tokens_in": 100,
|
|
"tokens_out": 50,
|
|
"cost_usd": 0.04,
|
|
},
|
|
{
|
|
"provider": "claude_cli",
|
|
"model": "opus",
|
|
"turns": 1,
|
|
"token_metered_turns": 1,
|
|
"token_unmetered_turns": 0,
|
|
"tokens_in": 200,
|
|
"tokens_out": 50,
|
|
"cost_usd": 0.36,
|
|
},
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(report["schema"], REPORT_SCHEMA)
|
|
self.assertEqual(report["summary"]["metered_coverage"], 0.8)
|
|
self.assertEqual(report["summary"]["token_metered_coverage"], 0.75)
|
|
self.assertEqual(report["summary"]["cost_per_1k_tokens_usd"], 1.0)
|
|
self.assertEqual(report["models"][0]["provider"], "claude_cli")
|
|
self.assertEqual(report["models"][0]["cost_share"], 0.9)
|
|
self.assertEqual(report["models"][0]["cost_per_1k_tokens_usd"], 1.44)
|
|
self.assertEqual(report["models"][1]["cost_per_turn_usd"], 0.013333)
|
|
self.assertIn("partial_metering", report["warnings"])
|
|
self.assertIn("partial_token_metering", report["warnings"])
|
|
self.assertIn("budget_warn", report["warnings"])
|
|
self.assertIn("low_evaluator_cache_hit_rate", report["warnings"])
|
|
self.assertIn("dominant_model_cost", report["warnings"])
|
|
|
|
def test_model_cost_report_handles_empty_usage(self) -> None:
|
|
report = build_model_cost_report({"budget": {}, "evaluator_cache": {}})
|
|
|
|
self.assertEqual(report["models"], [])
|
|
self.assertIsNone(report["top_cost_model"])
|
|
self.assertEqual(report["summary"]["metered_coverage"], 0.0)
|
|
|
|
def test_model_cost_report_warns_when_only_part_of_a_model_is_priced(self) -> None:
|
|
report = build_model_cost_report(
|
|
{
|
|
"cost_usd": 4.5,
|
|
"cost_basis": "partial",
|
|
"by_provider": [
|
|
{
|
|
"provider": "agy_cli",
|
|
"model": "gemini-3.7-flash-high",
|
|
"turns": 2,
|
|
"tokens_in": 2_000_000,
|
|
"tokens_out": 2_000_000,
|
|
"cost_usd": 4.5,
|
|
"cost_basis": "partial",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertIn("partial_model_cost", report["warnings"])
|
|
self.assertIn("unavailable_model_cost", report["warnings"])
|
|
self.assertIsNone(report["summary"]["cost_per_turn_usd"])
|
|
self.assertIsNone(report["models"][0]["cost_share"])
|
|
|
|
def test_model_cost_report_warns_for_reference_upper_bound(self) -> None:
|
|
report = build_model_cost_report(
|
|
{
|
|
"cost_usd": 0.1,
|
|
"by_provider": [
|
|
{
|
|
"provider": "agy_cli",
|
|
"model": "gemini-3.7-flash-high",
|
|
"turns": 1,
|
|
"tokens_in": 100_000,
|
|
"tokens_out": 1_000,
|
|
"cost_usd": 0.1,
|
|
"cost_basis": "reference_upper_bound",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertIn("reference_upper_bound_cost", report["warnings"])
|
|
|
|
def test_model_cost_report_preserves_partial_upper_bound_uncertainty(self) -> None:
|
|
report = build_model_cost_report(
|
|
{
|
|
"cost_usd": 4.5,
|
|
"cost_basis": "partial_upper_bound",
|
|
"budget": {
|
|
"status": "indeterminate",
|
|
"remaining_usd": None,
|
|
"cost_basis": "partial_upper_bound",
|
|
},
|
|
"by_provider": [
|
|
{
|
|
"provider": "agy_cli",
|
|
"model": "gemini-3.7-flash-high",
|
|
"turns": 2,
|
|
"tokens_in": 2_000_000,
|
|
"tokens_out": 2_000_000,
|
|
"cost_usd": 4.5,
|
|
"cost_basis": "partial_upper_bound",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(report["summary"]["cost_basis"], "partial_upper_bound")
|
|
self.assertIsNone(report["summary"]["cost_per_turn_usd"])
|
|
self.assertIsNone(report["budget"]["remaining_usd"])
|
|
self.assertIn("partial_upper_bound_model_cost", report["warnings"])
|
|
self.assertIn("unavailable_model_cost", report["warnings"])
|
|
self.assertIn("budget_indeterminate", report["warnings"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|