운영 지표와 지원 요청 저장소 추가

apps/api/app/services/admin_health_maintenance.py

@@ -0,0 +1,177 @@
+"""Maintenance helpers for persisted admin health samples."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..db import acquire
+
+
+@dataclass(frozen=True)
+class AdminHealthMaintenanceResult:
+    applied: bool
+    rollup_days: int
+    retention_days: int
+    rollup_event_count: int
+    rollup_bucket_count: int
+    upserted_rollups: int
+    prunable_event_count: int
+    deleted_events: int
+
+
+def _validate_windows(*, rollup_days: int, retention_days: int) -> None:
+    if rollup_days < 1:
+        raise ValueError("rollup_days must be 1 or greater")
+    if retention_days < rollup_days:
+        raise ValueError("retention_days must be greater than or equal to rollup_days")
+
+
+def _deleted_count(command_tag: str) -> int:
+    parts = (command_tag or "").split()
+    if len(parts) >= 2 and parts[0].upper() == "DELETE":
+        try:
+            return int(parts[-1])
+        except ValueError:
+            return 0
+    return 0
+
+
+def _row_int(row, key: str) -> int:
+    if row is None:
+        return 0
+    try:
+        return int(row[key] or 0)
+    except (KeyError, TypeError, ValueError):
+        return 0
+
+
+async def maintain_admin_health_events(
+    *,
+    rollup_days: int,
+    retention_days: int,
+    apply: bool = False,
+) -> AdminHealthMaintenanceResult:
+    """Roll up old health samples and optionally prune retained raw rows.
+
+    Dry-run mode returns the affected event and bucket counts without mutating
+    data.  Apply mode upserts daily service rollups first, then deletes only raw
+    events older than the retention window.
+    """
+    _validate_windows(rollup_days=rollup_days, retention_days=retention_days)
+    async with acquire(role="admin") as conn:
+        rollup_stats = await conn.fetchrow(
+            """
+            SELECT
+              COUNT(*)::int AS event_count,
+              COUNT(DISTINCT (observed_at::date, environment, engine_mode, service_key))::int
+                AS bucket_count
+            FROM app.admin_health_event
+            WHERE observed_at::date < current_date - $1::int
+            """,
+            rollup_days,
+        )
+        prune_stats = await conn.fetchrow(
+            """
+            SELECT COUNT(*)::int AS event_count
+            FROM app.admin_health_event
+            WHERE observed_at::date < current_date - $1::int
+            """,
+            retention_days,
+        )
+        upserted_rollups = 0
+        deleted_events = 0
+        if apply:
+            rows = await conn.fetch(
+                """
+                WITH rolled AS (
+                  SELECT
+                    observed_at::date AS rollup_date,
+                    environment,
+                    engine_mode,
+                    service_key,
+                    (array_agg(service_name ORDER BY observed_at DESC, id DESC))[1] AS service_name,
+                    COUNT(*)::int AS sample_count,
+                    COUNT(*) FILTER (WHERE service_status = 'ok')::int AS ok_samples,
+                    COUNT(*) FILTER (WHERE service_status = 'degraded')::int AS degraded_samples,
+                    COUNT(*) FILTER (WHERE service_status = 'down')::int AS down_samples,
+                    MIN(observed_at) AS first_observed_at,
+                    MAX(observed_at) AS last_observed_at,
+                    (array_agg(service_status ORDER BY observed_at DESC, id DESC))[1] AS latest_status,
+                    MAX(observed_at) FILTER (WHERE service_status = 'down') AS last_down_at,
+                    MAX(load)::real AS max_load
+                  FROM app.admin_health_event
+                  WHERE observed_at::date < current_date - $1::int
+                  GROUP BY observed_at::date, environment, engine_mode, service_key
+                )
+                INSERT INTO app.admin_health_daily_rollup (
+                  rollup_date,
+                  environment,
+                  engine_mode,
+                  service_key,
+                  service_name,
+                  sample_count,
+                  ok_samples,
+                  degraded_samples,
+                  down_samples,
+                  first_observed_at,
+                  last_observed_at,
+                  latest_status,
+                  last_down_at,
+                  max_load,
+                  generated_at
+                )
+                SELECT
+                  rollup_date,
+                  environment,
+                  engine_mode,
+                  service_key,
+                  service_name,
+                  sample_count,
+                  ok_samples,
+                  degraded_samples,
+                  down_samples,
+                  first_observed_at,
+                  last_observed_at,
+                  latest_status,
+                  last_down_at,
+                  max_load,
+                  now()
+                FROM rolled
+                ON CONFLICT (rollup_date, environment, engine_mode, service_key)
+                DO UPDATE SET
+                  service_name = EXCLUDED.service_name,
+                  sample_count = EXCLUDED.sample_count,
+                  ok_samples = EXCLUDED.ok_samples,
+                  degraded_samples = EXCLUDED.degraded_samples,
+                  down_samples = EXCLUDED.down_samples,
+                  first_observed_at = EXCLUDED.first_observed_at,
+                  last_observed_at = EXCLUDED.last_observed_at,
+                  latest_status = EXCLUDED.latest_status,
+                  last_down_at = EXCLUDED.last_down_at,
+                  max_load = EXCLUDED.max_load,
+                  generated_at = now()
+                RETURNING 1
+                """,
+                rollup_days,
+            )
+            upserted_rollups = len(rows)
+            deleted_events = _deleted_count(
+                await conn.execute(
+                    """
+                    DELETE FROM app.admin_health_event
+                    WHERE observed_at::date < current_date - $1::int
+                    """,
+                    retention_days,
+                )
+            )
+
+    return AdminHealthMaintenanceResult(
+        applied=apply,
+        rollup_days=rollup_days,
+        retention_days=retention_days,
+        rollup_event_count=_row_int(rollup_stats, "event_count"),
+        rollup_bucket_count=_row_int(rollup_stats, "bucket_count"),
+        upserted_rollups=upserted_rollups,
+        prunable_event_count=_row_int(prune_stats, "event_count"),
+        deleted_events=deleted_events,
+    )

apps/api/app/services/phase3_kpi_export.py

@@ -0,0 +1,276 @@
+"""Phase 3 KPI evidence export helpers.
+
+This module turns persisted pre/post aggregate scores into the Phase 3 evidence
+shape checked by scripts/check-phase3-artifacts.py. It does not claim clinical
+effectiveness; it only produces pilot evidence files for operator review.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from collections import defaultdict
+from datetime import UTC, date, datetime
+from decimal import Decimal
+from pathlib import Path
+from typing import Any, Iterable, Mapping, Sequence
+from uuid import UUID
+
+PREPOST_MEASURE_NAMES = (
+    "self_efficacy",
+    "skill_proficiency",
+    "training_satisfaction",
+)
+PREPOST_TIMEPOINTS = ("pre", "post")
+PHASE3_KPI_METRICS = (
+    "embedding_consistency",
+    "hallucination_rate",
+    "icc",
+    "inter_rater_kappa",
+    "pilot_completion",
+    "self_efficacy_prepost",
+    "session_completion",
+    "sus",
+    "top1",
+)
+PREPOST_CSV_PATH = "02-measures/prepost_measures.csv"
+KPI_REPORT_PATH = "02-measures/kpi_report.json"
+
+
+class ParticipantKeys:
+    def __init__(self) -> None:
+        self._keys: dict[str, str] = {}
+
+    def key(self, raw_id: Any) -> str:
+        value = str(raw_id or "unknown-participant")
+        if value not in self._keys:
+            self._keys[value] = f"P3-{len(self._keys) + 1:03d}"
+        return self._keys[value]
+
+    def __len__(self) -> int:
+        return len(self._keys)
+
+
+def json_safe(value: Any) -> Any:
+    if isinstance(value, datetime):
+        if value.tzinfo is None:
+            value = value.replace(tzinfo=UTC)
+        return value.isoformat().replace("+00:00", "Z")
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, (Decimal, UUID)):
+        return str(value)
+    if isinstance(value, Mapping):
+        return {str(key): json_safe(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [json_safe(item) for item in value]
+    if isinstance(value, tuple):
+        return [json_safe(item) for item in value]
+    return value
+
+
+def iso_timestamp(value: Any) -> str:
+    safe = json_safe(value)
+    return safe if isinstance(safe, str) else str(safe or "")
+
+
+def normalized_score(raw_score: float, min_score: float, max_score: float) -> float:
+    if max_score <= min_score:
+        return 0.0
+    return round(((raw_score - min_score) / (max_score - min_score)) * 100.0, 3)
+
+
+def latest_prepost_rows(rows: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    latest: dict[tuple[str, str, str], dict[str, Any]] = {}
+    for row in rows:
+        learner_id = str(row.get("learner_id") or row.get("participant_id") or "")
+        measure_name = str(row.get("measure_name") or "")
+        timepoint = str(row.get("timepoint") or "")
+        if measure_name not in PREPOST_MEASURE_NAMES or timepoint not in PREPOST_TIMEPOINTS:
+            continue
+        key = (learner_id, measure_name, timepoint)
+        current = dict(row)
+        current_order = iso_timestamp(current.get("updated_at") or current.get("collected_at"))
+        previous = latest.get(key)
+        previous_order = iso_timestamp(previous.get("updated_at") or previous.get("collected_at")) if previous else ""
+        if previous is None or current_order >= previous_order:
+            latest[key] = current
+    return sorted(
+        latest.values(),
+        key=lambda item: (
+            str(item.get("learner_id") or item.get("participant_id") or ""),
+            str(item.get("measure_name") or ""),
+            str(item.get("timepoint") or ""),
+        ),
+    )
+
+
+def build_prepost_csv_rows(
+    rows: Iterable[Mapping[str, Any]],
+    *,
+    participant_keys: ParticipantKeys | None = None,
+) -> list[dict[str, str]]:
+    keys = participant_keys if participant_keys is not None else ParticipantKeys()
+    output: list[dict[str, str]] = []
+    for row in latest_prepost_rows(rows):
+        raw_score = float(row.get("raw_score") or row.get("score") or 0.0)
+        output.append(
+            {
+                "participant_id": keys.key(row.get("learner_id") or row.get("participant_id")),
+                "measure_name": str(row.get("measure_name") or ""),
+                "timepoint": str(row.get("timepoint") or ""),
+                "score": _format_number(raw_score),
+                "collected_at": iso_timestamp(row.get("collected_at") or row.get("updated_at")),
+            }
+        )
+    return output
+
+
+def paired_prepost_summary(rows: Iterable[Mapping[str, Any]], measure_name: str) -> dict[str, Any]:
+    by_participant: dict[str, dict[str, float]] = defaultdict(dict)
+    for row in latest_prepost_rows(rows):
+        if str(row.get("measure_name") or "") != measure_name:
+            continue
+        participant_id = str(row.get("learner_id") or row.get("participant_id") or "")
+        raw_score = float(row.get("raw_score") or row.get("score") or 0.0)
+        min_score = float(row.get("min_score") or 1.0)
+        max_score = float(row.get("max_score") or 5.0)
+        by_participant[participant_id][str(row.get("timepoint") or "")] = normalized_score(
+            raw_score,
+            min_score,
+            max_score,
+        )
+
+    deltas: list[float] = []
+    pre_values: list[float] = []
+    post_values: list[float] = []
+    missing_pairs = 0
+    for values in by_participant.values():
+        if "pre" not in values or "post" not in values:
+            missing_pairs += 1
+            continue
+        pre_values.append(values["pre"])
+        post_values.append(values["post"])
+        deltas.append(values["post"] - values["pre"])
+
+    return {
+        "participants_with_any_measure": len(by_participant),
+        "complete_pairs": len(deltas),
+        "missing_pairs": missing_pairs,
+        "mean_pre": _mean(pre_values),
+        "mean_post": _mean(post_values),
+        "mean_delta": _mean(deltas),
+    }
+
+
+def build_kpi_report(
+    rows: Sequence[Mapping[str, Any]],
+    *,
+    pilot_id: str,
+    generated_at: str,
+    source_window: Mapping[str, Any] | None = None,
+    review_operator: str = "",
+) -> dict[str, Any]:
+    latest_rows = latest_prepost_rows(rows)
+    participants = {str(row.get("learner_id") or row.get("participant_id") or "") for row in latest_rows}
+    metrics = {name: _placeholder_metric(name) for name in PHASE3_KPI_METRICS}
+
+    self_efficacy = paired_prepost_summary(latest_rows, "self_efficacy")
+    metrics["self_efficacy_prepost"] = {
+        "value": self_efficacy["mean_delta"],
+        "threshold": 0.0,
+        "pass": False,
+        "numerator": self_efficacy["complete_pairs"],
+        "denominator": max(self_efficacy["participants_with_any_measure"], 0),
+        "method": "paired normalized post-pre delta for pilot review; no official pass/fail gate",
+        "source_files": [PREPOST_CSV_PATH],
+        "mean_pre": self_efficacy["mean_pre"],
+        "mean_post": self_efficacy["mean_post"],
+        "mean_delta": self_efficacy["mean_delta"],
+        "complete_pairs": self_efficacy["complete_pairs"],
+        "missing_pairs": self_efficacy["missing_pairs"],
+    }
+
+    for measure_name in ("skill_proficiency", "training_satisfaction"):
+        summary = paired_prepost_summary(latest_rows, measure_name)
+        metrics[f"{measure_name}_prepost"] = {
+            **_placeholder_metric(f"{measure_name}_prepost"),
+            **summary,
+            "value": summary["mean_delta"],
+            "numerator": summary["complete_pairs"],
+            "denominator": summary["participants_with_any_measure"],
+            "method": "paired normalized post-pre delta for pilot review; not a required KPI gate yet",
+            "source_files": [PREPOST_CSV_PATH],
+        }
+
+    return {
+        "pilot_id": pilot_id,
+        "generated_at": generated_at,
+        "source_window": dict(source_window or _source_window(latest_rows)),
+        "cohort_size": len(participants),
+        "metrics": metrics,
+        "exclusions": [],
+        "open_schema_gaps": [
+            "official item text and validated scoring rules are not encoded here",
+            "experimental/control assignment and statistical testing require owner/evaluation-design approval",
+        ],
+        "review": {
+            "operator": review_operator,
+            "reviewed_at": "",
+            "decision": "pending",
+        },
+    }
+
+
+def write_prepost_csv(rows: Sequence[Mapping[str, str]], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=("participant_id", "measure_name", "timepoint", "score", "collected_at"),
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def write_kpi_report(report: Mapping[str, Any], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(json_safe(report), ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _placeholder_metric(name: str) -> dict[str, Any]:
+    return {
+        "value": 0.0,
+        "threshold": 0.0,
+        "pass": False,
+        "numerator": 0,
+        "denominator": 0,
+        "method": f"not computed by prepost export scaffold: {name}",
+        "source_files": [],
+    }
+
+
+def _source_window(rows: Sequence[Mapping[str, Any]]) -> dict[str, str]:
+    timestamps = [
+        iso_timestamp(row.get("collected_at") or row.get("updated_at"))
+        for row in rows
+        if iso_timestamp(row.get("collected_at") or row.get("updated_at"))
+    ]
+    if not timestamps:
+        return {"started_at": "", "ended_at": ""}
+    return {"started_at": min(timestamps), "ended_at": max(timestamps)}
+
+
+def _mean(values: Sequence[float]) -> float:
+    if not values:
+        return 0.0
+    return round(sum(values) / len(values), 3)
+
+
+def _format_number(value: float) -> str:
+    if value.is_integer():
+        return str(int(value))
+    return f"{value:.3f}".rstrip("0").rstrip(".")

apps/api/app/services/support_tickets.py

@@ -0,0 +1,31 @@
+"""Support-ticket helpers shared across user and admin routes."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+
+_SPACE_RE = re.compile(r"\s+")
+
+
+def _normalize_fingerprint_part(value: str | None) -> str:
+    return _SPACE_RE.sub(" ", (value or "").strip().lower())
+
+
+def support_ticket_fingerprint(
+    *,
+    category: str,
+    subject: str,
+    body: str,
+    source_path: str,
+) -> str:
+    """Return a stable hash for exact-ish duplicate support-ticket hints."""
+    payload = {
+        "body": _normalize_fingerprint_part(body),
+        "category": _normalize_fingerprint_part(category),
+        "source_path": _normalize_fingerprint_part(source_path),
+        "subject": _normalize_fingerprint_part(subject),
+    }
+    encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()

apps/api/app/services/usage_report.py

@@ -0,0 +1,134 @@
+"""AI usage cost verification reports.
+
+The admin API already owns collection. This module turns that existing usage
+shape into a deterministic model/provider cost report for ops evidence without
+adding any enforcement policy.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+
+REPORT_SCHEMA = "vignette.ai_usage_model_cost_report.v1"
+
+
+def _number(value: Any, default: float = 0.0) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _integer(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _ratio(part: float, whole: float) -> float:
+    if whole <= 0:
+        return 0.0
+    return round(part / whole, 6)
+
+
+def _cost_per_1k_tokens(cost_usd: float, tokens: int) -> float | None:
+    if tokens <= 0:
+        return None
+    return round(cost_usd / tokens * 1000.0, 6)
+
+
+def _cost_per_turn(cost_usd: float, turns: int) -> float | None:
+    if turns <= 0:
+        return None
+    return round(cost_usd / turns, 6)
+
+
+def build_model_cost_report(usage: Mapping[str, Any]) -> dict[str, Any]:
+    """Build an ops report from an AdminUsageResponse-like mapping."""
+    total_cost = round(_number(usage.get("cost_usd")), 6)
+    total_turns = _integer(usage.get("total_turns"))
+    metered_turns = _integer(usage.get("metered_turns"))
+    tokens_in = _integer(usage.get("tokens_in"))
+    tokens_out = _integer(usage.get("tokens_out"))
+    total_tokens = tokens_in + tokens_out
+    by_provider = list(usage.get("by_provider") or [])
+    budget = dict(usage.get("budget") or {})
+    evaluator_cache = dict(usage.get("evaluator_cache") or {})
+
+    models: list[dict[str, Any]] = []
+    for item in by_provider:
+        row = dict(item or {})
+        turns = _integer(row.get("turns"))
+        row_tokens_in = _integer(row.get("tokens_in"))
+        row_tokens_out = _integer(row.get("tokens_out"))
+        row_tokens = row_tokens_in + row_tokens_out
+        row_cost = round(_number(row.get("cost_usd")), 6)
+        models.append(
+            {
+                "provider": str(row.get("provider") or "unknown"),
+                "model": str(row.get("model") or "unknown"),
+                "turns": turns,
+                "tokens_in": row_tokens_in,
+                "tokens_out": row_tokens_out,
+                "tokens_total": row_tokens,
+                "cost_usd": row_cost,
+                "cost_share": _ratio(row_cost, total_cost),
+                "token_share": _ratio(float(row_tokens), float(total_tokens)),
+                "cost_per_turn_usd": _cost_per_turn(row_cost, turns),
+                "cost_per_1k_tokens_usd": _cost_per_1k_tokens(row_cost, row_tokens),
+            }
+        )
+    models.sort(key=lambda item: (-float(item["cost_usd"]), item["provider"], item["model"]))
+
+    warnings: list[str] = []
+    if total_turns > 0 and metered_turns < total_turns:
+        warnings.append("partial_metering")
+    if total_cost == 0 and metered_turns > 0:
+        warnings.append("zero_cost_metered_usage")
+    if str(budget.get("status") or "") in {"warn", "exceeded"}:
+        warnings.append(f"budget_{budget.get('status')}")
+    cache_hit_rate = _number(evaluator_cache.get("hit_rate"))
+    if bool(evaluator_cache.get("enabled")) and _integer(evaluator_cache.get("requests")) > 0:
+        if cache_hit_rate < 0.25:
+            warnings.append("low_evaluator_cache_hit_rate")
+    if models and models[0]["cost_share"] >= 0.8:
+        warnings.append("dominant_model_cost")
+
+    return {
+        "schema": REPORT_SCHEMA,
+        "source": str(usage.get("source") or "unknown"),
+        "durable": bool(usage.get("durable")),
+        "window_days": _integer(usage.get("window_days")),
+        "summary": {
+            "total_turns": total_turns,
+            "metered_turns": metered_turns,
+            "metered_coverage": _ratio(float(metered_turns), float(total_turns)),
+            "tokens_in": tokens_in,
+            "tokens_out": tokens_out,
+            "tokens_total": total_tokens,
+            "cost_usd": total_cost,
+            "cost_per_turn_usd": _cost_per_turn(total_cost, metered_turns),
+            "cost_per_1k_tokens_usd": _cost_per_1k_tokens(total_cost, total_tokens),
+        },
+        "budget": {
+            "status": str(budget.get("status") or "disabled"),
+            "limit_usd": round(_number(budget.get("limit_usd")), 6),
+            "used_ratio": round(_number(budget.get("used_ratio")), 6),
+            "remaining_usd": round(_number(budget.get("remaining_usd")), 6),
+        },
+        "evaluator_cache": {
+            "enabled": bool(evaluator_cache.get("enabled")),
+            "requests": _integer(evaluator_cache.get("requests")),
+            "hits": _integer(evaluator_cache.get("hits")),
+            "misses": _integer(evaluator_cache.get("misses")),
+            "hit_rate": round(cache_hit_rate, 6),
+        },
+        "models": models,
+        "top_cost_model": models[0] if models else None,
+        "warnings": warnings,
+    }
+
+
+__all__ = ["REPORT_SCHEMA", "build_model_cost_report"]