#!/usr/bin/env python3
"""Jev 감정 판단 API의 공개 합성 한국어 사례 실측 러너."""

from __future__ import annotations

import argparse
import asyncio
import hashlib
import json
import math
import os
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from pydantic import ValidationError


REPO_ROOT = Path(__file__).resolve().parents[1]
API_ROOT = REPO_ROOT / "apps" / "api"
FIXTURE_PATH = REPO_ROOT / "scripts" / "fixtures" / "jev-client-korean-cases.json"
REQUIRED_STATE_KEYS = frozenset(
    {
        "counselor_utterance",
        "recent_turns",
        "client_profile",
        "pinned_facts",
        "recall_summary",
        "relationship",
        "previous_feelings",
    }
)
EMOTION_KEYS = frozenset(
    {"anxiety", "sadness", "anger", "shame", "guilt", "loneliness", "relief", "hope", "trust"}
)
MOOD_WORDS = frozenset({"absent", "slight", "moderate", "strong", "overwhelming"})
OPENNESS_WORDS = frozenset({"closed", "guarded", "partly_open", "open", "deep"})
RESISTANCE_WORDS = frozenset({"low", "moderate", "high"})
SENSITIVE_FIELD_PATTERN = re.compile(r"(?:api[_-]?key|authorization|password|secret|token)", re.IGNORECASE)
SECRET_VALUE_PATTERN = re.compile(r"(?:sk-|bearer\s+|AIza|AKIA)[A-Za-z0-9_\-]{8,}", re.IGNORECASE)


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="공개 합성 한국어 가상 내담자 사례로 Jev 감정 판단 API를 실측한다."
    )
    parser.add_argument(
        "--output",
        required=True,
        help="로컬 JSON 보고서 저장 경로",
    )
    parser.add_argument(
        "--fixtures",
        type=Path,
        default=FIXTURE_PATH,
        help="공개 합성 사례 JSON 경로",
    )
    parser.add_argument(
        "--repeats",
        type=int,
        default=1,
        help="각 사례의 반복 횟수(1~20, 기본 1)",
    )
    return parser


def load_cases(path: Path) -> list[dict[str, Any]]:
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as exc:
        raise ValueError("fixture_load_failed") from exc
    if not isinstance(payload, dict) or payload.get("provenance") != "public_synthetic":
        raise ValueError("fixture_provenance_invalid")
    cases = payload.get("cases")
    if not isinstance(cases, list) or not 8 <= len(cases) <= 12:
        raise ValueError("fixture_case_count_invalid")

    identifiers: set[str] = set()
    for case in cases:
        validate_case(case, identifiers)
    return cases


def validate_case(case: Any, identifiers: set[str]) -> None:
    if not isinstance(case, dict) or set(case) != {
        "id",
        "description",
        "state",
        "review_questions",
    }:
        raise ValueError("fixture_case_shape_invalid")
    case_id = case["id"]
    if not isinstance(case_id, str) or not case_id or case_id in identifiers:
        raise ValueError("fixture_case_id_invalid")
    identifiers.add(case_id)
    if not isinstance(case["description"], str) or not case["description"]:
        raise ValueError("fixture_description_invalid")
    state = case["state"]
    if not isinstance(state, dict) or set(state) != REQUIRED_STATE_KEYS:
        raise ValueError("fixture_state_contract_invalid")
    client_profile = state["client_profile"]
    recent_turns = state["recent_turns"]
    feelings = state["previous_feelings"]
    relationship = state["relationship"]
    if (
        not isinstance(client_profile, dict)
        or not {"presenting", "history", "sore_spots", "forbidden"} <= set(client_profile)
        or not all(isinstance(client_profile[key], str) and client_profile[key] for key in ("presenting", "history"))
        or not isinstance(client_profile["sore_spots"], list)
        or not isinstance(client_profile["forbidden"], list)
        or not all(isinstance(item, str) and item for item in client_profile["sore_spots"])
        or not all(isinstance(item, str) and item for item in client_profile["forbidden"])
        or ("temperament" in client_profile and not isinstance(client_profile["temperament"], list))
    ):
        raise ValueError("fixture_client_profile_invalid")
    if (
        not isinstance(state["pinned_facts"], list)
        or not all(isinstance(item, str) for item in state["pinned_facts"])
        or not isinstance(state["recall_summary"], str)
    ):
        raise ValueError("fixture_memory_invalid")
    if (
        not isinstance(recent_turns, list)
        or len(recent_turns) > 12
        or not all(
            isinstance(turn, dict)
            and set(turn) == {"speaker", "text"}
            and turn["speaker"] in {"counselor", "client"}
            and isinstance(turn["text"], str)
            for turn in recent_turns
        )
        or not isinstance(state["counselor_utterance"], str)
    ):
        raise ValueError("fixture_recent_turns_invalid")
    if (
        not isinstance(feelings, dict)
        or set(feelings) != EMOTION_KEYS
        or not all(value in MOOD_WORDS for value in feelings.values())
    ):
        raise ValueError("fixture_previous_feelings_invalid")
    if (
        not isinstance(relationship, dict)
        or set(relationship) != {"stage", "openness", "resistance"}
        or not isinstance(relationship["stage"], str)
        or not relationship["stage"]
        or relationship["openness"] not in OPENNESS_WORDS
        or relationship["resistance"] not in RESISTANCE_WORDS
    ):
        raise ValueError("fixture_relationship_invalid")
    questions = case["review_questions"]
    if (
        not isinstance(questions, list)
        or len(questions) != 2
        or not all(isinstance(question, str) and question for question in questions)
    ):
        raise ValueError("fixture_review_questions_invalid")
    if contains_sensitive_content(case):
        raise ValueError("fixture_sensitive_content")


def contains_sensitive_content(value: Any, field_name: str = "") -> bool:
    if SENSITIVE_FIELD_PATTERN.search(field_name):
        return True
    if isinstance(value, str):
        return bool(SECRET_VALUE_PATTERN.search(value))
    if isinstance(value, dict):
        return any(contains_sensitive_content(item, str(key)) for key, item in value.items())
    if isinstance(value, list):
        return any(contains_sensitive_content(item) for item in value)
    return False


def percentile(values: list[int], percent: float) -> int | None:
    if not values:
        return None
    ordered = sorted(values)
    position = (len(ordered) - 1) * percent
    lower = math.floor(position)
    upper = math.ceil(position)
    if lower == upper:
        return ordered[lower]
    return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower))


def base_report(
    *,
    fixture_path: Path,
    case_count: int,
    repeats: int,
    provider: str,
    requested_model: str,
    timeout_seconds: float,
    confidence_threshold: float,
) -> dict[str, Any]:
    return {
        "status": "blocked",
        "fixture": {
            "path": str(fixture_path),
            "sha256": fixture_sha256(fixture_path),
            "case_count": case_count,
            "repeats": repeats,
            "provenance": "public_synthetic",
            "contains_real_patient_data": False,
        },
        "measurement_started_at_utc": datetime.now(timezone.utc).isoformat(),
        "configuration": {
            "route_provider": provider,
            "requested_model": requested_model,
            "timeout_seconds": timeout_seconds,
            "confidence_threshold": confidence_threshold,
        },
        "metrics": {
            "attempted_calls": 0,
            "success_count": 0,
            "failure_count": 0,
            "appraisal_latency_ms": {"sample_count": 0, "p50": None, "p95": None},
            "input_tokens_total": 0,
            "output_tokens_total": 0,
            "cost_usd_total": None,
            "known_success_cost_usd": None,
            "failed_calls_cost_known": None,
            "actual_models": [],
        },
        "results": [],
        "limitations": [
            "이 결과는 Jev 감정 판단 API 실측이며 전체 응답 지연, TTFT, 임상 타당성의 증거가 아니다.",
            "전문가 검토 질문은 사례별 결과와 분리해 fixture에만 보관하며 자동 정답 또는 정확도 판정에 사용하지 않는다.",
        ],
    }


def fixture_sha256(path: Path) -> str | None:
    try:
        return hashlib.sha256(path.read_bytes()).hexdigest()
    except OSError:
        return None


def write_report(path: Path, report: dict[str, Any]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")


async def collect(report: dict[str, Any], cases: list[dict[str, Any]], repeats: int) -> None:
    sys.path.insert(0, str(API_ROOT))
    from app.services.jev_client import JevError, jev_client

    confidence_threshold = report["configuration"]["confidence_threshold"]
    latencies: list[int] = []
    models: set[str] = set()
    known_costs: list[float] = []
    try:
        await jev_client.startup()
        for repeat in range(1, repeats + 1):
            for case in cases:
                case_id = case["id"]
                report["metrics"]["attempted_calls"] += 1
                try:
                    appraisal = await jev_client.appraise(case["state"])
                except JevError as exc:
                    report["metrics"]["failure_count"] += 1
                    report["results"].append(
                        {"case_id": case_id, "repeat": repeat, "status": "failed", "error_code": exc.code}
                    )
                    continue
                except Exception:
                    report["metrics"]["failure_count"] += 1
                    report["results"].append(
                        {"case_id": case_id, "repeat": repeat, "status": "failed", "error_code": "unexpected_error"}
                    )
                    continue

                dimensions = {}
                for name, estimate in appraisal.emotions.items():
                    confidence_missing = estimate.confidence is None
                    dimensions[name] = {
                        "score": estimate.score,
                        "confidence": estimate.confidence,
                        "confidence_missing": confidence_missing,
                        "below_confidence_threshold": (
                            True if confidence_missing else estimate.confidence < confidence_threshold
                        ),
                        "probabilities": estimate.probabilities,
                    }
                report["metrics"]["success_count"] += 1
                report["metrics"]["input_tokens_total"] += appraisal.input_tokens
                report["metrics"]["output_tokens_total"] += appraisal.output_tokens
                latencies.append(appraisal.latency_ms)
                models.add(appraisal.model)
                if appraisal.cost_usd is not None:
                    known_costs.append(appraisal.cost_usd)
                report["results"].append(
                    {
                        "case_id": case_id,
                        "repeat": repeat,
                        "status": "collected",
                        "provider": appraisal.provider,
                        "actual_model": appraisal.model,
                        "appraisal_latency_ms": appraisal.latency_ms,
                        "input_tokens": appraisal.input_tokens,
                        "output_tokens": appraisal.output_tokens,
                        "dimensions": dimensions,
                    }
                )
    except JevError as exc:
        report["metrics"]["failure_count"] += 1
        report["results"].append(
            {"case_id": "runner", "repeat": 0, "status": "failed", "error_code": exc.code}
        )
    except Exception:
        report["metrics"]["failure_count"] += 1
        report["results"].append(
            {"case_id": "runner", "repeat": 0, "status": "failed", "error_code": "unexpected_error"}
        )
    finally:
        try:
            await jev_client.shutdown()
        except Exception:
            report["metrics"]["failure_count"] += 1
            report["results"].append(
                {"case_id": "runner", "repeat": 0, "status": "failed", "error_code": "shutdown_error"}
            )

    report["metrics"]["actual_models"] = sorted(models)
    if known_costs:
        report["metrics"]["known_success_cost_usd"] = sum(known_costs)
    report["metrics"]["appraisal_latency_ms"] = {
        "sample_count": len(latencies),
        "p50": percentile(latencies, 0.50),
        "p95": percentile(latencies, 0.95),
    }
    success_count = report["metrics"]["success_count"]
    failure_count = report["metrics"]["failure_count"]
    report["metrics"]["failed_calls_cost_known"] = failure_count == 0
    if (
        failure_count == 0
        and success_count > 0
        and success_count == len(known_costs)
    ):
        report["metrics"]["cost_usd_total"] = report["metrics"]["known_success_cost_usd"]
    if failure_count == 0:
        report["status"] = "collected"
    elif success_count == 0:
        report["status"] = "failed"
    else:
        report["status"] = "partial"


def load_settings() -> Any:
    sys.path.insert(0, str(API_ROOT))
    original_cwd = Path.cwd()
    try:
        os.chdir(API_ROOT)
        from app.config import settings

        return settings
    finally:
        os.chdir(original_cwd)


def provider_key_present(settings: Any) -> bool:
    if settings.jev_provider == "openrouter":
        return bool(settings.openrouter_api_key.get_secret_value().strip())
    if settings.jev_provider == "typesafe":
        return bool(settings.typesafe_api_key.get_secret_value().strip())
    return False


def invalid_configuration_report(*, fixture_path: Path, case_count: int, repeats: int) -> dict[str, Any]:
    report = base_report(
        fixture_path=fixture_path,
        case_count=case_count,
        repeats=repeats,
        provider="unavailable",
        requested_model="unavailable",
        timeout_seconds=0.0,
        confidence_threshold=0.0,
    )
    report["blocking_reason"] = "configuration_invalid"
    return report


def main() -> int:
    args = build_parser().parse_args()
    if not 1 <= args.repeats <= 20:
        raise SystemExit("--repeats must be between 1 and 20")
    try:
        cases = load_cases(args.fixtures)
    except ValueError as exc:
        report = {
            "status": "failed",
            "failure_reason": str(exc),
            "measurement_started_at_utc": datetime.now(timezone.utc).isoformat(),
        }
        write_report(Path(args.output), report)
        return 1

    try:
        settings = load_settings()
    except ValidationError:
        write_report(
            Path(args.output),
            invalid_configuration_report(
                fixture_path=args.fixtures,
                case_count=len(cases),
                repeats=args.repeats,
            ),
        )
        return 2
    report = base_report(
        fixture_path=args.fixtures,
        case_count=len(cases),
        repeats=args.repeats,
        provider=settings.jev_provider,
        requested_model=settings.jev_model,
        timeout_seconds=settings.jev_timeout_seconds,
        confidence_threshold=settings.jev_min_confidence,
    )
    if not provider_key_present(settings):
        report["blocking_reason"] = f"{settings.jev_provider}_api_key_missing"
        write_report(Path(args.output), report)
        return 2

    asyncio.run(collect(report, cases, args.repeats))
    write_report(Path(args.output), report)
    return 0 if report["status"] == "collected" else 1


if __name__ == "__main__":
    raise SystemExit(main())