#!/usr/bin/env node
// designpaca 인터뷰 게이트 로컬 평가 — 실행 스크립트.
// 자세한 사용법·판정 기준은 build/eval/interview/README.md 참고.
//
//   node build/eval/interview/run.mjs --skill-dir <path> --label <name> \
//     [--runners claude,claude-sdk-auto,claude-sdk-default,codex] [--repeat 3] \
//     [--scenarios S1,S2] [--claude-model sonnet] [--codex-model gpt-5.1-codex] \
//     [--permission-mode default] [--timeout-ms 300000]
//
// --runners 항목은 "이름:반복수"로 개별 반복수를 줄 수 있다(생략하면 --repeat).
//   예: --runners claude-sdk-auto:3,claude-sdk-default:1,codex:3
// 러너 종류:
//   claude              - run-claude.mjs (`claude -p` CLI, v1)
//   claude-sdk-auto      - run-claude-sdk.mjs, permissionMode=auto (v2, 주 지표)
//   claude-sdk-default   - run-claude-sdk.mjs, permissionMode=default (v2, 보조 지표)
//   codex                - run-codex.mjs

import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn } from "node:child_process";
import { copyDirRecursive } from "./lib/fsutil.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(HERE, "..", "..", "..");

function parseArgs(argv) {
  const out = {};
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a.startsWith("--")) {
      const key = a.slice(2);
      const next = argv[i + 1];
      if (next === undefined || next.startsWith("--")) {
        out[key] = true;
      } else {
        out[key] = next;
        i++;
      }
    }
  }
  return out;
}

function runNode(scriptPath, args, logFile) {
  return new Promise((resolve) => {
    const child = spawn(process.execPath, [scriptPath, ...args], {
      cwd: REPO_ROOT,
      stdio: ["ignore", "pipe", "pipe"],
    });
    let out = "";
    let err = "";
    child.stdout.on("data", (d) => (out += d));
    child.stderr.on("data", (d) => (err += d));
    child.on("close", (code) => {
      if (logFile) {
        fs.writeFileSync(logFile, `--- stdout ---\n${out}\n--- stderr ---\n${err}\n--- exit ---\n${code}\n`);
      }
      resolve({ code, out, err });
    });
  });
}

/**
 * 통과 여부를 판정한다. 기계적으로 판정 가능한 것만 pass/fail 로 확정하고,
 * 원문을 직접 읽어야 하는 경우(S7의 "범위 질문 1개 이하" 같은 것)는 REVIEW 로 남긴다.
 */
function computeVerdict(scenario, record) {
  if (record.error) return { verdict: "ERROR", note: record.error };
  if (scenario.expect === "ask") {
    if (record.asked && record.wroteBeforeAnswer === 0) return { verdict: "PASS", note: "" };
    if (!record.asked) return { verdict: "FAIL", note: "질문 없이 끝남(또는 타임아웃)" };
    return { verdict: "FAIL", note: `질문 전에 파일 ${record.wroteBeforeAnswer}건 생성/수정` };
  }
  // expect === "no-interview"
  if (!record.asked) return { verdict: "PASS", note: "" };
  if (scenario.allowScopeQuestion) {
    return { verdict: "REVIEW", note: "질문이 나왔다 — 범위 확인 질문 1개 이하인지 원문을 직접 읽고 판정할 것" };
  }
  return { verdict: "FAIL", note: "위임/국소 경로인데 인터뷰가 나옴" };
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const skillDirArg = args["skill-dir"];
  const label = args.label;
  if (!skillDirArg || !label) {
    console.error("사용법: run.mjs --skill-dir <path> --label <name> [--runners claude,codex] [--repeat 3] [--scenarios S1,S2] ...");
    process.exit(2);
  }
  const skillDir = path.resolve(skillDirArg);
  if (!fs.existsSync(path.join(skillDir, "SKILL.md"))) {
    console.error(`오류: ${skillDir} 에 SKILL.md 가 없다.`);
    process.exit(2);
  }

  const defaultRepeat = Number(args.repeat || 3);
  const runnerSpecs = String(args.runners || "claude,codex")
    .split(",")
    .map((s) => s.trim())
    .filter(Boolean)
    .map((spec) => {
      const [name, repeatStr] = spec.split(":");
      return { name, repeat: repeatStr ? Number(repeatStr) : defaultRepeat };
    });
  const scenarioFilter = args.scenarios ? String(args.scenarios).split(",").map((s) => s.trim()) : null;
  const claudeModel = args["claude-model"] || "sonnet";
  const codexModel = args["codex-model"] || null;
  const permissionMode = args["permission-mode"] || null;
  const timeoutMs = args["timeout-ms"] || "300000";

  const scenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8")).filter(
    (s) => !scenarioFilter || scenarioFilter.includes(s.id)
  );

  const runsRoot = path.join(REPO_ROOT, "outputs", "eval", "interview", "runs", label);
  fs.mkdirSync(runsRoot, { recursive: true });
  const resultsPath = path.join(runsRoot, "results.jsonl");
  const resultsStream = fs.createWriteStream(resultsPath, { flags: "a" });


  for (const { name: runner, repeat } of runnerSpecs) {
    for (const scenario of scenarios) {
      for (let n = 1; n <= repeat; n++) {
        const runId = `${scenario.id}-${n}`;
        const runDir = path.join(runsRoot, runner, runId);
        const workdir = path.join(runDir, "work");
        fs.mkdirSync(workdir, { recursive: true });

        if (scenario.fixture) {
          copyDirRecursive(path.join(HERE, "fixtures", scenario.fixture), workdir);
        }

        let skillPathForCodex = null;
        if (runner === "claude" || runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
          const dest = path.join(workdir, ".claude", "skills", "designpaca");
          fs.mkdirSync(dest, { recursive: true });
          copyDirRecursive(skillDir, dest);
        } else if (runner === "codex") {
          const dest = path.join(workdir, ".agents", "skills", "designpaca");
          fs.mkdirSync(dest, { recursive: true });
          copyDirRecursive(skillDir, dest);
          skillPathForCodex = path.join(dest, "SKILL.md");
        }

        const rawLogExt = runner === "codex" ? "raw.jsonl" : "raw.ndjson";
        const resultPath = path.join(runDir, "result.json");
        const rawLogPath = path.join(runDir, rawLogExt);
        const procLogPath = path.join(runDir, "process.log");

        process.stdout.write(`[${label}] ${runner} ${runId} 실행 중...\n`);

        if (runner === "claude") {
          const cliArgs = [
            "--workdir", workdir,
            "--scenario", scenario.id,
            "--prompt", scenario.prompt,
            "--model", claudeModel,
            "--out", resultPath,
            "--log", rawLogPath,
            "--timeout-ms", String(timeoutMs),
          ];
          if (permissionMode) cliArgs.push("--permission-mode", permissionMode);
          await runNode(path.join(HERE, "run-claude.mjs"), cliArgs, procLogPath);
        } else if (runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
          const mode = runner === "claude-sdk-auto" ? "auto" : "default";
          const cliArgs = [
            "--workdir", workdir,
            "--scenario", scenario.id,
            "--prompt", scenario.prompt,
            "--model", claudeModel,
            "--permission-mode", mode,
            "--out", resultPath,
            "--log", rawLogPath,
            "--timeout-ms", String(timeoutMs),
          ];
          await runNode(path.join(HERE, "run-claude-sdk.mjs"), cliArgs, procLogPath);
        } else if (runner === "codex") {
          const cliArgs = [
            "--workdir", workdir,
            "--scenario", scenario.id,
            "--prompt", scenario.prompt,
            "--skill-path", skillPathForCodex,
            "--out", resultPath,
            "--log", rawLogPath,
            "--timeout-ms", String(timeoutMs),
          ];
          if (codexModel) cliArgs.push("--model", codexModel);
          await runNode(path.join(HERE, "run-codex.mjs"), cliArgs, procLogPath);
        } else {
          console.error(`알 수 없는 러너: ${runner}`);
          continue;
        }

        let record;
        try {
          record = JSON.parse(fs.readFileSync(resultPath, "utf8"));
        } catch (e) {
          record = { scenario: scenario.id, runner, run: n, error: `결과 파일을 읽지 못함: ${e.message}` };
        }
        record.run = n;
        record.label = label;
        const { verdict, note } = computeVerdict(scenario, record);
        record.verdict = verdict;
        record.verdictNote = note;

        resultsStream.write(JSON.stringify(record) + "\n");
        process.stdout.write(`  -> asked=${record.asked} wroteBeforeAnswer=${record.wroteBeforeAnswer} verdict=${verdict}\n`);
      }
    }
  }

  resultsStream.end();

  // summary.md — 같은 label로 여러 번(러너를 나눠서) 실행해도 누적되도록
  // 이번 호출의 기록만이 아니라 results.jsonl 전체를 다시 읽어서 만든다.
  const cumulativeRecords = fs
    .readFileSync(resultsPath, "utf8")
    .split("\n")
    .filter(Boolean)
    .map((line) => JSON.parse(line));

  const bySc = {};
  for (const r of cumulativeRecords) {
    const key = `${r.scenario}\t${r.runner}`;
    if (!bySc[key]) bySc[key] = [];
    bySc[key].push(r);
  }
  let md = `# 인터뷰 게이트 평가 요약 — ${label}\n\n`;
  md += `생성 시각: ${new Date().toISOString()}\n\n`;
  md += `스킬 경로: \`${path.relative(REPO_ROOT, skillDir)}\`\n\n`;
  const allScenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8"));
  const scenarioById = Object.fromEntries(allScenarios.map((s) => [s.id, s]));
  const pairs = [...new Set(Object.keys(bySc))]
    .map((k) => k.split("\t"))
    .sort((a, b) => (a[0] + a[1]).localeCompare(b[0] + b[1]));

  md += `| 시나리오 | expect | 러너 | PASS | FAIL | REVIEW | ERROR | 통과율 |\n`;
  md += `|---|---|---|---|---|---|---|---|\n`;
  for (const [scId, runner] of pairs) {
    const list = bySc[`${scId}\t${runner}`] || [];
    const pass = list.filter((r) => r.verdict === "PASS").length;
    const fail = list.filter((r) => r.verdict === "FAIL").length;
    const review = list.filter((r) => r.verdict === "REVIEW").length;
    const error = list.filter((r) => r.verdict === "ERROR").length;
    const rate = list.length ? `${Math.round((pass / list.length) * 100)}%` : "-";
    const expect = scenarioById[scId] ? scenarioById[scId].expect : "?";
    md += `| ${scId} | ${expect} | ${runner} | ${pass} | ${fail} | ${review} | ${error} | ${rate} |\n`;
  }

  md += `\n## 러너별 메타(askChannel 분포, AskUserQuestion 유무, 모델)\n\n`;
  const runnerNames = [...new Set(cumulativeRecords.map((r) => r.runner))].sort();
  md += `| 러너 | 실행 수 | askChannel=tool | askChannel=text | asked=false | AskUserQuestion 노출 | 모델 |\n`;
  md += `|---|---|---|---|---|---|---|\n`;
  for (const runner of runnerNames) {
    const list = cumulativeRecords.filter((r) => r.runner === runner);
    const tool = list.filter((r) => r.askChannel === "tool").length;
    const text = list.filter((r) => r.askChannel === "text").length;
    const notAsked = list.filter((r) => !r.asked).length;
    const auq = list.some((r) => r.askUserQuestionAvailable === true)
      ? "있음"
      : list.some((r) => r.askUserQuestionAvailable === false)
        ? "없음"
        : "N/A(v1은 미측정)";
    const models = [...new Set(list.map((r) => r.model).filter(Boolean))];
    md += `| ${runner} | ${list.length} | ${tool} | ${text} | ${notAsked} | ${auq} | ${models.join(", ") || "-"} |\n`;
  }

  md += `\n## 실행별 상세\n\n`;
  for (const r of cumulativeRecords) {
    md += `### ${r.scenario} / ${r.runner} / run ${r.run} — ${r.verdict}\n\n`;
    if (r.verdictNote) md += `판정 메모: ${r.verdictNote}\n\n`;
    md += `- asked: ${r.asked}, askChannel: ${r.askChannel}, wroteBeforeAnswer: ${r.wroteBeforeAnswer}\n`;
    if (r.model || r.askUserQuestionAvailable !== undefined) {
      md += `- model: ${r.model || "-"}, askUserQuestionAvailable: ${r.askUserQuestionAvailable}\n`;
    }
    md += `- firstVisibleAction: ${String(r.firstVisibleAction || "").slice(0, 200)}\n`;
    if (r.sandboxPolicyRejections && r.sandboxPolicyRejections.length) {
      md += `- sandboxPolicyRejections: ${r.sandboxPolicyRejections.length}건 (이 실행의 exec가 막혔을 수 있음, README 참고)\n`;
    }
    if (r.questionText) {
      md += `- 질문 원문:\n\n\`\`\`\n${r.questionText.slice(0, 1500)}\n\`\`\`\n\n`;
    }
  }

  const summaryPath = path.join(runsRoot, "summary.md");
  fs.writeFileSync(summaryPath, md);
  process.stdout.write(`\n요약 저장: ${summaryPath}\n`);
  process.stdout.write(`결과 JSONL: ${resultsPath}\n`);
}

main().catch((err) => {
  console.error("run.mjs 실패:", err);
  process.exit(1);
});