295 lines
12 KiB
JavaScript
295 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
// designpaca 인터뷰 게이트 로컬 평가 — 실행 스크립트.
|
|
// 자세한 사용법·판정 기준은 build/eval/interview/README.md 참고.
|
|
//
|
|
// node build/eval/interview/run.mjs --skill-dir <path> --label <name> \
|
|
// [--runners claude,claude-sdk-auto,claude-sdk-default,codex] [--repeat 3] \
|
|
// [--scenarios S1,S2] [--claude-model sonnet] [--codex-model gpt-5.1-codex] \
|
|
// [--permission-mode default] [--timeout-ms 300000]
|
|
//
|
|
// --runners 항목은 "이름:반복수"로 개별 반복수를 줄 수 있다(생략하면 --repeat).
|
|
// 예: --runners claude-sdk-auto:3,claude-sdk-default:1,codex:3
|
|
// 러너 종류:
|
|
// claude - run-claude.mjs (`claude -p` CLI, v1)
|
|
// claude-sdk-auto - run-claude-sdk.mjs, permissionMode=auto (v2, 주 지표)
|
|
// claude-sdk-default - run-claude-sdk.mjs, permissionMode=default (v2, 보조 지표)
|
|
// codex - run-codex.mjs
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { spawn } from "node:child_process";
|
|
import { copyDirRecursive } from "./lib/fsutil.mjs";
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
const REPO_ROOT = path.resolve(HERE, "..", "..", "..");
|
|
|
|
function parseArgs(argv) {
|
|
const out = {};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a.startsWith("--")) {
|
|
const key = a.slice(2);
|
|
const next = argv[i + 1];
|
|
if (next === undefined || next.startsWith("--")) {
|
|
out[key] = true;
|
|
} else {
|
|
out[key] = next;
|
|
i++;
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function runNode(scriptPath, args, logFile) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(process.execPath, [scriptPath, ...args], {
|
|
cwd: REPO_ROOT,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
let out = "";
|
|
let err = "";
|
|
child.stdout.on("data", (d) => (out += d));
|
|
child.stderr.on("data", (d) => (err += d));
|
|
child.on("close", (code) => {
|
|
if (logFile) {
|
|
fs.writeFileSync(logFile, `--- stdout ---\n${out}\n--- stderr ---\n${err}\n--- exit ---\n${code}\n`);
|
|
}
|
|
resolve({ code, out, err });
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 통과 여부를 판정한다. 기계적으로 판정 가능한 것만 pass/fail 로 확정하고,
|
|
* 원문을 직접 읽어야 하는 경우(S7의 "범위 질문 1개 이하" 같은 것)는 REVIEW 로 남긴다.
|
|
*/
|
|
function computeVerdict(scenario, record) {
|
|
if (record.error) return { verdict: "ERROR", note: record.error };
|
|
if (scenario.expect === "ask") {
|
|
if (record.asked && record.wroteBeforeAnswer === 0) return { verdict: "PASS", note: "" };
|
|
if (!record.asked) return { verdict: "FAIL", note: "질문 없이 끝남(또는 타임아웃)" };
|
|
return { verdict: "FAIL", note: `질문 전에 파일 ${record.wroteBeforeAnswer}건 생성/수정` };
|
|
}
|
|
// expect === "no-interview"
|
|
if (!record.asked) return { verdict: "PASS", note: "" };
|
|
if (scenario.allowScopeQuestion) {
|
|
return { verdict: "REVIEW", note: "질문이 나왔다 — 범위 확인 질문 1개 이하인지 원문을 직접 읽고 판정할 것" };
|
|
}
|
|
return { verdict: "FAIL", note: "위임/국소 경로인데 인터뷰가 나옴" };
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const skillDirArg = args["skill-dir"];
|
|
const label = args.label;
|
|
if (!skillDirArg || !label) {
|
|
console.error("사용법: run.mjs --skill-dir <path> --label <name> [--runners claude,codex] [--repeat 3] [--scenarios S1,S2] ...");
|
|
process.exit(2);
|
|
}
|
|
const skillDir = path.resolve(skillDirArg);
|
|
if (!fs.existsSync(path.join(skillDir, "SKILL.md"))) {
|
|
console.error(`오류: ${skillDir} 에 SKILL.md 가 없다.`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const defaultRepeat = Number(args.repeat || 3);
|
|
const runnerSpecs = String(args.runners || "claude,codex")
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean)
|
|
.map((spec) => {
|
|
const [name, repeatStr] = spec.split(":");
|
|
return { name, repeat: repeatStr ? Number(repeatStr) : defaultRepeat };
|
|
});
|
|
const scenarioFilter = args.scenarios ? String(args.scenarios).split(",").map((s) => s.trim()) : null;
|
|
const claudeModel = args["claude-model"] || "sonnet";
|
|
const codexModel = args["codex-model"] || null;
|
|
const permissionMode = args["permission-mode"] || null;
|
|
const timeoutMs = args["timeout-ms"] || "300000";
|
|
|
|
const scenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8")).filter(
|
|
(s) => !scenarioFilter || scenarioFilter.includes(s.id)
|
|
);
|
|
|
|
const runsRoot = path.join(REPO_ROOT, "outputs", "eval", "interview", "runs", label);
|
|
fs.mkdirSync(runsRoot, { recursive: true });
|
|
const resultsPath = path.join(runsRoot, "results.jsonl");
|
|
const resultsStream = fs.createWriteStream(resultsPath, { flags: "a" });
|
|
|
|
|
|
for (const { name: runner, repeat } of runnerSpecs) {
|
|
for (const scenario of scenarios) {
|
|
for (let n = 1; n <= repeat; n++) {
|
|
const runId = `${scenario.id}-${n}`;
|
|
const runDir = path.join(runsRoot, runner, runId);
|
|
const workdir = path.join(runDir, "work");
|
|
fs.mkdirSync(workdir, { recursive: true });
|
|
|
|
if (scenario.fixture) {
|
|
copyDirRecursive(path.join(HERE, "fixtures", scenario.fixture), workdir);
|
|
}
|
|
|
|
let skillPathForCodex = null;
|
|
if (runner === "claude" || runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
|
|
const dest = path.join(workdir, ".claude", "skills", "designpaca");
|
|
fs.mkdirSync(dest, { recursive: true });
|
|
copyDirRecursive(skillDir, dest);
|
|
} else if (runner === "codex") {
|
|
const dest = path.join(workdir, ".agents", "skills", "designpaca");
|
|
fs.mkdirSync(dest, { recursive: true });
|
|
copyDirRecursive(skillDir, dest);
|
|
skillPathForCodex = path.join(dest, "SKILL.md");
|
|
}
|
|
|
|
const rawLogExt = runner === "codex" ? "raw.jsonl" : "raw.ndjson";
|
|
const resultPath = path.join(runDir, "result.json");
|
|
const rawLogPath = path.join(runDir, rawLogExt);
|
|
const procLogPath = path.join(runDir, "process.log");
|
|
|
|
process.stdout.write(`[${label}] ${runner} ${runId} 실행 중...\n`);
|
|
|
|
if (runner === "claude") {
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--model", claudeModel,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", String(timeoutMs),
|
|
];
|
|
if (permissionMode) cliArgs.push("--permission-mode", permissionMode);
|
|
await runNode(path.join(HERE, "run-claude.mjs"), cliArgs, procLogPath);
|
|
} else if (runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
|
|
const mode = runner === "claude-sdk-auto" ? "auto" : "default";
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--model", claudeModel,
|
|
"--permission-mode", mode,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", String(timeoutMs),
|
|
];
|
|
await runNode(path.join(HERE, "run-claude-sdk.mjs"), cliArgs, procLogPath);
|
|
} else if (runner === "codex") {
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--skill-path", skillPathForCodex,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", String(timeoutMs),
|
|
];
|
|
if (codexModel) cliArgs.push("--model", codexModel);
|
|
await runNode(path.join(HERE, "run-codex.mjs"), cliArgs, procLogPath);
|
|
} else {
|
|
console.error(`알 수 없는 러너: ${runner}`);
|
|
continue;
|
|
}
|
|
|
|
let record;
|
|
try {
|
|
record = JSON.parse(fs.readFileSync(resultPath, "utf8"));
|
|
} catch (e) {
|
|
record = { scenario: scenario.id, runner, run: n, error: `결과 파일을 읽지 못함: ${e.message}` };
|
|
}
|
|
record.run = n;
|
|
record.label = label;
|
|
const { verdict, note } = computeVerdict(scenario, record);
|
|
record.verdict = verdict;
|
|
record.verdictNote = note;
|
|
|
|
resultsStream.write(JSON.stringify(record) + "\n");
|
|
process.stdout.write(` -> asked=${record.asked} wroteBeforeAnswer=${record.wroteBeforeAnswer} verdict=${verdict}\n`);
|
|
}
|
|
}
|
|
}
|
|
|
|
resultsStream.end();
|
|
|
|
// summary.md — 같은 label로 여러 번(러너를 나눠서) 실행해도 누적되도록
|
|
// 이번 호출의 기록만이 아니라 results.jsonl 전체를 다시 읽어서 만든다.
|
|
const cumulativeRecords = fs
|
|
.readFileSync(resultsPath, "utf8")
|
|
.split("\n")
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
|
|
const bySc = {};
|
|
for (const r of cumulativeRecords) {
|
|
const key = `${r.scenario}\t${r.runner}`;
|
|
if (!bySc[key]) bySc[key] = [];
|
|
bySc[key].push(r);
|
|
}
|
|
let md = `# 인터뷰 게이트 평가 요약 — ${label}\n\n`;
|
|
md += `생성 시각: ${new Date().toISOString()}\n\n`;
|
|
md += `스킬 경로: \`${path.relative(REPO_ROOT, skillDir)}\`\n\n`;
|
|
const allScenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8"));
|
|
const scenarioById = Object.fromEntries(allScenarios.map((s) => [s.id, s]));
|
|
const pairs = [...new Set(Object.keys(bySc))]
|
|
.map((k) => k.split("\t"))
|
|
.sort((a, b) => (a[0] + a[1]).localeCompare(b[0] + b[1]));
|
|
|
|
md += `| 시나리오 | expect | 러너 | PASS | FAIL | REVIEW | ERROR | 통과율 |\n`;
|
|
md += `|---|---|---|---|---|---|---|---|\n`;
|
|
for (const [scId, runner] of pairs) {
|
|
const list = bySc[`${scId}\t${runner}`] || [];
|
|
const pass = list.filter((r) => r.verdict === "PASS").length;
|
|
const fail = list.filter((r) => r.verdict === "FAIL").length;
|
|
const review = list.filter((r) => r.verdict === "REVIEW").length;
|
|
const error = list.filter((r) => r.verdict === "ERROR").length;
|
|
const rate = list.length ? `${Math.round((pass / list.length) * 100)}%` : "-";
|
|
const expect = scenarioById[scId] ? scenarioById[scId].expect : "?";
|
|
md += `| ${scId} | ${expect} | ${runner} | ${pass} | ${fail} | ${review} | ${error} | ${rate} |\n`;
|
|
}
|
|
|
|
md += `\n## 러너별 메타(askChannel 분포, AskUserQuestion 유무, 모델)\n\n`;
|
|
const runnerNames = [...new Set(cumulativeRecords.map((r) => r.runner))].sort();
|
|
md += `| 러너 | 실행 수 | askChannel=tool | askChannel=text | asked=false | AskUserQuestion 노출 | 모델 |\n`;
|
|
md += `|---|---|---|---|---|---|---|\n`;
|
|
for (const runner of runnerNames) {
|
|
const list = cumulativeRecords.filter((r) => r.runner === runner);
|
|
const tool = list.filter((r) => r.askChannel === "tool").length;
|
|
const text = list.filter((r) => r.askChannel === "text").length;
|
|
const notAsked = list.filter((r) => !r.asked).length;
|
|
const auq = list.some((r) => r.askUserQuestionAvailable === true)
|
|
? "있음"
|
|
: list.some((r) => r.askUserQuestionAvailable === false)
|
|
? "없음"
|
|
: "N/A(v1은 미측정)";
|
|
const models = [...new Set(list.map((r) => r.model).filter(Boolean))];
|
|
md += `| ${runner} | ${list.length} | ${tool} | ${text} | ${notAsked} | ${auq} | ${models.join(", ") || "-"} |\n`;
|
|
}
|
|
|
|
md += `\n## 실행별 상세\n\n`;
|
|
for (const r of cumulativeRecords) {
|
|
md += `### ${r.scenario} / ${r.runner} / run ${r.run} — ${r.verdict}\n\n`;
|
|
if (r.verdictNote) md += `판정 메모: ${r.verdictNote}\n\n`;
|
|
md += `- asked: ${r.asked}, askChannel: ${r.askChannel}, wroteBeforeAnswer: ${r.wroteBeforeAnswer}\n`;
|
|
if (r.model || r.askUserQuestionAvailable !== undefined) {
|
|
md += `- model: ${r.model || "-"}, askUserQuestionAvailable: ${r.askUserQuestionAvailable}\n`;
|
|
}
|
|
md += `- firstVisibleAction: ${String(r.firstVisibleAction || "").slice(0, 200)}\n`;
|
|
if (r.sandboxPolicyRejections && r.sandboxPolicyRejections.length) {
|
|
md += `- sandboxPolicyRejections: ${r.sandboxPolicyRejections.length}건 (이 실행의 exec가 막혔을 수 있음, README 참고)\n`;
|
|
}
|
|
if (r.questionText) {
|
|
md += `- 질문 원문:\n\n\`\`\`\n${r.questionText.slice(0, 1500)}\n\`\`\`\n\n`;
|
|
}
|
|
}
|
|
|
|
const summaryPath = path.join(runsRoot, "summary.md");
|
|
fs.writeFileSync(summaryPath, md);
|
|
process.stdout.write(`\n요약 저장: ${summaryPath}\n`);
|
|
process.stdout.write(`결과 JSONL: ${resultsPath}\n`);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("run.mjs 실패:", err);
|
|
process.exit(1);
|
|
});
|