9bcb45cbd3
- update only touches the recorded project root; project installs do not follow symlinks or junctions; repeated --force rotates .orig backups; uninstall never deletes outside the recorded root - run npm/npx without a shell on Windows and pin optional tool versions - skip the update check in CI and non-interactive output - lint: cross-document section pointers, preflight row ids, gate rule numbers, per-document size budgets, gate copy equality - eval: scenarios S19-S21, route and read-cost rejudging, interpretation card judging, temp-dir previews and table-form cards no longer fail
363 lines
17 KiB
JavaScript
363 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
// designpaca 인터뷰 게이트 로컬 평가 — 실행 스크립트.
|
|
// 자세한 사용법·판정 기준은 build/eval/interview/README.md 참고.
|
|
//
|
|
// node build/eval/interview/run.mjs --skill-dir <path> --label <name> \
|
|
// [--runners claude,claude-sdk-auto,claude-sdk-default,codex] [--repeat 3] \
|
|
// [--scenarios S1,S2] [--claude-model sonnet] [--codex-model gpt-5.1-codex] \
|
|
// [--permission-mode default] [--timeout-ms 300000]
|
|
//
|
|
// --runners 항목은 "이름:반복수"로 개별 반복수를 줄 수 있다(생략하면 --repeat).
|
|
// 예: --runners claude-sdk-auto:3,claude-sdk-default:1,codex:3
|
|
// 러너 종류:
|
|
// claude - run-claude.mjs (`claude -p` CLI, v1)
|
|
// claude-sdk-auto - run-claude-sdk.mjs, permissionMode=auto (v2, 주 지표)
|
|
// claude-sdk-default - run-claude-sdk.mjs, permissionMode=default (v2, 보조 지표)
|
|
// codex - run-codex.mjs
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { spawn } from "node:child_process";
|
|
import { copyDirRecursive, findCanaryFiles } from "./lib/fsutil.mjs";
|
|
import { judge, normalizeClaude, normalizeCodex } from "./lib/judge.mjs";
|
|
import { staticJudgements, findSkillCopy } from "./lib/static.mjs";
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
const REPO_ROOT = path.resolve(HERE, "..", "..", "..");
|
|
|
|
function parseArgs(argv) {
|
|
const out = {};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a.startsWith("--")) {
|
|
const key = a.slice(2);
|
|
const next = argv[i + 1];
|
|
if (next === undefined || next.startsWith("--")) {
|
|
out[key] = true;
|
|
} else {
|
|
out[key] = next;
|
|
i++;
|
|
}
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function runNode(scriptPath, args, logFile) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(process.execPath, [scriptPath, ...args], {
|
|
cwd: REPO_ROOT,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
let out = "";
|
|
let err = "";
|
|
child.stdout.on("data", (d) => (out += d));
|
|
child.stderr.on("data", (d) => (err += d));
|
|
child.on("close", (code) => {
|
|
if (logFile) {
|
|
fs.writeFileSync(logFile, `--- stdout ---\n${out}\n--- stderr ---\n${err}\n--- exit ---\n${code}\n`);
|
|
}
|
|
resolve({ code, out, err });
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 통과 여부를 판정한다. 러너는 원본 스트림만 남기고, 판정은 lib/judge.mjs 가 원본을 다시 읽어 내린다
|
|
* (스킬 호출 여부, 질문 후 턴 종료, 브리프 슬롯, 방향 카드 — README "판정 기준" 참고).
|
|
*/
|
|
function computeVerdict(scenario, record, rawLogPath, runner, extra) {
|
|
if (record.error) return { verdict: "ERROR", note: record.error };
|
|
let raw = "";
|
|
try {
|
|
raw = fs.readFileSync(rawLogPath, "utf8");
|
|
} catch {
|
|
return { verdict: "ERROR", note: `원본 스트림이 없다: ${rawLogPath}` };
|
|
}
|
|
const events = runner === "codex" ? normalizeCodex(raw) : normalizeClaude(raw);
|
|
return judge(scenario, events, extra);
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const skillDirArg = args["skill-dir"];
|
|
const label = args.label;
|
|
if (!skillDirArg || !label) {
|
|
console.error("사용법: run.mjs --skill-dir <path> --label <name> [--runners claude,codex] [--repeat 3] [--scenarios S1,S2] ...");
|
|
process.exit(2);
|
|
}
|
|
const skillDir = path.resolve(skillDirArg);
|
|
if (!fs.existsSync(path.join(skillDir, "SKILL.md"))) {
|
|
console.error(`오류: ${skillDir} 에 SKILL.md 가 없다.`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const defaultRepeat = Number(args.repeat || 3);
|
|
const runnerSpecs = String(args.runners || "claude,codex")
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean)
|
|
.map((spec) => {
|
|
const [name, repeatStr] = spec.split(":");
|
|
return { name, repeat: repeatStr ? Number(repeatStr) : defaultRepeat };
|
|
});
|
|
const scenarioFilter = args.scenarios ? String(args.scenarios).split(",").map((s) => s.trim()) : null;
|
|
const claudeModel = args["claude-model"] || "sonnet";
|
|
const codexModel = args["codex-model"] || null;
|
|
const permissionMode = args["permission-mode"] || null;
|
|
const timeoutMs = args["timeout-ms"] || "300000";
|
|
|
|
const scenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8")).filter(
|
|
(s) => !scenarioFilter || scenarioFilter.includes(s.id)
|
|
);
|
|
|
|
const runsRoot = path.join(REPO_ROOT, "outputs", "eval", "interview", "runs", label);
|
|
fs.mkdirSync(runsRoot, { recursive: true });
|
|
const resultsPath = path.join(runsRoot, "results.jsonl");
|
|
const resultsStream = fs.createWriteStream(resultsPath, { flags: "a" });
|
|
|
|
|
|
for (const { name: runner, repeat } of runnerSpecs) {
|
|
for (const scenario of scenarios) {
|
|
for (let n = 1; n <= repeat; n++) {
|
|
const runId = `${scenario.id}-${n}`;
|
|
const runDir = path.join(runsRoot, runner, runId);
|
|
const workdir = path.join(runDir, "work");
|
|
fs.mkdirSync(workdir, { recursive: true });
|
|
|
|
if (scenario.fixture) {
|
|
copyDirRecursive(path.join(HERE, "fixtures", scenario.fixture), workdir);
|
|
}
|
|
// 저장소에 넣을 수 없는 파일(.env 는 .gitignore 대상)은 시나리오의 seedFiles 로 실행 때 심는다.
|
|
// 인젝션 시나리오(S17·S18)의 가짜 비밀값(canary)이 여기에 들어간다.
|
|
for (const [rel, content] of Object.entries(scenario.seedFiles || {})) {
|
|
const dest = path.join(workdir, rel);
|
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
fs.writeFileSync(dest, content);
|
|
}
|
|
|
|
let skillPathForCodex = null;
|
|
if (runner === "claude" || runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
|
|
const dest = path.join(workdir, ".claude", "skills", "designpaca");
|
|
fs.mkdirSync(dest, { recursive: true });
|
|
copyDirRecursive(skillDir, dest);
|
|
} else if (runner === "codex") {
|
|
const dest = path.join(workdir, ".agents", "skills", "designpaca");
|
|
fs.mkdirSync(dest, { recursive: true });
|
|
copyDirRecursive(skillDir, dest);
|
|
skillPathForCodex = path.join(dest, "SKILL.md");
|
|
}
|
|
|
|
const rawLogExt = runner === "codex" ? "raw.jsonl" : "raw.ndjson";
|
|
const resultPath = path.join(runDir, "result.json");
|
|
const rawLogPath = path.join(runDir, rawLogExt);
|
|
const procLogPath = path.join(runDir, "process.log");
|
|
// 같은 label 로 다시 돌리면 이전 원본 스트림이 이어 붙어 판정이 섞인다.
|
|
fs.rmSync(rawLogPath, { force: true });
|
|
const scenarioTimeout = String(scenario.timeoutMs || timeoutMs);
|
|
let followUpsPath = null;
|
|
if (Array.isArray(scenario.followUps) && scenario.followUps.length) {
|
|
followUpsPath = path.join(runDir, "follow-ups.json");
|
|
fs.writeFileSync(followUpsPath, JSON.stringify(scenario.followUps));
|
|
}
|
|
|
|
process.stdout.write(`[${label}] ${runner} ${runId} 실행 중...\n`);
|
|
|
|
if (runner === "claude") {
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--model", claudeModel,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", scenarioTimeout,
|
|
];
|
|
if (permissionMode) cliArgs.push("--permission-mode", permissionMode);
|
|
if (followUpsPath) cliArgs.push("--follow-ups", followUpsPath);
|
|
await runNode(path.join(HERE, "run-claude.mjs"), cliArgs, procLogPath);
|
|
} else if (runner === "claude-sdk-auto" || runner === "claude-sdk-default") {
|
|
const mode = runner === "claude-sdk-auto" ? "auto" : "default";
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--model", claudeModel,
|
|
"--permission-mode", mode,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", scenarioTimeout,
|
|
];
|
|
if (followUpsPath) cliArgs.push("--follow-ups", followUpsPath);
|
|
await runNode(path.join(HERE, "run-claude-sdk.mjs"), cliArgs, procLogPath);
|
|
} else if (runner === "codex") {
|
|
const cliArgs = [
|
|
"--workdir", workdir,
|
|
"--scenario", scenario.id,
|
|
"--prompt", scenario.prompt,
|
|
"--skill-path", skillPathForCodex,
|
|
"--out", resultPath,
|
|
"--log", rawLogPath,
|
|
"--timeout-ms", scenarioTimeout,
|
|
];
|
|
if (codexModel) cliArgs.push("--model", codexModel);
|
|
if (followUpsPath) cliArgs.push("--follow-ups", followUpsPath);
|
|
await runNode(path.join(HERE, "run-codex.mjs"), cliArgs, procLogPath);
|
|
} else {
|
|
console.error(`알 수 없는 러너: ${runner}`);
|
|
continue;
|
|
}
|
|
|
|
let record;
|
|
try {
|
|
record = JSON.parse(fs.readFileSync(resultPath, "utf8"));
|
|
} catch (e) {
|
|
record = { scenario: scenario.id, runner, run: n, error: `결과 파일을 읽지 못함: ${e.message}` };
|
|
}
|
|
record.run = n;
|
|
record.label = label;
|
|
// 인젝션 시나리오: 심어 둔 canary 가 .env 밖의 작업 파일로 새어 나갔는지 훑는다(스킬 사본은 제외).
|
|
const extra = {};
|
|
if (scenario.injection?.canary) {
|
|
extra.workdirCanaryHits = findCanaryFiles(workdir, scenario.injection.canary, Object.keys(scenario.seedFiles || {}));
|
|
record.workdirCanaryHits = extra.workdirCanaryHits;
|
|
}
|
|
const judged = computeVerdict(scenario, record, rawLogPath, runner, extra);
|
|
const { verdict, note, ...fields } = judged;
|
|
Object.assign(record, fields);
|
|
record.verdict = verdict;
|
|
record.verdictNote = note;
|
|
// 경로 판정(V-36)·읽기 비용(V-40). 기본 판정과 따로 기록한다 — 통과율에 섞지 않는다.
|
|
// 여기서 실패해도 실행 기록은 남긴다(rejudge.mjs 로 다시 낼 수 있다).
|
|
try {
|
|
const raw = fs.readFileSync(rawLogPath, "utf8");
|
|
const events = runner === "codex" ? normalizeCodex(raw) : normalizeClaude(raw);
|
|
const st = staticJudgements(scenario, events, { skillDir: findSkillCopy(runDir) });
|
|
record.route = st.route;
|
|
record.readsVerdict = st.readsVerdict;
|
|
record.readStats = {
|
|
docCount: st.reads.docCount,
|
|
totalReads: st.reads.totalReads,
|
|
totalTokens: st.reads.totalTokens,
|
|
forbidden: st.reads.forbidden,
|
|
wholeLargeDocs: st.reads.wholeLargeDocs,
|
|
docs: st.reads.docs.map((d) => ({ doc: d.doc, reads: d.reads, wholeReads: d.wholeReads, searches: d.searches, tokens: d.tokens })),
|
|
};
|
|
} catch (e) {
|
|
record.staticJudgementError = String(e?.message || e);
|
|
}
|
|
|
|
resultsStream.write(JSON.stringify(record) + "\n");
|
|
process.stdout.write(` -> asked=${record.asked} wroteBeforeAnswer=${record.wroteBeforeAnswer} verdict=${verdict}\n`);
|
|
}
|
|
}
|
|
}
|
|
|
|
resultsStream.end();
|
|
|
|
// summary.md — 같은 label로 여러 번(러너를 나눠서) 실행해도 누적되도록
|
|
// 이번 호출의 기록만이 아니라 results.jsonl 전체를 다시 읽어서 만든다.
|
|
const cumulativeRecords = fs
|
|
.readFileSync(resultsPath, "utf8")
|
|
.split("\n")
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
|
|
const bySc = {};
|
|
for (const r of cumulativeRecords) {
|
|
const key = `${r.scenario}\t${r.runner}`;
|
|
if (!bySc[key]) bySc[key] = [];
|
|
bySc[key].push(r);
|
|
}
|
|
let md = `# 인터뷰 게이트 평가 요약 — ${label}\n\n`;
|
|
md += `생성 시각: ${new Date().toISOString()}\n\n`;
|
|
md += `스킬 경로: \`${path.relative(REPO_ROOT, skillDir)}\`\n\n`;
|
|
const allScenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8"));
|
|
const scenarioById = Object.fromEntries(allScenarios.map((s) => [s.id, s]));
|
|
const pairs = [...new Set(Object.keys(bySc))]
|
|
.map((k) => k.split("\t"))
|
|
.sort((a, b) => (a[0] + a[1]).localeCompare(b[0] + b[1]));
|
|
|
|
md += `| 시나리오 | expect | 러너 | PASS | FAIL | REVIEW | NO-TRIGGER | ERROR | 통과율 |\n`;
|
|
md += `|---|---|---|---|---|---|---|---|---|\n`;
|
|
for (const [scId, runner] of pairs) {
|
|
const list = bySc[`${scId}\t${runner}`] || [];
|
|
const pass = list.filter((r) => r.verdict === "PASS").length;
|
|
const fail = list.filter((r) => r.verdict === "FAIL").length;
|
|
const review = list.filter((r) => r.verdict === "REVIEW").length;
|
|
const error = list.filter((r) => r.verdict === "ERROR").length;
|
|
const noTrigger = list.filter((r) => r.verdict === "NO-TRIGGER").length;
|
|
const rate = list.length ? `${Math.round((pass / list.length) * 100)}%` : "-";
|
|
const expect = scenarioById[scId] ? scenarioById[scId].expect : "?";
|
|
md += `| ${scId} | ${expect} | ${runner} | ${pass} | ${fail} | ${review} | ${noTrigger} | ${error} | ${rate} |\n`;
|
|
}
|
|
|
|
md += `\n## 러너별 메타(askChannel 분포, AskUserQuestion 유무, 모델)\n\n`;
|
|
const runnerNames = [...new Set(cumulativeRecords.map((r) => r.runner))].sort();
|
|
md += `| 러너 | 실행 수 | askChannel=tool | askChannel=text | asked=false | AskUserQuestion 노출 | 모델 |\n`;
|
|
md += `|---|---|---|---|---|---|---|\n`;
|
|
for (const runner of runnerNames) {
|
|
const list = cumulativeRecords.filter((r) => r.runner === runner);
|
|
const tool = list.filter((r) => r.askChannel === "tool").length;
|
|
const text = list.filter((r) => r.askChannel === "text").length;
|
|
const notAsked = list.filter((r) => !r.asked).length;
|
|
const auq = list.some((r) => r.askUserQuestionAvailable === true)
|
|
? "있음"
|
|
: list.some((r) => r.askUserQuestionAvailable === false)
|
|
? "없음"
|
|
: "N/A(v1은 미측정)";
|
|
const models = [...new Set(list.map((r) => r.model).filter(Boolean))];
|
|
md += `| ${runner} | ${list.length} | ${tool} | ${text} | ${notAsked} | ${auq} | ${models.join(", ") || "-"} |\n`;
|
|
}
|
|
|
|
// 경로·읽기 비용 표. 이 필드가 없는 이전 기록(같은 label 에 누적된 옛 실행)은 건너뛴다.
|
|
const withStatic = cumulativeRecords.filter((r) => r.readStats);
|
|
if (withStatic.length) {
|
|
md += `\n## 경로·읽기 비용(정적 판정)\n\n`;
|
|
md += `기본 판정과 따로 낸다. 토큰은 추정이다. 기존 기록은 \`node build/eval/interview/rejudge.mjs --label ${label}\` 로 다시 낼 수 있다.\n\n`;
|
|
md += `| 시나리오 | 러너 | run | 기대 경로 | 말한 경로 | 경로 판정 | 읽은 문서 | 추정 토큰 | 읽기 판정 | 메모 |\n`;
|
|
md += `|---|---|---|---|---|---|---|---|---|---|\n`;
|
|
for (const r of withStatic) {
|
|
const memo = [
|
|
r.route && r.route.verdict !== "PASS" ? r.route.note : "",
|
|
r.readsVerdict && r.readsVerdict.verdict !== "PASS" ? r.readsVerdict.note : "",
|
|
r.readStats.wholeLargeDocs.length ? `큰 문서 통째: ${r.readStats.wholeLargeDocs.join(", ")}` : "",
|
|
]
|
|
.filter(Boolean)
|
|
.join(" / ")
|
|
.replace(/\|/g, "\\|");
|
|
md += `| ${r.scenario} | ${r.runner} | ${r.run} | ${r.route?.expected ?? "-"} | ${r.route?.route ?? "-"} | ${r.route?.verdict ?? "-"} | ${r.readStats.docCount} | ${r.readStats.totalTokens ?? "-"} | ${r.readsVerdict?.verdict ?? "-"} | ${memo} |\n`;
|
|
}
|
|
}
|
|
|
|
md += `\n## 실행별 상세\n\n`;
|
|
for (const r of cumulativeRecords) {
|
|
md += `### ${r.scenario} / ${r.runner} / run ${r.run} — ${r.verdict}\n\n`;
|
|
if (r.verdictNote) md += `판정 메모: ${r.verdictNote}\n\n`;
|
|
if (r.injectionViolations !== undefined) {
|
|
md += `- injection: 위반 ${r.injectionViolations.length}건, 주입 파일 열람 ${r.injectionSourceRead}, 보고 ${r.injectionReported}\n`;
|
|
}
|
|
md += `- asked: ${r.asked}, askChannel: ${r.askChannel}, wroteBeforeAnswer: ${r.wroteBeforeAnswer}\n`;
|
|
if (r.model || r.askUserQuestionAvailable !== undefined) {
|
|
md += `- model: ${r.model || "-"}, askUserQuestionAvailable: ${r.askUserQuestionAvailable}\n`;
|
|
}
|
|
md += `- firstVisibleAction: ${String(r.firstVisibleAction || "").slice(0, 200)}\n`;
|
|
if (r.sandboxPolicyRejections && r.sandboxPolicyRejections.length) {
|
|
md += `- sandboxPolicyRejections: ${r.sandboxPolicyRejections.length}건 (이 실행의 exec가 막혔을 수 있음, README 참고)\n`;
|
|
}
|
|
if (r.questionText) {
|
|
md += `- 질문 원문:\n\n\`\`\`\n${r.questionText.slice(0, 1500)}\n\`\`\`\n\n`;
|
|
}
|
|
}
|
|
|
|
const summaryPath = path.join(runsRoot, "summary.md");
|
|
fs.writeFileSync(summaryPath, md);
|
|
process.stdout.write(`\n요약 저장: ${summaryPath}\n`);
|
|
process.stdout.write(`결과 JSONL: ${resultsPath}\n`);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("run.mjs 실패:", err);
|
|
process.exit(1);
|
|
});
|