feat(core,cli): harden install engine and extend skill lint and interview eval

- update only touches the recorded project root; project installs do not
  follow symlinks or junctions; repeated --force rotates .orig backups;
  uninstall never deletes outside the recorded root
- run npm/npx without a shell on Windows and pin optional tool versions
- skip the update check in CI and non-interactive output
- lint: cross-document section pointers, preflight row ids, gate rule
  numbers, per-document size budgets, gate copy equality
- eval: scenarios S19-S21, route and read-cost rejudging, interpretation
  card judging, temp-dir previews and table-form cards no longer fail
This commit is contained in:
Yun Chan 2026-10-06 16:55:38 +09:00
parent ee3bef2ce5
commit 9bcb45cbd3
122 changed files with 5836 additions and 831 deletions

View file

@ -0,0 +1,122 @@
#!/usr/bin/env node
// 기존 실행 기록의 정적 재판정. 모델을 부르지 않는다 — run.mjs 가 남긴 원본 스트림(raw.ndjson·raw.jsonl)을
// 다시 읽어 (1) 기본 판정, (2) 경로 판정(V-36), (3) 읽기 비용(V-40)을 낸다.
//
// node build/eval/interview/rejudge.mjs --label redteam-2026-10-02[,다른-라벨]
// [--runs-root outputs/eval/interview/runs] [--scenarios S4,S6] [--out <표.md>] [--json <상세.json>]
//
// --label 은 콤마로 여럿 줄 수 있다. 결과는 표준 출력으로 내고, --out·--json 을 주면 그 파일에도 쓴다
// (저장소 안에 쓰지 않도록 outputs/ 아래나 임시 경로를 준다).
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { judge, normalizeClaude, normalizeCodex } from "./lib/judge.mjs";
import { findCanaryFiles } from "./lib/fsutil.mjs";
import { staticJudgements, findSkillCopy } from "./lib/static.mjs";
const HERE = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(HERE, "..", "..", "..");
function parseArgs(argv) {
const out = {};
for (let i = 0; i < argv.length; i++) {
if (!argv[i].startsWith("--")) continue;
const next = argv[i + 1];
if (next === undefined || next.startsWith("--")) out[argv[i].slice(2)] = true;
else out[argv[i].slice(2)] = argv[++i];
}
return out;
}
/**
* 실행 디렉터리 하나를 재판정한다. 원본 스트림이 없으면 null.
* @returns {{scenario, runner, run, verdict, note, route, reads, readsVerdict, readsLine}|null}
*/
export function rejudgeRun(scenario, runner, runDir) {
const isCodex = runner === "codex";
const rawPath = path.join(runDir, isCodex ? "raw.jsonl" : "raw.ndjson");
if (!fs.existsSync(rawPath)) return null;
const events = (isCodex ? normalizeCodex : normalizeClaude)(fs.readFileSync(rawPath, "utf8"));
const extra = {};
if (scenario.injection?.canary) {
extra.workdirCanaryHits = findCanaryFiles(path.join(runDir, "work"), scenario.injection.canary, Object.keys(scenario.seedFiles || {}));
}
const base = judge(scenario, events, extra);
const st = staticJudgements(scenario, events, { skillDir: findSkillCopy(runDir) });
return { scenario: scenario.id, runner, run: path.basename(runDir), verdict: base.verdict, note: base.note, ...st };
}
/** 라벨 하나(runs/<label>/<runner>/<S6-1>/)를 훑는다. */
export function rejudgeLabel(runsRoot, label, scenarios, filter) {
const byId = Object.fromEntries(scenarios.map((s) => [s.id, s]));
const out = [];
const root = path.join(runsRoot, label);
for (const runner of fs.readdirSync(root).sort()) {
const rd = path.join(root, runner);
if (!fs.statSync(rd).isDirectory()) continue;
for (const run of fs.readdirSync(rd).sort((a, b) => a.localeCompare(b, "en", { numeric: true }))) {
const sc = byId[run.split("-")[0]];
if (!sc || (filter && !filter.includes(sc.id))) continue;
const r = rejudgeRun(sc, runner, path.join(rd, run));
if (r) out.push({ label, ...r });
}
}
return out;
}
const cell = (s) => String(s ?? "-").replace(/\|/g, "\\|").replace(/\n/g, " ");
export function renderMarkdown(rows) {
let md = "# 정적 재판정 — 경로·읽기 비용\n\n모델 호출 없이 기존 원본 스트림을 다시 읽은 결과다. 토큰은 추정이다(한글 음절 × 1 + 그 밖의 문자 ÷ 3.5).\n\n";
md += "## 실행별\n\n| 라벨 | 러너 | 실행 | 기본 판정 | 기대 경로 | 말한 경로 | 경로 판정 | 읽은 문서 | 추정 토큰 | 읽기 판정 | 메모 |\n|---|---|---|---|---|---|---|---:|---:|---|---|\n";
for (const r of rows) {
const memo = [r.route?.verdict && r.route.verdict !== "PASS" ? r.route.note : "", r.readsVerdict?.verdict && r.readsVerdict.verdict !== "PASS" ? r.readsVerdict.note : "", r.reads.wholeLargeDocs.length ? `큰 문서 통째: ${r.reads.wholeLargeDocs.join(", ")}` : ""]
.filter(Boolean)
.join(" / ");
md += `| ${cell(r.label)} | ${cell(r.runner)} | ${cell(r.run)} | ${cell(r.verdict)} | ${cell(r.route?.expected)} | ${cell(r.route?.route)} | ${cell(r.route?.verdict)} | ${r.reads.docCount} | ${cell(r.reads.totalTokens)} | ${cell(r.readsVerdict?.verdict)} | ${cell(memo)} |\n`;
}
// 시나리오 × 러너 집계. 읽기 토큰은 스킬이 실제로 문서를 읽은 실행만 평균 낸다.
const groups = new Map();
for (const r of rows) {
const k = `${r.scenario}\t${r.runner}`;
if (!groups.has(k)) groups.set(k, []);
groups.get(k).push(r);
}
md += "\n## 시나리오 × 러너\n\n| 시나리오 | 러너 | 실행 수 | 경로 PASS | 경로 FAIL | 경로 REVIEW(미표기 포함) | 평균 읽은 문서 | 평균 추정 토큰 | 최대 추정 토큰 | 금지 읽기 |\n|---|---|---:|---:|---:|---:|---:|---:|---:|---:|\n";
const keys = [...groups.keys()].sort((a, b) => a.localeCompare(b, "en", { numeric: true }));
for (const k of keys) {
const list = groups.get(k);
const [sc, runner] = k.split("\t");
const routed = list.filter((r) => r.route);
const n = (v) => (routed.length ? routed.filter((r) => r.route.verdict === v).length : "-");
const withReads = list.filter((r) => r.reads.docCount > 0);
const toks = withReads.map((r) => r.reads.totalTokens).filter((t) => t !== null);
const avg = (a) => (a.length ? Math.round(a.reduce((x, y) => x + y, 0) / a.length) : "-");
md += `| ${sc} | ${runner} | ${list.length} | ${n("PASS")} | ${n("FAIL")} | ${n("REVIEW")} | ${avg(withReads.map((r) => r.reads.docCount))} | ${avg(toks)} | ${toks.length ? Math.max(...toks) : "-"} | ${list.filter((r) => r.reads.forbidden.length).length} |\n`;
}
return md;
}
const isMain = process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMain) {
const args = parseArgs(process.argv.slice(2));
if (!args.label || args.label === true) {
console.error("사용법: rejudge.mjs --label <라벨[,라벨2]> [--runs-root <dir>] [--scenarios S4,S6] [--out <표.md>] [--json <상세.json>]");
process.exit(2);
}
const runsRoot = path.resolve(args["runs-root"] && args["runs-root"] !== true ? args["runs-root"] : path.join(REPO_ROOT, "outputs", "eval", "interview", "runs"));
const scenarios = JSON.parse(fs.readFileSync(path.join(HERE, "scenarios.json"), "utf8"));
const filter = typeof args.scenarios === "string" ? args.scenarios.split(",").map((s) => s.trim()) : null;
const rows = [];
for (const label of String(args.label).split(",").map((s) => s.trim()).filter(Boolean)) {
if (!fs.existsSync(path.join(runsRoot, label))) {
console.error(`오류: 실행 기록이 없다: ${path.join(runsRoot, label)}`);
process.exit(2);
}
rows.push(...rejudgeLabel(runsRoot, label, scenarios, filter));
}
const md = renderMarkdown(rows);
process.stdout.write(md);
if (typeof args.out === "string") fs.writeFileSync(args.out, md);
if (typeof args.json === "string") fs.writeFileSync(args.json, JSON.stringify(rows, null, 1));
}