feat(core,cli): harden install engine and extend skill lint and interview eval
- update only touches the recorded project root; project installs do not follow symlinks or junctions; repeated --force rotates .orig backups; uninstall never deletes outside the recorded root - run npm/npx without a shell on Windows and pin optional tool versions - skip the update check in CI and non-interactive output - lint: cross-document section pointers, preflight row ids, gate rule numbers, per-document size budgets, gate copy equality - eval: scenarios S19-S21, route and read-cost rejudging, interpretation card judging, temp-dir previews and table-form cards no longer fail
This commit is contained in:
Yun Chan
parent
ee3bef2ce5
commit
9bcb45cbd3
122 changed files with 5836 additions and 831 deletions
|
|
@ -42,7 +42,7 @@ import os from "node:os";
|
|||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import { snapshotDir, diffSnapshots } from "./lib/fsutil.mjs";
|
||||
import { scoreQuestionText } from "./lib/text-heuristic.mjs";
|
||||
import { readFollowUps } from "./lib/followups.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
|
|
@ -175,6 +175,7 @@ async function main() {
|
|||
const outPath = args.out;
|
||||
const logPath = args.log;
|
||||
const skillPath = args["skill-path"]; // workdir 안 .agents/skills/designpaca/SKILL.md
|
||||
const followUps = readFollowUps(args["follow-ups"]);
|
||||
|
||||
if (!workdir || !prompt || !outPath || !skillPath) {
|
||||
console.error("사용법: run-codex.mjs --workdir <dir> --scenario <id> --prompt <text> --skill-path <SKILL.md 경로> --out <result.json> [--log <raw.jsonl>] [--model <m>] [--timeout-ms <ms>]");
|
||||
|
|
@ -200,33 +201,17 @@ async function main() {
|
|||
const isolationToml = buildSkillIsolationToml(realHome, skillPath) + (windowsSection ? "\n" + windowsSection : "");
|
||||
fs.writeFileSync(path.join(isolatedCodexHome, "config.toml"), isolationToml);
|
||||
|
||||
const codexArgs = [
|
||||
"exec",
|
||||
"--json",
|
||||
"--skip-git-repo-check",
|
||||
"-s",
|
||||
"workspace-write",
|
||||
"-C",
|
||||
workdir,
|
||||
];
|
||||
if (model) {
|
||||
codexArgs.push("-m", model);
|
||||
}
|
||||
// 프롬프트는 인자로 넘기지 않고 stdin으로 넣는다. codex.cmd -> powershell.exe ->
|
||||
// codex.ps1 로 이어지는 Windows 셸 릴레이가 공백·한글이 섞인 긴 인자를 깨뜨리는
|
||||
// 것을 실측으로 확인했다(`codex exec` 는 프롬프트 인자가 없으면 stdin으로 읽는다).
|
||||
|
||||
const snapshotBefore = snapshotDir(workdir);
|
||||
// exec 공통 옵션. 후속 답은 `codex exec resume <thread_id>` 로 같은 세션에 이어 보낸다(프롬프트는 stdin).
|
||||
// 주의: resume 경로는 이 저장소 환경에서 아직 실측하지 않았다 — G 단계 첫 실행에서 raw.jsonl 로 확인한다.
|
||||
const baseExecArgs = ["exec", "--json", "--skip-git-repo-check", "-s", "workspace-write", "-C", workdir];
|
||||
if (model) baseExecArgs.push("-m", model);
|
||||
|
||||
const result = {
|
||||
scenario,
|
||||
runner: "codex",
|
||||
model: model || "<default>",
|
||||
asked: false,
|
||||
askChannel: null,
|
||||
questionText: null,
|
||||
wroteBeforeAnswer: null,
|
||||
firstVisibleAction: null,
|
||||
sessionId: null,
|
||||
turnsRun: 0,
|
||||
finalMessageText: null,
|
||||
toolsUsed: [],
|
||||
durationMs: null,
|
||||
|
|
@ -257,112 +242,81 @@ async function main() {
|
|||
}
|
||||
result.codexBinaryUsed = codexBin;
|
||||
|
||||
const child = spawn(codexBin, codexArgs, {
|
||||
cwd: workdir,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
shell: useShell,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: isolatedHome,
|
||||
USERPROFILE: isolatedHome,
|
||||
CODEX_HOME: isolatedCodexHome,
|
||||
},
|
||||
});
|
||||
child.stdin.write(prompt, "utf8");
|
||||
child.stdin.end();
|
||||
|
||||
let killedByUs = false;
|
||||
function killChild(reason) {
|
||||
if (killedByUs || child.killed) return;
|
||||
killedByUs = true;
|
||||
result.killedReason = reason;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// 무시
|
||||
}
|
||||
setTimeout(() => {
|
||||
// 한 턴 = codex exec 한 번. 질문이 나와도 죽이지 않고 턴이 끝날 때까지 관찰한다(01 F13).
|
||||
async function runTurn(text, resumeId) {
|
||||
const codexArgs = resumeId ? [...baseExecArgs, "resume", resumeId] : baseExecArgs;
|
||||
const snapshotBefore = snapshotDir(workdir);
|
||||
const child = spawn(codexBin, codexArgs, {
|
||||
cwd: workdir,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
shell: useShell,
|
||||
env: { ...process.env, HOME: isolatedHome, USERPROFILE: isolatedHome, CODEX_HOME: isolatedCodexHome },
|
||||
});
|
||||
child.stdin.write(text, "utf8");
|
||||
child.stdin.end();
|
||||
const timer = setTimeout(() => {
|
||||
result.killedReason = "timeout";
|
||||
try {
|
||||
if (!child.killed) child.kill("SIGKILL");
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// 무시
|
||||
}
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
const overallTimeout = setTimeout(() => killChild("timeout"), timeoutMs);
|
||||
|
||||
function markAskedIfNeeded(channel, text) {
|
||||
if (result.asked) return;
|
||||
result.asked = true;
|
||||
result.askChannel = channel;
|
||||
result.questionText = text;
|
||||
const after = snapshotDir(workdir);
|
||||
result.wroteBeforeAnswer = diffSnapshots(snapshotBefore, after).count;
|
||||
killChild("asked");
|
||||
}
|
||||
|
||||
child.stderr.on("data", (chunk) => {
|
||||
const text = chunk.toString();
|
||||
if (logStream) logStream.write(`[stderr] ${text}`);
|
||||
if (/rejected: blocked by policy|Rejected\(/.test(text)) {
|
||||
result.sandboxPolicyRejections.push(text.trim().slice(0, 500));
|
||||
}
|
||||
});
|
||||
|
||||
const rl = readline.createInterface({ input: child.stdout, crlfDelay: Infinity });
|
||||
rl.on("line", (line) => {
|
||||
if (logStream) logStream.write(line + "\n");
|
||||
if (!line.trim()) return;
|
||||
let evt;
|
||||
try {
|
||||
evt = JSON.parse(line);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (evt.type === "item.completed" && evt.item) {
|
||||
const item = evt.item;
|
||||
if (item.type === "agent_message" && item.text) {
|
||||
if (!result.firstVisibleAction) {
|
||||
result.firstVisibleAction = `text:${item.text.slice(0, 120)}`;
|
||||
setTimeout(() => {
|
||||
try {
|
||||
if (!child.killed) child.kill("SIGKILL");
|
||||
} catch {
|
||||
// 무시
|
||||
}
|
||||
const score = scoreQuestionText(item.text);
|
||||
if (score.isQuestion) {
|
||||
markAskedIfNeeded("text", item.text);
|
||||
}
|
||||
result.finalMessageText = item.text;
|
||||
} else if (item.type === "command_execution") {
|
||||
const label = `command_execution:${(item.command || "").slice(0, 80)}`;
|
||||
result.toolsUsed.push(label);
|
||||
if (!result.firstVisibleAction) {
|
||||
result.firstVisibleAction = `tool:${label}`;
|
||||
}
|
||||
} else if (item.type === "error") {
|
||||
result.toolsUsed.push(`error:${(item.message || "").slice(0, 80)}`);
|
||||
}, 3000);
|
||||
}, timeoutMs);
|
||||
child.stderr.on("data", (chunk) => {
|
||||
const t = chunk.toString();
|
||||
if (logStream) logStream.write(`[stderr] ${t}`);
|
||||
if (/rejected: blocked by policy|Rejected\(/.test(t)) result.sandboxPolicyRejections.push(t.trim().slice(0, 500));
|
||||
});
|
||||
const rl = readline.createInterface({ input: child.stdout, crlfDelay: Infinity });
|
||||
rl.on("line", (line) => {
|
||||
if (logStream) logStream.write(line + "\n");
|
||||
if (!line.trim()) return;
|
||||
let evt;
|
||||
try {
|
||||
evt = JSON.parse(line);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (evt.type === "thread.started" && evt.thread_id) result.sessionId = result.sessionId || evt.thread_id;
|
||||
if (evt.type === "item.completed" && evt.item) {
|
||||
const item = evt.item;
|
||||
if (item.type === "agent_message" && item.text) result.finalMessageText = item.text;
|
||||
else if (item.type === "command_execution") result.toolsUsed.push(`command_execution:${(item.command || "").slice(0, 80)}`);
|
||||
else if (item.type === "file_change") result.toolsUsed.push("file_change");
|
||||
else if (item.type === "error") result.toolsUsed.push(`error:${(item.message || "").slice(0, 80)}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
child.on("close", (code) => {
|
||||
result.exitCode = code;
|
||||
resolve();
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
result.error = String(err && err.message ? err.message : err);
|
||||
resolve();
|
||||
await new Promise((resolve) => {
|
||||
child.on("close", (code) => {
|
||||
result.exitCode = code;
|
||||
resolve();
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
result.error = String(err && err.message ? err.message : err);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
clearTimeout(overallTimeout);
|
||||
if (logStream) logStream.end();
|
||||
|
||||
if (result.wroteBeforeAnswer === null) {
|
||||
const after = snapshotDir(workdir);
|
||||
result.wroteBeforeAnswer = diffSnapshots(snapshotBefore, after).count;
|
||||
clearTimeout(timer);
|
||||
const changed = diffSnapshots(snapshotBefore, snapshotDir(workdir));
|
||||
if (logStream) logStream.write(JSON.stringify({ type: "eval_turn_end", filesChanged: [...changed.added, ...changed.modified] }) + "\n");
|
||||
result.turnsRun++;
|
||||
}
|
||||
|
||||
await runTurn(prompt, null);
|
||||
for (const answer of followUps) {
|
||||
if (result.error || result.killedReason === "timeout" || !result.sessionId) break;
|
||||
if (logStream) logStream.write(JSON.stringify({ type: "eval_user", text: answer, via: "resume" }) + "\n");
|
||||
await runTurn(answer, result.sessionId);
|
||||
}
|
||||
|
||||
if (logStream) await new Promise((r) => logStream.end(r));
|
||||
result.durationMs = Date.now() - startedAt;
|
||||
|
||||
if (authCopied) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue