designpaca/build/eval/interview/test/read-stats.test.mjs

// 읽기 비용 집계(V-40) 단위 테스트. 모델을 부르지 않는다. 명령 예문은 실제 실행 기록
// (outputs/eval/interview/runs/redteam-2026-10-02, 2026-10-02)의 Claude `Read`·Codex `command_execution` 입력 형식이다.
//
//   node --test build/eval/interview/test/read-stats.test.mjs
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { extractSkillReads, readStats, judgeReads, formatReadStats, FORBIDDEN_READS } from "../lib/read-stats.mjs";
import { normalizeClaude, normalizeCodex } from "../lib/judge.mjs";
import { staticJudgements, findSkillCopy } from "../lib/static.mjs";
import { rejudgeLabel, renderMarkdown } from "../rejudge.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const SCENARIOS = JSON.parse(fs.readFileSync(path.join(HERE, "..", "scenarios.json"), "utf8"));
const byId = Object.fromEntries(SCENARIOS.map((s) => [s.id, s]));

const claudeLine = (name, input) => JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", id: "tu", name, input }] } });
const codexLine = (command) => JSON.stringify({ type: "item.completed", item: { type: "command_execution", command } });
const WIN = "D:\\workspace\\runs\\S6-1\\work\\.claude\\skills\\designpaca\\";
const PWSH = '"C:\\\\Program Files\\\\PowerShell\\\\7\\\\pwsh.exe" -Command ';

let tmp;
let skillDir;
// 가짜 스킬 사본: 문서마다 줄 수를 알고 있어야 범위 읽기의 토큰·비율을 단언할 수 있다.
const line = (i) => `줄 ${String(i).padStart(4, "0")} 가나다라마바사아자차카타파하`;
const doc = (n) => Array.from({ length: n }, (_, i) => line(i + 1)).join("\n");

before(() => {
  tmp = fs.mkdtempSync(path.join(os.tmpdir(), "dp-reads-"));
  skillDir = path.join(tmp, "run", "claude-sdk-auto", "S6-1", "work", ".claude", "skills", "designpaca");
  fs.mkdirSync(path.join(skillDir, "references", "presets"), { recursive: true });
  fs.writeFileSync(path.join(skillDir, "SKILL.md"), doc(100));
  fs.writeFileSync(path.join(skillDir, "references", "color.md"), doc(200));
  fs.writeFileSync(path.join(skillDir, "references", "evidence-ledger.md"), doc(400));
  fs.writeFileSync(path.join(skillDir, "references", "typography.md"), doc(1000));
  fs.writeFileSync(path.join(skillDir, "references", "presets", "editorial.md"), doc(50));
});

after(() => {
  fs.rmSync(tmp, { recursive: true, force: true });
});

test("Claude: Read 의 경로·offset·limit, Grep(검색), Skill 호출을 읽는다", () => {
  const raw = [
    claudeLine("Skill", { skill: "designpaca" }),
    claudeLine("Read", { file_path: `${WIN}references\\color.md` }),
    claudeLine("Read", { file_path: `${WIN}references\\typography.md`, offset: 101, limit: 50 }),
    claudeLine("Read", { file_path: `${WIN}references\\presets\\editorial.md`, limit: 20 }),
    claudeLine("Grep", { pattern: "hover", path: `${WIN}references\\color.md`, output_mode: "content" }),
    claudeLine("Read", { file_path: "D:\\workspace\\runs\\S6-1\\work\\index.html" }),
    claudeLine("Write", { file_path: `${WIN}references\\color.md`, content: "x" }),
  ].join("\n");
  const reads = extractSkillReads(normalizeClaude(raw));
  assert.deepEqual(
    reads.map((r) => `${r.kind}:${r.doc}:${r.from ?? "-"}:${r.lines ?? "-"}`),
    ["read:SKILL.md:1:-", "read:references/color.md:1:-", "read:references/typography.md:101:50", "read:references/presets/editorial.md:1:20", "search:references/color.md:-:-"],
  );
});

test("Codex: 한 명령에 `;` 로 이어진 Get-Content 여러 개와 -TotalCount·-Skip/-First·-Raw 를 읽는다", () => {
  const raw = [
    codexLine(`${PWSH}'Get-Content -Raw .agents/skills/designpaca/SKILL.md'`),
    codexLine(`${PWSH}'Get-Content .agents/skills/designpaca/references/color.md -TotalCount 80; Get-Content .agents/skills/designpaca/references/typography.md | Select-Object -Skip 91 -First 75; Get-Content index.html'`),
    codexLine(`${PWSH}'Get-Content -Raw README.md, package.json, .agents/skills/designpaca/references/color.md, .agents/skills/designpaca/references/evidence-ledger.md'`),
    codexLine(`${PWSH}'rg -n "hover" .agents/skills/designpaca/references/color.md'`),
    codexLine(`${PWSH}'Get-ChildItem .agents/skills/designpaca/references/color.md | Select-Object Length'`),
  ].join("\n");
  const reads = extractSkillReads(normalizeCodex(raw));
  assert.deepEqual(
    reads.map((r) => `${r.kind}:${r.doc}:${r.from ?? "-"}:${r.lines ?? "-"}`),
    [
      "read:SKILL.md:1:-",
      "read:references/color.md:1:80",
      "read:references/typography.md:92:75",
      "read:references/color.md:1:-",
      "read:references/evidence-ledger.md:1:-",
      "search:references/color.md:-:-",
    ],
  );
});

test("셸: cat·head·tail·sed -n 과, 스킬 디렉터리로 들어간 뒤의 상대 경로를 읽는다", () => {
  const ev = [
    { kind: "tool", name: "Bash", input: { command: "cat .claude/skills/designpaca/references/color.md | head -n 40" }, paths: [] },
    { kind: "tool", name: "Bash", input: { command: "sed -n '120,180p' .claude/skills/designpaca/references/typography.md" }, paths: [] },
    { kind: "tool", name: "Bash", input: { command: "tail -n 30 .claude/skills/designpaca/SKILL.md" }, paths: [] },
    { kind: "tool", name: "Bash", input: { command: "cd .claude/skills/designpaca && cat references/color.md" }, paths: [] },
    { kind: "tool", name: "Bash", input: { command: "cat references/color.md" }, paths: [] }, // 스킬 이름이 없으면 사용자 프로젝트의 파일일 수 있다
    { kind: "tool", name: "Bash", input: { command: "ls .claude/skills/designpaca/references/color.md" }, paths: [] },
  ];
  assert.deepEqual(
    extractSkillReads(ev).map((r) => `${r.doc}:${r.from ?? "tail"}:${r.lines ?? "-"}`),
    ["references/color.md:1:40", "references/typography.md:120:61", "SKILL.md:tail:30", "references/color.md:1:-"],
  );
});

test("readStats: 문서별 횟수·추정 토큰·읽은 비율을 내고, 여러 번 나눠 읽은 줄은 합쳐 센다", () => {
  const raw = [
    claudeLine("Skill", { skill: "designpaca" }),
    claudeLine("Read", { file_path: `${WIN}references\\typography.md`, offset: 1, limit: 100 }),
    claudeLine("Read", { file_path: `${WIN}references\\typography.md`, offset: 51, limit: 150 }),
    claudeLine("Grep", { pattern: "x", path: `${WIN}references\\color.md` }),
  ].join("\n");
  const st = readStats(normalizeClaude(raw), { skillDir });
  assert.equal(st.docCount, 2); // 검색만 한 color.md 는 읽은 문서로 세지 않는다
  assert.equal(st.totalReads, 3);
  assert.equal(st.totalSearches, 1);
  assert.equal(st.tokensKnown, true);
  const typo = st.docs.find((d) => d.doc === "references/typography.md");
  assert.equal(typo.reads, 2);
  assert.equal(typo.wholeReads, 0);
  assert.equal(typo.coverage, 0.2); // 1~200줄 / 1000줄 — 겹친 51~100줄을 두 번 세지 않는다
  const skill = st.docs.find((d) => d.doc === "SKILL.md");
  assert.equal(skill.wholeReads, 1);
  assert.equal(skill.tokens, skill.docTokens);
  // 토큰은 실제로 읽은 줄(100 + 150줄)의 글자로 추정한다 — 겹쳐 읽은 줄은 비용이 두 번 든다.
  assert.ok(typo.tokens > typo.docTokens * 0.22 && typo.tokens < typo.docTokens * 0.28, `${typo.tokens} / ${typo.docTokens}`);
  assert.equal(st.totalTokens, skill.tokens + typo.tokens);
  assert.deepEqual(st.forbidden, []);
  assert.match(formatReadStats(st), /^문서 2개 · 읽기 3회 · 추정 \d+토큰$/);
});

test("금지된 읽기: evidence-ledger.md 통째 읽기는 FAIL, 일부만 읽으면 통과", () => {
  assert.equal(FORBIDDEN_READS[0].doc, "references/evidence-ledger.md");
  const whole = readStats(normalizeCodex(codexLine(`${PWSH}'Get-Content -Raw .agents/skills/designpaca/references/evidence-ledger.md'`)), { skillDir });
  assert.equal(whole.forbidden.length, 1);
  const v = judgeReads(byId.S1, whole);
  assert.equal(v.verdict, "FAIL");
  assert.match(v.note, /evidence-ledger\.md 를 통째로 읽었다/);

  const part = readStats(normalizeClaude(claudeLine("Read", { file_path: `${WIN}references\\evidence-ledger.md`, offset: 40, limit: 30 })), { skillDir });
  assert.deepEqual(part.forbidden, []);
  assert.equal(judgeReads(byId.S1, part).verdict, "PASS");
});

test("금지된 읽기: 세 번에 나눠 끝까지 읽어도 통째로 본다", () => {
  const raw = [0, 1, 2].map((i) => claudeLine("Read", { file_path: `${WIN}references\\evidence-ledger.md`, offset: 1 + i * 150, limit: 150 })).join("\n");
  const st = readStats(normalizeClaude(raw), { skillDir });
  assert.equal(st.docs[0].wholeReads, 0);
  assert.equal(st.docs[0].coverage, 1);
  assert.equal(st.forbidden.length, 1);
});

test("스킬 사본이 없으면 횟수·통째 여부만 내고 토큰은 미상이다(금지 패턴은 그대로 잡는다)", () => {
  const raw = [
    claudeLine("Read", { file_path: `${WIN}references\\evidence-ledger.md` }),
    claudeLine("Read", { file_path: `${WIN}references\\color.md`, limit: 40 }),
  ].join("\n");
  const st = readStats(normalizeClaude(raw));
  assert.equal(st.tokensKnown, false);
  assert.equal(st.totalTokens, null);
  assert.equal(st.forbidden.length, 1);
  assert.match(formatReadStats(st), /토큰 미상/);
  const capped = readStats(normalizeClaude(claudeLine("Read", { file_path: `${WIN}references\\color.md` })));
  assert.equal(judgeReads({ maxSkillReadTokens: 100 }, capped).verdict, "REVIEW");
});

test("국소 시나리오의 읽기 상한(maxSkillReadTokens): 넘으면 FAIL, 스킬 문서를 읽지 않았으면 판정하지 않는다", () => {
  for (const id of ["S6", "S13", "S20"]) assert.equal(typeof byId[id].maxSkillReadTokens, "number", `${id} 에 읽기 상한이 없다`);
  const raw = ["SKILL.md", "references\\typography.md", "references\\color.md"].map((f) => claudeLine("Read", { file_path: `${WIN}${f}` })).join("\n");
  const st = readStats(normalizeClaude(raw), { skillDir });
  assert.deepEqual(st.wholeLargeDocs, ["references/typography.md"]); // 관찰값: 읽는 법 표가 필요한 크기의 문서를 통째로 읽었다
  const over = judgeReads({ maxSkillReadTokens: st.totalTokens - 1 }, st);
  assert.equal(over.verdict, "FAIL");
  assert.match(over.note, /상한/);
  assert.equal(judgeReads({ maxSkillReadTokens: st.totalTokens }, st).verdict, "PASS");
  assert.equal(judgeReads(byId.S6, readStats([])), null);
});

test("녹화 트랜스크립트(S6 통과)에서 SKILL.md 읽기 1회를 집계한다", () => {
  const raw = fs.readFileSync(path.join(HERE, "transcripts", "claude-s6-pass.ndjson"), "utf8");
  const st = staticJudgements(byId.S6, normalizeClaude(raw));
  assert.equal(st.reads.docCount, 1);
  assert.equal(st.reads.docs[0].doc, "SKILL.md");
  assert.equal(st.reads.docs[0].wholeReads, 1);
  assert.equal(st.readsVerdict.verdict, "REVIEW"); // 상한이 있는데 스킬 사본이 없어 토큰을 추정하지 못한다
});

test("rejudge: 실행 디렉터리(원본 스트림 + 스킬 사본)를 다시 읽어 경로·읽기 표를 낸다", () => {
  const runDir = path.join(tmp, "run", "claude-sdk-auto", "S6-1");
  assert.equal(findSkillCopy(runDir), skillDir);
  const abs = (f) => path.join(skillDir, f);
  fs.writeFileSync(
    path.join(runDir, "raw.ndjson"),
    [
      JSON.stringify({ type: "system", subtype: "init", session_id: "s", tools: ["Read", "Edit", "Skill"] }),
      claudeLine("Read", { file_path: abs("SKILL.md") }),
      claudeLine("Read", { file_path: abs(path.join("references", "color.md")), limit: 60 }),
      claudeLine("Edit", { file_path: "index.html", old_string: "a", new_string: "b" }),
      JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: "국소 — 새 화면 없음, 섹션 1개. hover 색을 한 단계 진하게 바꿨습니다." }] } }),
      JSON.stringify({ type: "eval_turn_end", filesChanged: ["index.html"] }),
    ].join("\n"),
  );
  const rows = rejudgeLabel(tmp, "run", SCENARIOS, null);
  assert.equal(rows.length, 1);
  const r = rows[0];
  assert.equal(r.verdict, "PASS");
  assert.equal(r.route.verdict, "PASS");
  assert.equal(r.reads.docCount, 2);
  assert.equal(r.readsVerdict.verdict, "PASS");
  const md = renderMarkdown(rows);
  assert.match(md, /\| run \| claude-sdk-auto \| S6-1 \| PASS \| 국소 \| 국소 \| PASS \| 2 \| \d+ \| PASS \|/);
  assert.match(md, /\| S6 \| claude-sdk-auto \| 1 \| 1 \| 0 \| 0 \| 2 \| \d+ \| \d+ \| 0 \|/);
});