d3ro-voice/scripts/verification/typing-race-proof.mjs

103 lines
6.7 KiB
JavaScript

// Deterministic source A/B. No GUI, input injection, checkout mutation, or personal data.
// Run from this checkout: node scripts/verification/typing-race-proof.mjs [--baseline <commit>]
import { execFileSync, spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const root = process.cwd()
const desktop = path.join(root, 'apps/desktop')
const requiredRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
if (root.toLowerCase() !== requiredRoot.toLowerCase()) throw new Error('Run from the authoritative checkout root')
const require = createRequire(path.join(desktop, 'package.json'))
const vitest = path.join(path.dirname(require.resolve('vitest/package.json')), 'vitest.mjs')
const sources = [
'apps/desktop/src/main/services/TextInsertService.ts',
'apps/desktop/src/main/services/TypoCorrectionService.ts',
'apps/desktop/src/main/services/InputTelemetryService.ts'
]
const tests = [
'tests/main/services/typing-race-integration.test.ts',
'tests/main/services/input-telemetry-stale-snapshot.test.ts'
]
const stamp = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-')
const output = path.join(root, 'test-results/typing-load', `race-proof-${stamp}`)
mkdirSync(output, { recursive: true })
const sha256 = (text) => createHash('sha256').update(text).digest('hex')
const normalized = (file) => path.resolve(file).replaceAll('\\', '/').toLowerCase()
const head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim()
const args = process.argv.slice(2)
if (args.length !== 0 && (args.length !== 2 || args[0] !== '--baseline')) {
throw new Error('Usage: typing-race-proof.mjs [--baseline <commit>]')
}
// The fix is now committed: HEAD is no longer a broken baseline.
const baselineRef = args[1] ?? 'f49acf01057cc100b9caeace6eada26cf7bf4b54'
const baselineCommit = execFileSync('git', ['rev-parse', '--verify', '--end-of-options', `${baselineRef}^{commit}`],
{ cwd: root, encoding: 'utf8' }).trim()
const overrides = {}
const evidence = sources.map((file) => {
const baseline = execFileSync('git', ['show', `${baselineCommit}:${file}`], { cwd: root, encoding: 'utf8', maxBuffer: 4_000_000 })
const current = readFileSync(path.join(root, file), 'utf8')
overrides[normalized(path.join(root, file))] = baseline
return { file, baselineSha256: sha256(baseline), currentSha256: sha256(current) }
})
const testEvidence = tests.map((file) => ({
file: `apps/desktop/${file}`, currentSha256: sha256(readFileSync(path.join(desktop, file), 'utf8'))
}))
writeFileSync(path.join(output, 'baseline-source-overrides.json'), JSON.stringify(overrides))
const configPath = path.join(output, 'vitest.proof.config.mjs')
writeFileSync(configPath, `
import { readFileSync } from 'node:fs';
const overrides = process.env.D3RO_RACE_BASELINE === '1'
? JSON.parse(readFileSync(${JSON.stringify(path.join(output, 'baseline-source-overrides.json'))}, 'utf8')) : {};
export default {
plugins: [{ name: 'read-only-baseline-source-overlay', enforce: 'pre', load(id) {
return overrides[id.split('?')[0].replaceAll('\\\\', '/').toLowerCase()] ?? null;
}}],
test: { environment: 'node', globals: true,
setupFiles: [${JSON.stringify(path.join(desktop, 'tests/setup.ts'))}], testTimeout: 10000 },
resolve: { alias: {
'@d3ro/core': ${JSON.stringify(path.join(root, 'packages/core/src'))},
'@d3ro/ui': ${JSON.stringify(path.join(root, 'packages/ui/src'))},
'@d3ro/i18n': ${JSON.stringify(path.join(root, 'packages/i18n/src/index.tsx'))}
}}
};
`)
const runs = []
for (const baseline of [true, false]) {
const label = baseline ? 'baseline-source' : 'current-fix'
const resultPath = path.join(output, `${label}.json`)
const result = spawnSync(process.execPath, [vitest, 'run', ...tests, '--config', configPath,
'--reporter=json', `--outputFile=${resultPath}`], {
cwd: desktop, env: { ...process.env, D3RO_RACE_BASELINE: baseline ? '1' : '0' },
encoding: 'utf8', timeout: 90_000, maxBuffer: 4_000_000
})
writeFileSync(path.join(output, `${label}.log`), `${result.stdout ?? ''}\n${result.stderr ?? ''}`)
if (result.error) throw result.error
if (!existsSync(resultPath)) throw new Error(`${label} did not produce a test report; inspect ${label}.log in ${output}`)
const report = JSON.parse(readFileSync(resultPath, 'utf8'))
runs.push({ label, exitCode: result.status, passed: report.numPassedTests, failed: report.numFailedTests,
tests: report.testResults.flatMap((suite) => suite.assertionResults.map((test) => ({
title: test.fullName, status: test.status, failureMessages: test.failureMessages
}))) })
}
const report = {
capturedAt: new Date().toISOString(), kind: 'deterministic-source-ab-not-gui',
head, baselineCommit, node: process.version, sources: evidence, tests: testEvidence, runs,
sourcesUnchangedDuringRun: [...evidence, ...testEvidence].every(({ file, currentSha256 }) => sha256(readFileSync(path.join(root, file), 'utf8')) === currentSha256),
baselineScope: 'Only the three listed services are loaded from baselineCommit; dependencies, test doubles and tests are current in both runs. This is not a full historical release test.',
scope: 'Actual TypoCorrectionService + TextInsertService with an in-memory text sink, and actual InputTelemetryService with delayed UIA responses. Three text-preservation cases isolate timestamp, revision and native-tick guards; one unchanged-input control must still correct. The remaining 13 cases check snapshot invalidation/metadata/coalescing contracts, not 13 independent text-loss reproductions. Native keys, OS IME, model loading and CPU stress are not exercised.'
}
writeFileSync(path.join(output, 'report.json'), JSON.stringify(report, null, 2) + '\n')
process.stdout.write(JSON.stringify({ output, runs: runs.map(({label, passed, failed}) => ({label, passed, failed})) }, null, 2) + '\n')
const baseline = runs[0]
const fixed = runs[1]
const integratedLosses = baseline.tests.filter((test) => test.title.includes('preserves the whole next Hangul word'))
const control = baseline.tests.filter((test) => test.title.includes('still replaces the intended typo'))
if (!report.sourcesUnchangedDuringRun || integratedLosses.length !== 3 ||
integratedLosses.some((test) => test.status !== 'failed' || !test.failureMessages.some((message) => message.includes('오늘 잘 됬어요 안녕됐어요 '))) ||
control.length !== 1 || control[0].status !== 'passed' || baseline.exitCode !== 1 ||
fixed.exitCode !== 0 || fixed.failed !== 0 || fixed.passed !== 17 ||
fixed.passed !== baseline.passed + baseline.failed) process.exitCode = 1