공개 런타임 롤백 경계 강화

This commit is contained in:
Yun Chan 2026-08-09 21:57:07 +09:00
parent ff3c79dfc2
commit aaebe4450e
7 changed files with 1271 additions and 30 deletions

View file

@ -28,6 +28,10 @@ param(
[int]$DbTimeoutSec = 90, [int]$DbTimeoutSec = 90,
[int]$DbPort = 55432, [int]$DbPort = 55432,
[int]$ApiPort = 8001, [int]$ApiPort = 8001,
[int]$EnginePort = 9099,
[int]$WhisperPort = 9882,
[int]$MeloTtsPort = 9883,
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
[string]$BootLog = "" [string]$BootLog = ""
) )
@ -36,6 +40,7 @@ $ErrorActionPreference = "Stop"
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
$expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
function Invoke-GitText { function Invoke-GitText {
param([string[]]$Arguments) param([string[]]$Arguments)
@ -48,7 +53,7 @@ function Invoke-GitText {
} }
function Assert-StableSourceProvenance { function Assert-StableSourceProvenance {
foreach ($requiredScript in @($expectedBootScript, $startScript)) { foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) {
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
throw "Pinned public runtime script not found at $requiredScript" throw "Pinned public runtime script not found at $requiredScript"
} }
@ -97,7 +102,8 @@ function Assert-StableSourceProvenance {
} }
foreach ($relativePath in @( foreach ($relativePath in @(
"scripts/boot-public-runtime.ps1", "scripts/boot-public-runtime.ps1",
"scripts/start-public-runtime.ps1" "scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py"
)) { )) {
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
} }
@ -159,7 +165,7 @@ function Test-ApiControlPlaneHealthy {
$body = $reader.ReadToEnd() $body = $reader.ReadToEnd()
$reader.Close(); $resp.Close() $reader.Close(); $resp.Close()
$h = $body | ConvertFrom-Json $h = $body | ConvertFrom-Json
if ($h.environment -eq "prod" -and $h.db -eq $true) { return $true } if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true }
Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body) Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body)
return $false return $false
} catch { } catch {
@ -172,13 +178,72 @@ function Test-ApiControlPlaneHealthy {
function Test-EngineHealthy { function Test-EngineHealthy {
try { try {
$response = Invoke-RestMethod -Uri "http://127.0.0.1:9099/health" -TimeoutSec 10 $response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10
return $response.ok -eq $true return $response.ok -eq $true
} catch { } catch {
return $false return $false
} }
} }
function Test-VoiceApiReady {
param([object]$Health)
return (
$null -ne $Health -and
$Health.status -eq "ok" -and
$Health.available -eq $true -and
$Health.stt_available -eq $true -and
$Health.tts_available -eq $true -and
$Health.stt_provider -eq "local_whisper" -and
$Health.stt_model -eq "small" -and
$Health.tts_provider -eq "melotts" -and
$Health.tts_model -eq "melotts-korean" -and
$Health.limits.uvicorn_ws_max_queue -eq 4
)
}
function Test-VoiceApiHealthy {
for ($i = 1; $i -le 3; $i++) {
try {
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health")
$req.Timeout = 5000
$req.ReadWriteTimeout = 5000
$req.Proxy = $null
$resp = $req.GetResponse()
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
$body = $reader.ReadToEnd()
$reader.Close(); $resp.Close()
$health = $body | ConvertFrom-Json
if (Test-VoiceApiReady -Health $health) { return $true }
Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body)
return $false
} catch {
Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
Start-Sleep -Seconds 2
}
}
return $false
}
function Test-VoiceSidecarStack {
$probeArgs = @(
"-X", "utf8", "-B", $voiceSidecarProbe,
"--component", "all",
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
"--stt-provider", "local_whisper",
"--stt-model", "small",
"--stt-language", "ko",
"--stt-device", "cpu",
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
"--tts-provider", "melotts",
"--tts-model", "melotts-korean",
"--tts-language", "KR",
"--timeout-seconds", "5"
)
& $Python @probeArgs 1>$null 2>$null
return $LASTEXITCODE -eq 0
}
Write-BootLog "================ boot start ================" Write-BootLog "================ boot start ================"
# 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기) # 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기)
@ -217,12 +282,16 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) {
Write-BootLog "postgres 127.0.0.1:$DbPort up" Write-BootLog "postgres 127.0.0.1:$DbPort up"
# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지) # 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지)
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) { if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
Write-BootLog "control plane and engine already healthy; skipping runtime restart" Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart"
} else { } else {
Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart" Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart"
$out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript ` $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
-Workspace $resolvedSourceRoot ` -Workspace $resolvedSourceRoot `
-Python $Python `
-EnginePort $EnginePort `
-WhisperPort $WhisperPort `
-MeloTtsPort $MeloTtsPort `
-SkipWebRestart 2>&1 -SkipWebRestart 2>&1
$out | ForEach-Object { Write-BootLog (" pub> " + $_) } $out | ForEach-Object { Write-BootLog (" pub> " + $_) }
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
@ -233,12 +302,13 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
# 4) 최종 확인 # 4) 최종 확인
if (Test-ApiControlPlaneHealthy) { if (Test-ApiControlPlaneHealthy) {
if (Test-EngineHealthy) { if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
Write-BootLog "boot OK: control plane and engine healthy" Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy"
} else {
Write-BootLog "boot OK: admin/auth control plane healthy; engine remains degraded"
}
exit 0 exit 0
} else {
Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded"
exit 2
}
} else { } else {
Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log" Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log"
exit 2 exit 2

View file

@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
$installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime-task.ps1" $installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime-task.ps1"
$watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
function Invoke-GitText { function Invoke-GitText {
param([string[]]$Arguments) param([string[]]$Arguments)
@ -30,7 +31,7 @@ function Invoke-GitText {
return (@($value) -join [Environment]::NewLine).Trim() return (@($value) -join [Environment]::NewLine).Trim()
} }
foreach ($requiredScript in @($installerScript, $watchScript, $startScript)) { foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe)) {
if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) { if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
throw "Public runtime script not found at $requiredScript" throw "Public runtime script not found at $requiredScript"
} }
@ -71,7 +72,8 @@ if ($dirty) {
foreach ($relativePath in @( foreach ($relativePath in @(
"scripts/install-public-runtime-task.ps1", "scripts/install-public-runtime-task.ps1",
"scripts/watch-public-runtime.ps1", "scripts/watch-public-runtime.ps1",
"scripts/start-public-runtime.ps1" "scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py"
)) { )) {
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
} }

View file

@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
$RegisterScript = Join-Path $resolvedSourceRoot "scripts\register-boot-task.ps1" $RegisterScript = Join-Path $resolvedSourceRoot "scripts\register-boot-task.ps1"
$BootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $BootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
$StartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $StartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
$VoiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
$UserName = "$env:USERDOMAIN\$env:USERNAME" $UserName = "$env:USERDOMAIN\$env:USERNAME"
function Invoke-GitText { function Invoke-GitText {
@ -31,7 +32,7 @@ function Invoke-GitText {
return (@($value) -join [Environment]::NewLine).Trim() return (@($value) -join [Environment]::NewLine).Trim()
} }
foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript)) { foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript, $VoiceSidecarProbe)) {
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
throw "Public runtime script not found: $requiredScript" throw "Public runtime script not found: $requiredScript"
} }
@ -72,7 +73,8 @@ if ($dirty) {
foreach ($relativePath in @( foreach ($relativePath in @(
"scripts/register-boot-task.ps1", "scripts/register-boot-task.ps1",
"scripts/boot-public-runtime.ps1", "scripts/boot-public-runtime.ps1",
"scripts/start-public-runtime.ps1" "scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py"
)) { )) {
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
} }

View file

@ -15,6 +15,7 @@
[switch]$RouteCloudflareDns, [switch]$RouteCloudflareDns,
[string]$CloudflareTunnelName = "vignette", [string]$CloudflareTunnelName = "vignette",
[switch]$SkipCloudflaredRestart, [switch]$SkipCloudflaredRestart,
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
[switch]$RequireFreshPublicProvenance, [switch]$RequireFreshPublicProvenance,
[string]$ExpectedSourceCommit = "", [string]$ExpectedSourceCommit = "",
[string]$ExpectedSourceTree = "", [string]$ExpectedSourceTree = "",
@ -51,6 +52,15 @@ $WhisperLanguage = "ko"
$WhisperDevice = "cpu" $WhisperDevice = "cpu"
$MeloTtsModel = "melotts-korean" $MeloTtsModel = "melotts-korean"
$MeloTtsLanguage = "KR" $MeloTtsLanguage = "KR"
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
$RequiredPublicApiPaths = @(
"/health",
"/voice/health",
"/voice/speech",
"/admin/voice-runtime"
)
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net") $PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
$PublicWebHostnames = @("vnet.18ka.net") $PublicWebHostnames = @("vnet.18ka.net")
@ -97,6 +107,18 @@ function Test-PortListener {
return $null -ne $listener return $null -ne $listener
} }
function Get-ListenerProcessIds {
param([int]$Port)
return @(
Get-NetTCPConnection `
-State Listen `
-LocalPort $Port `
-ErrorAction SilentlyContinue |
Select-Object -ExpandProperty OwningProcess -Unique
)
}
function Test-VoiceSidecarReady { function Test-VoiceSidecarReady {
param( param(
[ValidateSet("stt", "tts")] [ValidateSet("stt", "tts")]
@ -156,6 +178,74 @@ function Test-VoiceApiReady {
) )
} }
function Test-PriorVoiceApiReady {
param([object]$Health)
return (
$null -ne $Health -and
$Health.status -eq "ok" -and
$Health.available -eq $true -and
$Health.stt_available -eq $true -and
$Health.tts_available -eq $true -and
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
)
}
function ConvertTo-SafeVoiceHealthContract {
param([object]$Health)
return [ordered]@{
status = [string]$Health.status
available = [bool]$Health.available
stt_available = [bool]$Health.stt_available
tts_available = [bool]$Health.tts_available
stt_provider = [string]$Health.stt_provider
stt_model = [string]$Health.stt_model
tts_provider = [string]$Health.tts_provider
tts_model = [string]$Health.tts_model
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
}
}
function Test-VoiceHealthContract {
param(
[object]$Health,
[System.Collections.IDictionary]$Expected
)
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
return $false
}
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
foreach ($name in $Expected.Keys) {
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
return $false
}
}
return $true
}
function Test-RequiredOpenApiPaths {
param(
[object]$Document,
[string[]]$RequiredPaths
)
if ($null -eq $Document -or $null -eq $Document.paths) {
return $false
}
$actualPaths = @($Document.paths.PSObject.Properties.Name)
foreach ($requiredPath in $RequiredPaths) {
if ($actualPaths -notcontains $requiredPath) {
return $false
}
}
return $true
}
function Test-EngineReady { function Test-EngineReady {
param( param(
[int]$Port, [int]$Port,
@ -247,16 +337,15 @@ function Stop-ProcessesBounded {
throw "Timed out stopping $Role process IDs: $($remaining -join ',')" throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
} }
function Stop-UvicornByPort { function Get-UvicornProcessesByPort {
param( param(
[string]$AppImport, [string]$AppImport,
[int]$Port, [int]$Port
[int]$TimeoutSec = 15
) )
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해 # Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다. # 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
$processes = @( return @(
Get-CimInstance Win32_Process | Get-CimInstance Win32_Process |
Where-Object { Where-Object {
$_.Name -like "python*" -and $_.Name -like "python*" -and
@ -265,6 +354,16 @@ function Stop-UvicornByPort {
$_.CommandLine -like "*--port $Port*" $_.CommandLine -like "*--port $Port*"
} }
) )
}
function Stop-UvicornByPort {
param(
[string]$AppImport,
[int]$Port,
[int]$TimeoutSec = 15
)
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
return @( return @(
Stop-ProcessesBounded ` Stop-ProcessesBounded `
-Processes $processes ` -Processes $processes `
@ -298,7 +397,7 @@ function Wait-ProcessIdentity {
param( param(
[int]$ProcessId, [int]$ProcessId,
[string]$Role, [string]$Role,
[string]$ExpectedCwd, [string]$ExpectedCwd = "",
[int]$TimeoutSec = 15 [int]$TimeoutSec = 15
) )
@ -314,20 +413,26 @@ function Wait-ProcessIdentity {
) { ) {
$identityProbeArgs = @( $identityProbeArgs = @(
"-X", "utf8", "-c", "-X", "utf8", "-c",
"import hashlib,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(p.cmdline()).encode('utf-8', errors='strict')).hexdigest())", "import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
"$ProcessId" "$ProcessId"
) )
$identityProbe = @(& $Python @identityProbeArgs) $identityProbe = @(& $Python @identityProbeArgs)
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 3) { if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
throw "Could not prove $Role psutil identity for PID $ProcessId" throw "Could not prove $Role psutil identity for PID $ProcessId"
} }
$actualCwd = $identityProbe[0].Trim() $actualCwd = $identityProbe[0].Trim()
$startedAtUtc = $identityProbe[1].Trim() $startedAtUtc = $identityProbe[1].Trim()
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant() $commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
$environmentObject = $identityProbe[4] | ConvertFrom-Json
$processEnvironment = [ordered]@{}
foreach ($property in $environmentObject.PSObject.Properties) {
$processEnvironment[$property.Name] = [string]$property.Value
}
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") { if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
throw "$Role psutil identity is incomplete for PID $ProcessId" throw "$Role psutil identity is incomplete for PID $ProcessId"
} }
if (-not [string]::Equals( if ($ExpectedCwd -and -not [string]::Equals(
[System.IO.Path]::GetFullPath($actualCwd), [System.IO.Path]::GetFullPath($actualCwd),
[System.IO.Path]::GetFullPath($ExpectedCwd), [System.IO.Path]::GetFullPath($ExpectedCwd),
[System.StringComparison]::OrdinalIgnoreCase [System.StringComparison]::OrdinalIgnoreCase
@ -343,6 +448,8 @@ function Wait-ProcessIdentity {
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant() executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
command_line = $process.CommandLine command_line = $process.CommandLine
command_line_sha256 = $commandLineSha256 command_line_sha256 = $commandLineSha256
argument_list = $argumentList
environment = $processEnvironment
cwd = $actualCwd cwd = $actualCwd
} }
} }
@ -367,6 +474,274 @@ function ConvertTo-SafeProcessIdentity {
} }
} }
function Save-ManagedEnvironment {
param([string[]]$Names)
$snapshot = [ordered]@{}
foreach ($name in $Names) {
$value = [System.Environment]::GetEnvironmentVariable(
$name,
[System.EnvironmentVariableTarget]::Process
)
$snapshot[$name] = [ordered]@{
present = $null -ne $value
value = $value
}
}
return $snapshot
}
function Restore-ManagedEnvironment {
param([System.Collections.IDictionary]$Snapshot)
foreach ($name in $Snapshot.Keys) {
$entry = $Snapshot[$name]
if ($entry.present) {
[System.Environment]::SetEnvironmentVariable(
$name,
[string]$entry.value,
[System.EnvironmentVariableTarget]::Process
)
} else {
[System.Environment]::SetEnvironmentVariable(
$name,
$null,
[System.EnvironmentVariableTarget]::Process
)
}
}
}
function Save-CompleteProcessEnvironment {
$snapshot = [ordered]@{}
$environment = [System.Environment]::GetEnvironmentVariables(
[System.EnvironmentVariableTarget]::Process
)
foreach ($name in $environment.Keys) {
$snapshot[[string]$name] = [string]$environment[$name]
}
return $snapshot
}
function Set-CompleteProcessEnvironment {
param([System.Collections.IDictionary]$Environment)
$current = [System.Environment]::GetEnvironmentVariables(
[System.EnvironmentVariableTarget]::Process
)
foreach ($name in @($current.Keys)) {
[System.Environment]::SetEnvironmentVariable(
[string]$name,
$null,
[System.EnvironmentVariableTarget]::Process
)
}
foreach ($name in $Environment.Keys) {
[System.Environment]::SetEnvironmentVariable(
[string]$name,
[string]$Environment[$name],
[System.EnvironmentVariableTarget]::Process
)
}
}
function ConvertTo-WindowsCommandLineArgument {
param([AllowEmptyString()][string]$Argument)
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
return $Argument
}
$builder = New-Object System.Text.StringBuilder
$null = $builder.Append('"')
$backslashes = 0
foreach ($character in $Argument.ToCharArray()) {
if ($character -eq '\') {
$backslashes++
continue
}
if ($character -eq '"') {
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
$null = $builder.Append('"')
$backslashes = 0
continue
}
if ($backslashes -gt 0) {
$null = $builder.Append(('\' * $backslashes))
$backslashes = 0
}
$null = $builder.Append($character)
}
if ($backslashes -gt 0) {
$null = $builder.Append(('\' * ($backslashes * 2)))
}
$null = $builder.Append('"')
return $builder.ToString()
}
function Join-WindowsArgumentList {
param([object[]]$ArgumentList)
return (@(
foreach ($argument in $ArgumentList) {
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
}
) -join ' ')
}
function Start-PinnedPriorProcess {
param(
[System.Collections.IDictionary]$Identity,
[string]$Role,
[string]$StdoutLog,
[string]$StderrLog
)
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
throw "Prior $Role executable is unavailable"
}
$actualExecutableSha256 = (
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
).Hash.ToLowerInvariant()
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
throw "Prior $Role executable SHA256 drift"
}
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
throw "Prior $Role working directory is unavailable"
}
if (@($Identity.argument_list).Count -eq 0) {
throw "Prior $Role argument list is unavailable"
}
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
throw "Prior $Role environment is unavailable"
}
$callerEnvironment = Save-CompleteProcessEnvironment
try {
Set-CompleteProcessEnvironment -Environment $Identity.environment
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
return Start-Process -WindowStyle Hidden `
-FilePath $Identity.executable_path `
-ArgumentList $argumentString `
-WorkingDirectory $Identity.cwd `
-RedirectStandardOutput $StdoutLog `
-RedirectStandardError $StderrLog `
-PassThru
} finally {
Set-CompleteProcessEnvironment -Environment $callerEnvironment
}
}
function Restore-PriorPublicRuntime {
param(
[System.Collections.IDictionary]$PriorApi,
[System.Collections.IDictionary]$PriorCloudflared,
[System.Collections.IDictionary]$PriorLocalVoiceContract,
[System.Collections.IDictionary]$PriorPublicVoiceContract,
[System.Collections.IDictionary]$EnvironmentSnapshot,
[string]$ConfigPath,
[int]$ApiPortValue,
[string]$HealthUrl,
[string]$VoiceHealthUrl,
[int]$TimeoutSec
)
$null = @(
Stop-UvicornByPort `
-AppImport "app.main:app" `
-Port $ApiPortValue `
-TimeoutSec $TimeoutSec
)
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
$priorApiProcess = Start-PinnedPriorProcess `
-Identity $PriorApi `
-Role "api" `
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
$priorApiIdentity = Wait-ProcessIdentity `
-ProcessId $priorApiProcess.Id `
-Role "restored prior api" `
-ExpectedCwd $PriorApi.cwd `
-TimeoutSec $TimeoutSec
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
throw "Restored prior API identity drift: $field"
}
}
$null = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPortValue/health" `
-IsHealthy {
param($health)
$health.environment -eq "prod" -and
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 60
$null = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
-IsHealthy {
param($health)
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
} `
-TimeoutSec 30
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
throw "Restored prior runtime does not have the exact local voice sidecars"
}
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
$currentCloudflared = @(
Get-CloudflaredProcessesForConfig `
-ConfigPath $resolvedConfigPath `
-ExactPath
)
$null = @(
Stop-ProcessesBounded `
-Processes $currentCloudflared `
-TimeoutSec $TimeoutSec `
-Role "failed fresh cloudflared"
)
$priorCloudflaredProcess = Start-PinnedPriorProcess `
-Identity $PriorCloudflared `
-Role "cloudflared" `
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
$priorCloudflaredIdentity = Wait-ProcessIdentity `
-ProcessId $priorCloudflaredProcess.Id `
-Role "restored prior cloudflared" `
-ExpectedCwd $PriorCloudflared.cwd `
-TimeoutSec $TimeoutSec
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
throw "Restored prior cloudflared identity drift: $field"
}
}
$null = Wait-JsonHealth `
-Uri $HealthUrl `
-IsHealthy {
param($health)
$health.environment -eq "prod" -and
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 60
$null = Wait-JsonHealth `
-Uri $VoiceHealthUrl `
-IsHealthy {
param($health)
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
} `
-TimeoutSec 30
return [ordered]@{
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
local_health = $true
local_voice_health = $true
public_health = $true
public_voice_health = $true
}
}
function Stop-NodeByPortHint { function Stop-NodeByPortHint {
param([int]$Port) param([int]$Port)
@ -515,6 +890,39 @@ function Write-Utf8TextAtomically {
} }
} }
function Write-FailedFreshPromotionEvidence {
param(
[string]$OutputPath,
[string]$FailureStage,
[bool]$RollbackSucceeded,
[object]$RollbackResult,
[string]$SourceCommit,
[string]$SourceTree
)
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
$failedPath = "$OutputPath.failed.log"
$payload = [ordered]@{
schema_version = "vignette.public-runtime-launch-failure.v1"
status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" }
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
failure_stage = $FailureStage
source = [ordered]@{
git_commit = $SourceCommit.ToLowerInvariant()
git_tree = $SourceTree.ToLowerInvariant()
}
rollback = [ordered]@{
attempted = $true
succeeded = $RollbackSucceeded
result = $RollbackResult
}
}
$json = ConvertTo-Json -InputObject $payload -Depth 8
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
return $failedPath
}
function Assert-FreshPublicProvenanceContract { function Assert-FreshPublicProvenanceContract {
param( param(
[string]$SourceRoot, [string]$SourceRoot,
@ -534,6 +942,22 @@ function Assert-FreshPublicProvenanceContract {
if ($SkipCloudflaredRestart) { if ($SkipCloudflaredRestart) {
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart" throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
} }
if (-not $SkipEngineRestart) {
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
}
if (-not $SkipWebRestart) {
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
}
if ($RouteCloudflareDns) {
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
}
if (-not [string]::Equals(
$PublicHealthUrl,
$CanonicalPublicHealthUrl,
[System.StringComparison]::OrdinalIgnoreCase
)) {
throw "Fresh public promotion requires the canonical HTTPS public health URL"
}
foreach ($sourcePin in @($SourceCommit, $SourceTree)) { foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") { if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
throw "Fresh public provenance requires exact source commit and tree pins" throw "Fresh public provenance requires exact source commit and tree pins"
@ -657,6 +1081,85 @@ function Set-CloudflaredIngress {
} }
} }
$freshMutationStarted = $false
$freshPromotionCommitted = $false
$freshFailureStage = "preflight"
$freshPriorApiIdentity = $null
$freshPriorCloudflaredIdentity = $null
$freshPriorLocalVoiceContract = $null
$freshPriorPublicVoiceContract = $null
$freshEnvironmentSnapshot = $null
$resolvedRuntimeProvenancePath = $null
$freshManagedEnvironmentNames = @(
"ENVIRONMENT",
"ENGINE_URL",
"ENGINE_MODE",
"VIGNETTE_LIVE_CLIENT_PROVIDER",
"AUTH_DEV_LOGIN_ENABLED",
"AUTO_SEED_PERSONAS",
"ALLOW_SEED_PERSONA_FALLBACK",
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
"VIGNETTE_VOICE_STT_PROVIDER",
"VIGNETTE_LOCAL_WHISPER_STT_URL",
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
"VIGNETTE_VOICE_TTS_PROVIDER",
"VIGNETTE_MELOTTS_TTS_URL",
"FRONTEND_BASE_URL",
"CORS_ORIGINS",
"FRONTEND_ORIGIN_MAP"
)
trap {
$caught = $_
if (
$RequireFreshPublicProvenance -and
$freshMutationStarted -and
-not $freshPromotionCommitted
) {
$rollbackResult = $null
$rollbackSucceeded = $false
$rollbackFailureType = "none"
try {
$rollbackResult = Restore-PriorPublicRuntime `
-PriorApi $freshPriorApiIdentity `
-PriorCloudflared $freshPriorCloudflaredIdentity `
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
-EnvironmentSnapshot $freshEnvironmentSnapshot `
-ConfigPath $CloudflaredConfig `
-ApiPortValue $ApiPort `
-HealthUrl $PublicHealthUrl `
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
-TimeoutSec $ProcessStopTimeoutSeconds
$rollbackSucceeded = $true
} catch {
$rollbackFailureType = $_.Exception.GetType().Name
}
$failedEvidencePath = ""
if ($resolvedRuntimeProvenancePath) {
try {
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
-OutputPath $resolvedRuntimeProvenancePath `
-FailureStage $freshFailureStage `
-RollbackSucceeded $rollbackSucceeded `
-RollbackResult $rollbackResult `
-SourceCommit $ExpectedSourceCommit `
-SourceTree $ExpectedSourceTree
} catch {
$failedEvidencePath = "unavailable"
}
}
if ($rollbackSucceeded) {
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
}
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
}
throw $caught
}
if (!(Test-Path $Python)) { if (!(Test-Path $Python)) {
throw "Python 3.11 not found at $Python" throw "Python 3.11 not found at $Python"
} }
@ -705,6 +1208,64 @@ if ($RequireFreshPublicProvenance) {
-ApiPortValue $ApiPort ` -ApiPortValue $ApiPort `
-WebPortValue $WebPort ` -WebPortValue $WebPort `
-RequireUnchanged -RequireUnchanged
$priorApiProcesses = @(
Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort
)
if ($priorApiProcesses.Count -ne 1) {
throw "Fresh public promotion requires exactly one prior API process for transactional rollback"
}
$priorCloudflaredProcesses = @(
Get-CloudflaredProcessesForConfig `
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
-ExactPath
)
if ($priorCloudflaredProcesses.Count -ne 1) {
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
}
$freshPriorApiIdentity = Wait-ProcessIdentity `
-ProcessId $priorApiProcesses[0].ProcessId `
-Role "prior api" `
-TimeoutSec $ProcessStopTimeoutSeconds
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
-Role "prior cloudflared" `
-TimeoutSec $ProcessStopTimeoutSeconds
$null = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPort/health" `
-IsHealthy {
param($health)
$health.environment -eq "prod" -and
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 30
$priorLocalVoiceHealth = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
-TimeoutSec 30
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
-Health $priorLocalVoiceHealth
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
}
$null = Wait-JsonHealth `
-Uri $CanonicalPublicHealthUrl `
-IsHealthy {
param($health)
$health.environment -eq "prod" -and
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 30
$priorPublicVoiceHealth = Wait-JsonHealth `
-Uri $CanonicalPublicVoiceHealthUrl `
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
-TimeoutSec 30
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
-Health $priorPublicVoiceHealth
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
-Names $freshManagedEnvironmentNames
} }
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. # 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
@ -732,7 +1293,9 @@ if ($SkipEngineRestart) {
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss" $rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) { foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
if (Test-Path $logFile) { if (Test-Path $logFile) {
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak" -Force -ErrorAction SilentlyContinue # stable detached source의 provenance gate는 untracked 파일도 차단한다.
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
} }
} }
@ -781,6 +1344,9 @@ $env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를 # 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다. # 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
if (-not (Test-VoiceSidecarReady -Component "stt")) { if (-not (Test-VoiceSidecarReady -Component "stt")) {
if ($RequireFreshPublicProvenance) {
throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying"
}
if (Test-PortListener -Port $WhisperPort) { if (Test-PortListener -Port $WhisperPort) {
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol" throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
} }
@ -795,6 +1361,9 @@ if (-not (Test-VoiceSidecarReady -Component "stt")) {
} }
if (-not (Test-VoiceSidecarReady -Component "tts")) { if (-not (Test-VoiceSidecarReady -Component "tts")) {
if ($RequireFreshPublicProvenance) {
throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
}
if (Test-PortListener -Port $MeloTtsPort) { if (Test-PortListener -Port $MeloTtsPort) {
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract" throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
} }
@ -828,6 +1397,10 @@ $apiLaunchIdentity = $null
if ($apiControlPlaneReady -and -not $ForceApiRestart) { if ($apiControlPlaneReady -and -not $ForceApiRestart) {
Write-Output "Production API and exact local voice stack already healthy; skipping API restart" Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
} else { } else {
if ($RequireFreshPublicProvenance) {
$freshFailureStage = "api_cutover"
$freshMutationStarted = $true
}
$apiStoppedProcessIds = @( $apiStoppedProcessIds = @(
Stop-UvicornByPort ` Stop-UvicornByPort `
-AppImport "app.main:app" ` -AppImport "app.main:app" `
@ -890,6 +1463,9 @@ if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
} }
if (!$SkipWebRestart) { if (!$SkipWebRestart) {
if ($RequireFreshPublicProvenance) {
$freshFailureStage = "web_preview"
}
Stop-NodeByPortHint -Port $WebPort Stop-NodeByPortHint -Port $WebPort
$build = Start-Process -FilePath "cmd.exe" ` $build = Start-Process -FilePath "cmd.exe" `
@ -916,6 +1492,9 @@ $cloudflaredProcess = $null
$cloudflaredLaunchIdentity = $null $cloudflaredLaunchIdentity = $null
$cloudflaredStoppedProcessIds = @() $cloudflaredStoppedProcessIds = @()
if (!$SkipCloudflaredRestart) { if (!$SkipCloudflaredRestart) {
if ($RequireFreshPublicProvenance) {
$freshFailureStage = "cloudflared_cutover"
}
if (!(Test-Path $Cloudflared)) { if (!(Test-Path $Cloudflared)) {
throw "cloudflared not found at $Cloudflared" throw "cloudflared not found at $Cloudflared"
} }
@ -986,6 +1565,7 @@ if (!$SkipCloudflaredRestart) {
} }
if ($RequireFreshPublicProvenance) { if ($RequireFreshPublicProvenance) {
$freshFailureStage = "identity_revalidation"
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) { if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
throw "Fresh public promotion did not produce both API and cloudflared identities" throw "Fresh public promotion did not produce both API and cloudflared identities"
} }
@ -1015,6 +1595,32 @@ if ($RequireFreshPublicProvenance) {
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) { if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
throw "Pinned cloudflared config drifted before provenance receipt" throw "Pinned cloudflared config drifted before provenance receipt"
} }
$freshFailureStage = "public_health_validation"
$publicHealth = Wait-JsonHealth `
-Uri $CanonicalPublicHealthUrl `
-IsHealthy {
param($health)
$health.environment -eq "prod" -and
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 60
$publicVoiceHealth = Wait-JsonHealth `
-Uri $CanonicalPublicVoiceHealthUrl `
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
-TimeoutSec 30
$publicOpenApi = Wait-JsonHealth `
-Uri $CanonicalPublicOpenApiUrl `
-IsHealthy {
param($document)
Test-RequiredOpenApiPaths `
-Document $document `
-RequiredPaths $RequiredPublicApiPaths
} `
-TimeoutSec 30
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
throw "Exact local voice sidecars changed before provenance receipt"
}
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))") $psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim() $psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) { if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
@ -1038,6 +1644,16 @@ if ($RequireFreshPublicProvenance) {
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
sha256 = $finalConfigSha256 sha256 = $finalConfigSha256
} }
public_validation = [ordered]@{
health_url = $CanonicalPublicHealthUrl
health = $true
voice_health_url = $CanonicalPublicVoiceHealthUrl
voice_health = $true
openapi_url = $CanonicalPublicOpenApiUrl
required_openapi_paths = @($RequiredPublicApiPaths)
openapi = $true
local_voice_sidecars = $true
}
replacement = [ordered]@{ replacement = [ordered]@{
api_stopped_pids = @($apiStoppedProcessIds) api_stopped_pids = @($apiStoppedProcessIds)
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds) cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
@ -1068,6 +1684,7 @@ if ($RequireFreshPublicProvenance) {
} }
} }
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8 $provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
$freshFailureStage = "receipt_publish"
try { try {
Write-Utf8TextAtomically ` Write-Utf8TextAtomically `
-OutputPath $resolvedRuntimeProvenancePath ` -OutputPath $resolvedRuntimeProvenancePath `
@ -1077,6 +1694,7 @@ if ($RequireFreshPublicProvenance) {
# 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다. # 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다.
throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)" throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)"
} }
$freshPromotionCommitted = $true
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath" Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
} }

View file

@ -15,6 +15,7 @@ BOOT = SCRIPTS / "boot-public-runtime.ps1"
BOOT_REGISTER = SCRIPTS / "register-boot-task.ps1" BOOT_REGISTER = SCRIPTS / "register-boot-task.ps1"
HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs" HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs"
START = SCRIPTS / "start-public-runtime.ps1" START = SCRIPTS / "start-public-runtime.ps1"
VOICE_PROBE = SCRIPTS / "probe-public-voice-sidecars.py"
REPO_ROOT = SCRIPTS.parent REPO_ROOT = SCRIPTS.parent
RUNBOOK = REPO_ROOT / "docs" / "ops" / "public-runtime-watchdog.md" RUNBOOK = REPO_ROOT / "docs" / "ops" / "public-runtime-watchdog.md"
LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md" LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md"
@ -117,6 +118,97 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask") task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask")
self.assertLess(dirty_gate, task_registration) self.assertLess(dirty_gate, task_registration)
def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None:
for source in (WATCHDOG_SOURCE, BOOT_SOURCE):
for expected in (
'"--component", "all"',
'"--stt-provider", "local_whisper"',
'"--stt-model", "small"',
'"--stt-device", "cpu"',
'"--tts-provider", "melotts"',
'"--tts-model", "melotts-korean"',
'"scripts\\probe-public-voice-sidecars.py"',
"Test-VoiceSidecarStack",
):
with self.subTest(source=source[:32], expected=expected):
self.assertIn(expected, source)
self.assertIn("(Test-VoiceSidecarStack)", WATCHDOG_SOURCE)
self.assertIn("$voiceSidecarsAfter = Test-VoiceSidecarStack", WATCHDOG_SOURCE)
self.assertIn('-Name "voice-api"', WATCHDOG_SOURCE)
self.assertIn("$voiceApiAfter = Test-JsonHealth", WATCHDOG_SOURCE)
self.assertIn(
"(Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)",
BOOT_SOURCE,
)
self.assertGreaterEqual(BOOT_SOURCE.count("Test-VoiceApiHealthy"), 3)
self.assertIn("-WhisperPort $WhisperPort", BOOT_SOURCE)
self.assertIn("-MeloTtsPort $MeloTtsPort", BOOT_SOURCE)
self.assertIn("scripts/probe-public-voice-sidecars.py", INSTALLER_SOURCE)
self.assertIn("scripts/probe-public-voice-sidecars.py", BOOT_REGISTER_SOURCE)
def test_old_openai_api_is_not_healthy_when_exact_sidecars_are_ready(self) -> None:
powershell = shutil.which("powershell.exe")
if powershell is None:
self.skipTest("Windows PowerShell 5.1 is not available")
watchdog_contract = WATCHDOG_SOURCE[
WATCHDOG_SOURCE.index("function Test-VoiceApiReady") :
WATCHDOG_SOURCE.index("function Test-VoiceSidecarStack")
].strip()
boot_contract = BOOT_SOURCE[
BOOT_SOURCE.index("function Test-VoiceApiReady") :
BOOT_SOURCE.index("function Test-VoiceApiHealthy")
].strip()
self.assertEqual(watchdog_contract, boot_contract)
with tempfile.TemporaryDirectory() as temporary_directory:
harness = Path(temporary_directory) / "voice-api-contract.ps1"
harness.write_text(
watchdog_contract
+ r'''
$openAi = [pscustomobject]@{
status='ok';available=$true;stt_available=$true;tts_available=$true
stt_provider='openai';stt_model='gpt-4o-mini-transcribe'
tts_provider='openai';tts_model='gpt-4o-mini-tts'
limits=[pscustomobject]@{uvicorn_ws_max_queue=4}
}
if (Test-VoiceApiReady -Health $openAi) { throw 'old OpenAI API was accepted' }
$local = [pscustomobject]@{
status='ok';available=$true;stt_available=$true;tts_available=$true
stt_provider='local_whisper';stt_model='small'
tts_provider='melotts';tts_model='melotts-korean'
limits=[pscustomobject]@{uvicorn_ws_max_queue=4}
}
if (-not (Test-VoiceApiReady -Health $local)) { throw 'exact local API was rejected' }
$local.limits.uvicorn_ws_max_queue = 5
if (Test-VoiceApiReady -Health $local) { throw 'wrong websocket queue was accepted' }
''',
encoding="utf-8-sig",
)
completed = subprocess.run(
[
powershell,
"-NoLogo",
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-File",
str(harness),
],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=30,
check=False,
)
self.assertEqual(
completed.returncode,
0,
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
)
def test_hidden_trigger_never_executes_a_workspace_script_directly(self) -> None: def test_hidden_trigger_never_executes_a_workspace_script_directly(self) -> None:
for expected in ( for expected in (
"VignettePublicRuntimeWatchdog", "VignettePublicRuntimeWatchdog",
@ -178,11 +270,13 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
copied_boot = scripts / BOOT.name copied_boot = scripts / BOOT.name
copied_boot_register = scripts / BOOT_REGISTER.name copied_boot_register = scripts / BOOT_REGISTER.name
copied_start = scripts / START.name copied_start = scripts / START.name
copied_voice_probe = scripts / VOICE_PROBE.name
shutil.copy2(WATCHDOG, copied_watchdog) shutil.copy2(WATCHDOG, copied_watchdog)
shutil.copy2(INSTALLER, copied_installer) shutil.copy2(INSTALLER, copied_installer)
shutil.copy2(BOOT, copied_boot) shutil.copy2(BOOT, copied_boot)
shutil.copy2(BOOT_REGISTER, copied_boot_register) shutil.copy2(BOOT_REGISTER, copied_boot_register)
shutil.copy2(START, copied_start) shutil.copy2(START, copied_start)
shutil.copy2(VOICE_PROBE, copied_voice_probe)
self._git(root, "init") self._git(root, "init")
self._git(root, "config", "user.name", "Watchdog Contract Test") self._git(root, "config", "user.name", "Watchdog Contract Test")
@ -195,6 +289,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
"scripts/boot-public-runtime.ps1", "scripts/boot-public-runtime.ps1",
"scripts/register-boot-task.ps1", "scripts/register-boot-task.ps1",
"scripts/start-public-runtime.ps1", "scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py",
) )
self._git(root, "commit", "-m", "watchdog fixture") self._git(root, "commit", "-m", "watchdog fixture")
self._git(root, "checkout", "--detach") self._git(root, "checkout", "--detach")

View file

@ -3,6 +3,7 @@ from __future__ import annotations
import unittest import unittest
import shutil import shutil
import subprocess import subprocess
import sys
import tempfile import tempfile
from pathlib import Path from pathlib import Path
@ -83,6 +84,10 @@ class FreshPublicProvenanceContractTest(unittest.TestCase):
"[switch]$RequireFreshPublicProvenance", "[switch]$RequireFreshPublicProvenance",
'throw "-RequireFreshPublicProvenance requires -ForceApiRestart"', 'throw "-RequireFreshPublicProvenance requires -ForceApiRestart"',
'throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"', 'throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"',
'throw "-RequireFreshPublicProvenance requires -SkipEngineRestart;',
'throw "-RequireFreshPublicProvenance requires -SkipWebRestart;',
'throw "-RequireFreshPublicProvenance forbids DNS route mutation"',
"Fresh public promotion requires the canonical HTTPS public health URL",
"ExpectedSourceCommit", "ExpectedSourceCommit",
"ExpectedSourceTree", "ExpectedSourceTree",
"ExpectedPythonSha256", "ExpectedPythonSha256",
@ -171,6 +176,18 @@ if ($receipt.status -ne 'passed') {{ throw 'atomic receipt content mismatch' }}
Write-Utf8TextAtomically -OutputPath $resolved -Value '{{"status":"replaced"}}' Write-Utf8TextAtomically -OutputPath $resolved -Value '{{"status":"replaced"}}'
$replacement = Get-Content -LiteralPath $resolved -Raw -Encoding UTF8 | ConvertFrom-Json $replacement = Get-Content -LiteralPath $resolved -Raw -Encoding UTF8 | ConvertFrom-Json
if ($replacement.status -ne 'replaced') {{ throw 'atomic replacement mismatch' }} if ($replacement.status -ne 'replaced') {{ throw 'atomic replacement mismatch' }}
$failedPath = Write-FailedFreshPromotionEvidence `
-OutputPath $resolved `
-FailureStage 'receipt_publish' `
-RollbackSucceeded $true `
-RollbackResult @{{local_health=$true;public_health=$true}} `
-SourceCommit ('a' * 40) `
-SourceTree ('b' * 40)
$failed = Get-Content -LiteralPath $failedPath -Raw -Encoding UTF8 | ConvertFrom-Json
if ((Split-Path -Leaf $failedPath) -notlike '*.failed.log') {{ throw 'failure evidence suffix mismatch' }}
if ($failed.status -ne 'failed_rolled_back') {{ throw 'failure evidence status mismatch' }}
if ($failed.failure_stage -ne 'receipt_publish') {{ throw 'failure evidence stage mismatch' }}
if (-not $failed.rollback.succeeded) {{ throw 'failure evidence rollback mismatch' }}
if (@(Get-ChildItem -LiteralPath (Split-Path -Parent $resolved) -Filter '*.tmp').Count -ne 0) {{ if (@(Get-ChildItem -LiteralPath (Split-Path -Parent $resolved) -Filter '*.tmp').Count -ne 0) {{
throw 'temporary receipt files were not cleaned' throw 'temporary receipt files were not cleaned'
}} }}
@ -271,7 +288,7 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri
def test_receipt_projects_no_raw_command_or_config_contents(self) -> None: def test_receipt_projects_no_raw_command_or_config_contents(self) -> None:
projection = PUBLIC_RUNTIME[ projection = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") : PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") :
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint") PUBLIC_RUNTIME.index("function Save-ManagedEnvironment")
] ]
self.assertIn("command_line_sha256 =", projection) self.assertIn("command_line_sha256 =", projection)
self.assertNotIn("command_line =", projection) self.assertNotIn("command_line =", projection)
@ -297,9 +314,376 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri
PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity")
] ]
self.assertIn("datetime.fromtimestamp(p.create_time(), UTC)", identity) self.assertIn("datetime.fromtimestamp(p.create_time(), UTC)", identity)
self.assertIn("chr(0).join(p.cmdline())", identity) self.assertIn("argv=p.cmdline()", identity)
self.assertIn("chr(0).join(argv)", identity)
self.assertIn("argument_list = $argumentList", identity)
self.assertIn("environment = $processEnvironment", identity)
self.assertIn("command_line_sha256 = $commandLineSha256", identity) self.assertIn("command_line_sha256 = $commandLineSha256", identity)
def test_prior_process_argv_and_environment_round_trip_in_powershell_51(self) -> None:
powershell = shutil.which("powershell.exe")
if powershell is None:
self.skipTest("Windows PowerShell 5.1 is not available")
function_source = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Save-CompleteProcessEnvironment") :
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime")
]
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
functions_path = root / "rollback-process-functions.ps1"
harness_path = root / "rollback-process-harness.ps1"
functions_path.write_text(function_source, encoding="utf-8-sig")
quoted_functions = str(functions_path).replace("'", "''")
quoted_root = str(root).replace("'", "''")
quoted_python = sys.executable.replace("'", "''")
harness_path.write_text(
f"""$ErrorActionPreference = 'Stop'
. '{quoted_functions}'
$root = '{quoted_root}'
$python = '{quoted_python}'
if ($null -eq (Get-Command Get-FileHash -ErrorAction SilentlyContinue)) {{
function Get-FileHash {{
param([string]$LiteralPath, [string]$Algorithm)
$stream = [IO.File]::OpenRead($LiteralPath)
try {{
$hasher = [Security.Cryptography.SHA256]::Create()
try {{ $hash = $hasher.ComputeHash($stream) }} finally {{ $hasher.Dispose() }}
}} finally {{
$stream.Dispose()
}}
[pscustomobject]@{{Hash = ([BitConverter]::ToString($hash)).Replace('-', '')}}
}}
}}
$pythonSha256 = (Get-FileHash -LiteralPath $python -Algorithm SHA256).Hash.ToLowerInvariant()
$originalEnvironment = Save-CompleteProcessEnvironment
try {{
[Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'caller', 'Process')
[Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', 'caller-only', 'Process')
$callerEnvironment = Save-CompleteProcessEnvironment
[Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'prior', 'Process')
[Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', $null, 'Process')
$priorEnvironment = Save-CompleteProcessEnvironment
Set-CompleteProcessEnvironment -Environment $callerEnvironment
$output = Join-Path $root 'argv-result.json'
$code = "import json,os,sys; open(sys.argv[1], 'w', encoding='utf-8').write(json.dumps({{'argv':sys.argv[2:],'marker':os.environ.get('VIGNETTE_ROLLBACK_ARGV_TEST'),'leak':os.environ.get('VIGNETTE_CALLER_ONLY')}}, ensure_ascii=True))"
$expected = @('plain', 'space value', 'quote"value', 'trailing\', '', 'slashes\\before"quote')
$identity = [ordered]@{{
executable_path = $python
executable_sha256 = $pythonSha256
cwd = $root
argument_list = @('-X', 'utf8', '-c', $code, $output) + $expected
environment = $priorEnvironment
}}
$process = Start-PinnedPriorProcess `
-Identity $identity `
-Role 'argv-roundtrip' `
-StdoutLog (Join-Path $root 'child.out.log') `
-StderrLog (Join-Path $root 'child.err.log')
$process.WaitForExit()
if (-not (Test-Path -LiteralPath $output -PathType Leaf)) {{ throw 'child did not write argv result' }}
$actual = Get-Content -LiteralPath $output -Raw -Encoding UTF8 | ConvertFrom-Json
if (@($actual.argv).Count -ne $expected.Count) {{ throw 'argument count mismatch' }}
for ($i = 0; $i -lt $expected.Count; $i++) {{
if ($actual.argv[$i] -cne $expected[$i]) {{ throw "argument mismatch at $i" }}
}}
if ($actual.marker -cne 'prior') {{ throw 'prior environment was not inherited' }}
if ($null -ne $actual.leak) {{ throw 'caller-only environment leaked into prior process' }}
if ($env:VIGNETTE_ROLLBACK_ARGV_TEST -cne 'caller') {{ throw 'caller environment marker was not restored' }}
if ($env:VIGNETTE_CALLER_ONLY -cne 'caller-only') {{ throw 'caller-only environment was not restored' }}
}} finally {{
Set-CompleteProcessEnvironment -Environment $originalEnvironment
}}
""",
encoding="utf-8-sig",
)
completed = subprocess.run(
[
powershell,
"-NoLogo",
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-File",
str(harness_path),
],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=30,
check=False,
)
self.assertEqual(
completed.returncode,
0,
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
)
def test_rollback_restores_prior_openai_voice_contract_in_powershell_51(self) -> None:
powershell = shutil.which("powershell.exe")
if powershell is None:
self.skipTest("Windows PowerShell 5.1 is not available")
voice_contract_source = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Test-PriorVoiceApiReady") :
PUBLIC_RUNTIME.index("function Test-RequiredOpenApiPaths")
]
rollback_source = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") :
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint")
]
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
functions_path = root / "rollback-contract-functions.ps1"
harness_path = root / "rollback-contract-harness.ps1"
functions_path.write_text(
voice_contract_source + rollback_source,
encoding="utf-8-sig",
)
quoted_functions = str(functions_path).replace("'", "''")
quoted_root = str(root).replace("'", "''")
harness_path.write_text(
f"""$ErrorActionPreference = 'Stop'
. '{quoted_functions}'
$events = New-Object System.Collections.ArrayList
$root = '{quoted_root}'
function Stop-UvicornByPort {{ param($AppImport,$Port,$TimeoutSec); $null=$events.Add('stop-api'); 11 }}
function Restore-ManagedEnvironment {{ param($Snapshot); $null=$events.Add('restore-env') }}
function Start-PinnedPriorProcess {{
param($Identity,$Role,$StdoutLog,$StderrLog)
$null=$events.Add("start-$Role")
if ($Role -eq 'api') {{ [pscustomobject]@{{Id=101}} }} else {{ [pscustomobject]@{{Id=202}} }}
}}
function Wait-ProcessIdentity {{
param($ProcessId,$Role,$ExpectedCwd,$TimeoutSec)
$null=$events.Add("identity-$Role")
[ordered]@{{
pid=[int]$ProcessId; started_at_utc='2026-08-09T00:00:00Z';
executable_name='runtime.exe'; executable_sha256=('a' * 64);
command_line_sha256=('b' * 64); cwd=$root
}}
}}
function Wait-JsonHealth {{
param($Uri,$IsHealthy,$TimeoutSec)
$null=$events.Add("health-$Uri")
if ($Uri -like '*/voice/health') {{
$payload=[pscustomobject]@{{
status='ok';available=$true;stt_available=$true;tts_available=$true;
stt_provider='openai';stt_model='gpt-4o-mini-transcribe';
tts_provider='openai';tts_model='gpt-4o-mini-tts';
limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}}
}}
}} else {{
$payload=[pscustomobject]@{{environment='prod';db=$true;engine=$true}}
}}
if (-not (& $IsHealthy $payload)) {{ throw "health predicate failed: $Uri" }}
$payload
}}
function Test-VoiceSidecarReady {{ param($Component); $true }}
function Get-CloudflaredProcessesForConfig {{ param($ConfigPath,[switch]$ExactPath); @([pscustomobject]@{{ProcessId=303}}) }}
function Stop-ProcessesBounded {{ param($Processes,$TimeoutSec,$Role); $null=$events.Add('stop-cloudflared'); 303 }}
function ConvertTo-SafeProcessIdentity {{ param($Identity); $Identity }}
$configPath = Join-Path $root 'cloudflared.yml'
[IO.File]::WriteAllText($configPath, 'tunnel: test')
$priorVoice = [ordered]@{{
status='ok';available=$true;stt_available=$true;tts_available=$true;
stt_provider='openai';stt_model='gpt-4o-mini-transcribe';
tts_provider='openai';tts_model='gpt-4o-mini-tts';uvicorn_ws_max_queue=4
}}
$priorApi=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}}
$priorCloud=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}}
$result = Restore-PriorPublicRuntime `
-PriorApi $priorApi `
-PriorCloudflared $priorCloud `
-PriorLocalVoiceContract $priorVoice `
-PriorPublicVoiceContract $priorVoice `
-EnvironmentSnapshot @{{}} `
-ConfigPath $configPath `
-ApiPortValue 8001 `
-HealthUrl 'https://api-vignette.chanpaca.net/health' `
-VoiceHealthUrl 'https://api-vignette.chanpaca.net/voice/health' `
-TimeoutSec 5
foreach ($field in @('local_health','local_voice_health','public_health','public_voice_health')) {{
if (-not $result[$field]) {{ throw "rollback result missing $field" }}
}}
if ($events.IndexOf('stop-api') -ge $events.IndexOf('start-api')) {{ throw 'API restore ordering mismatch' }}
if ($events.IndexOf('stop-cloudflared') -ge $events.IndexOf('start-cloudflared')) {{ throw 'cloudflared restore ordering mismatch' }}
if ($events.IndexOf('start-api') -ge $events.IndexOf('health-http://127.0.0.1:8001/health')) {{ throw 'local health ordering mismatch' }}
if ($events.IndexOf('start-cloudflared') -ge $events.IndexOf('health-https://api-vignette.chanpaca.net/health')) {{ throw 'public health ordering mismatch' }}
$changed=[pscustomobject]@{{
status='ok';available=$true;stt_available=$true;tts_available=$true;
stt_provider='openai';stt_model='changed-model';tts_provider='openai';tts_model='gpt-4o-mini-tts';
limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}}
}}
if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'voice contract drift was accepted' }}
""",
encoding="utf-8-sig",
)
completed = subprocess.run(
[
powershell,
"-NoLogo",
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-File",
str(harness_path),
],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=30,
check=False,
)
self.assertEqual(
completed.returncode,
0,
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
)
def test_engine_log_rotation_cannot_dirty_the_stable_release(self) -> None:
self.assertIn('Destination "$logFile.$rotateStamp.bak.log"', PUBLIC_RUNTIME)
self.assertNotIn('Destination "$logFile.$rotateStamp.bak"', PUBLIC_RUNTIME)
def test_failure_evidence_keeps_a_clean_detached_git_source(self) -> None:
self.assertIn('$failedPath = "$OutputPath.failed.log"', PUBLIC_RUNTIME)
self.assertNotIn('$failedPath = "$OutputPath.failed.json"', PUBLIC_RUNTIME)
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
(root / ".gitignore").write_text("*.log\n", encoding="utf-8")
(root / "tracked.txt").write_text("release\n", encoding="utf-8")
commands = (
["git", "init", "--quiet"],
["git", "config", "user.name", "Yun Chan"],
["git", "config", "user.email", "yun.chan@example.invalid"],
["git", "add", ".gitignore", "tracked.txt"],
["git", "commit", "--quiet", "-m", "테스트 기준"],
["git", "checkout", "--quiet", "--detach", "HEAD"],
)
for command in commands:
subprocess.run(
command,
cwd=root,
capture_output=True,
check=True,
)
failure_evidence = root / "public-runtime-launch-provenance.log.failed.log"
failure_evidence.write_text('{"status":"failed_rolled_back"}\n', encoding="utf-8")
ignored = subprocess.run(
["git", "check-ignore", "--quiet", failure_evidence.name],
cwd=root,
check=False,
)
self.assertEqual(ignored.returncode, 0)
status = subprocess.run(
["git", "status", "--porcelain", "--untracked-files=normal"],
cwd=root,
capture_output=True,
check=True,
)
self.assertEqual(status.stdout, b"")
def test_fresh_cutover_requires_and_restores_a_pinned_prior_runtime(self) -> None:
prior_capture = PUBLIC_RUNTIME.index("$priorApiProcesses = @(")
mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"')
self.assertLess(prior_capture, mutation)
for expected in (
"requires exactly one prior API process for transactional rollback",
"requires exactly one prior cloudflared process for transactional rollback",
"$freshPriorApiIdentity = Wait-ProcessIdentity",
"$freshPriorCloudflaredIdentity = Wait-ProcessIdentity",
"$freshEnvironmentSnapshot = Save-ManagedEnvironment",
"Restore-PriorPublicRuntime `",
"Start-PinnedPriorProcess `",
"Restored prior API identity drift",
"Restored prior cloudflared identity drift",
"$freshPromotionCommitted = $true",
):
with self.subTest(expected=expected):
self.assertIn(expected, PUBLIC_RUNTIME)
def test_fresh_transaction_has_no_engine_web_sidecar_or_dns_mutation(self) -> None:
contract = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract") :
PUBLIC_RUNTIME.index("$freshMutationStarted = $false")
]
for expected in (
"if (-not $SkipEngineRestart)",
"if (-not $SkipWebRestart)",
"if ($RouteCloudflareDns)",
"$CanonicalPublicHealthUrl",
):
with self.subTest(expected=expected):
self.assertIn(expected, contract)
api_mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"')
preflight = PUBLIC_RUNTIME[:api_mutation]
self.assertIn("requires exact healthy voice sidecars as an unchanged precondition", preflight)
self.assertIn("-Uri $CanonicalPublicHealthUrl", preflight)
self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", preflight)
self.assertIn("$health.engine -eq $true", preflight)
stt_start = PUBLIC_RUNTIME.index("& $WhisperStartScript `")
tts_start = PUBLIC_RUNTIME.index("& $MeloTtsStartScript `")
self.assertLess(
PUBLIC_RUNTIME.index("Fresh public promotion will not mutate local_whisper"),
stt_start,
)
self.assertLess(
PUBLIC_RUNTIME.index("Fresh public promotion will not mutate MeloTTS"),
tts_start,
)
def test_success_and_rollback_require_canonical_public_engine_and_voice(self) -> None:
rollback = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") :
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint")
]
self.assertGreaterEqual(rollback.count("$health.engine -eq $true"), 2)
self.assertIn("Test-VoiceSidecarReady", rollback)
self.assertIn("Test-VoiceHealthContract", rollback)
success_gate = PUBLIC_RUNTIME.index('$freshFailureStage = "public_health_validation"')
receipt = PUBLIC_RUNTIME.index("$provenance = [ordered]@{")
committed = PUBLIC_RUNTIME.index("$freshPromotionCommitted = $true")
self.assertLess(success_gate, receipt)
self.assertLess(receipt, committed)
success_section = PUBLIC_RUNTIME[success_gate:receipt]
self.assertIn("-Uri $CanonicalPublicHealthUrl", success_section)
self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", success_section)
self.assertIn("-Uri $CanonicalPublicOpenApiUrl", success_section)
self.assertIn("Test-RequiredOpenApiPaths", success_section)
self.assertIn('"/admin/voice-runtime"', PUBLIC_RUNTIME)
self.assertIn("$health.engine -eq $true", success_section)
self.assertIn("Test-VoiceSidecarReady", success_section)
def test_failed_cutover_emits_metadata_only_rollback_evidence(self) -> None:
failure_writer = PUBLIC_RUNTIME[
PUBLIC_RUNTIME.index("function Write-FailedFreshPromotionEvidence") :
PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract")
]
for expected in (
'schema_version = "vignette.public-runtime-launch-failure.v1"',
'"failed_rolled_back"',
'"failed_rollback"',
"failure_stage = $FailureStage",
"attempted = $true",
"succeeded = $RollbackSucceeded",
"Write-Utf8TextAtomically",
):
with self.subTest(expected=expected):
self.assertIn(expected, failure_writer)
self.assertNotIn("command_line", failure_writer)
self.assertNotIn("executable_path", failure_writer)
def test_receipt_is_direct_input_for_windows_topology_capture(self) -> None: def test_receipt_is_direct_input_for_windows_topology_capture(self) -> None:
receipt = PUBLIC_RUNTIME.index( receipt = PUBLIC_RUNTIME.index(
'schema_version = "vignette.public-runtime-launch-provenance.v1"' 'schema_version = "vignette.public-runtime-launch-provenance.v1"'

View file

@ -16,6 +16,8 @@
[int]$ApiPort = 8001, [int]$ApiPort = 8001,
[int]$WebPort = 5174, [int]$WebPort = 5174,
[int]$EnginePort = 9099, [int]$EnginePort = 9099,
[int]$WhisperPort = 9882,
[int]$MeloTtsPort = 9883,
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
@ -36,6 +38,7 @@ $ErrorActionPreference = "Stop"
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
$expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
function Invoke-GitText { function Invoke-GitText {
param([string[]]$Arguments) param([string[]]$Arguments)
@ -54,6 +57,9 @@ function Assert-StableSourceProvenance {
if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) { if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) {
throw "Pinned start script not found at $startScript" throw "Pinned start script not found at $startScript"
} }
if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) {
throw "Pinned voice sidecar probe not found at $voiceSidecarProbe"
}
$runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path $runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path
if (-not [string]::Equals( if (-not [string]::Equals(
@ -99,7 +105,8 @@ function Assert-StableSourceProvenance {
foreach ($relativePath in @( foreach ($relativePath in @(
"scripts/watch-public-runtime.ps1", "scripts/watch-public-runtime.ps1",
"scripts/start-public-runtime.ps1" "scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py"
)) { )) {
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
} }
@ -203,6 +210,47 @@ function Get-EngineHeaders {
return $null return $null
} }
function Test-VoiceApiReady {
param([object]$Health)
return (
$null -ne $Health -and
$Health.status -eq "ok" -and
$Health.available -eq $true -and
$Health.stt_available -eq $true -and
$Health.tts_available -eq $true -and
$Health.stt_provider -eq "local_whisper" -and
$Health.stt_model -eq "small" -and
$Health.tts_provider -eq "melotts" -and
$Health.tts_model -eq "melotts-korean" -and
$Health.limits.uvicorn_ws_max_queue -eq 4
)
}
function Test-VoiceSidecarStack {
$probeArgs = @(
"-X", "utf8", "-B", $voiceSidecarProbe,
"--component", "all",
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
"--stt-provider", "local_whisper",
"--stt-model", "small",
"--stt-language", "ko",
"--stt-device", "cpu",
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
"--tts-provider", "melotts",
"--tts-model", "melotts-korean",
"--tts-language", "KR",
"--timeout-seconds", "5"
)
$output = @(& $Python @probeArgs 2>$null)
$probeExit = $LASTEXITCODE
return [pscustomobject]@{
Name = "voice-sidecars"
Ok = $probeExit -eq 0
Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" }
}
}
function Test-CloudflaredProcess { function Test-CloudflaredProcess {
if ($SkipCloudflaredRestart) { if ($SkipCloudflaredRestart) {
return [pscustomobject]@{ return [pscustomobject]@{
@ -245,10 +293,16 @@ $checks = @(
-Uri "http://127.0.0.1:$ApiPort/health" ` -Uri "http://127.0.0.1:$ApiPort/health" `
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
-TimeoutSec 60), -TimeoutSec 60),
(Test-JsonHealth `
-Name "voice-api" `
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
-TimeoutSec 30),
(Test-JsonHealth ` (Test-JsonHealth `
-Name "web-preview" ` -Name "web-preview" `
-Uri "http://127.0.0.1:$WebPort/" ` -Uri "http://127.0.0.1:$WebPort/" `
-IsHealthy { param($body) $true }), -IsHealthy { param($body) $true }),
(Test-VoiceSidecarStack),
(Test-CloudflaredProcess) (Test-CloudflaredProcess)
) )
@ -294,6 +348,8 @@ $startArgs = @{
ApiPort = $ApiPort ApiPort = $ApiPort
WebPort = $WebPort WebPort = $WebPort
EnginePort = $EnginePort EnginePort = $EnginePort
WhisperPort = $WhisperPort
MeloTtsPort = $MeloTtsPort
Python = $Python Python = $Python
Cloudflared = $Cloudflared Cloudflared = $Cloudflared
CloudflaredConfig = $CloudflaredConfig CloudflaredConfig = $CloudflaredConfig
@ -324,6 +380,15 @@ if (!$apiAfter.Ok) {
throw "Public API still unhealthy after restart: $($apiAfter.Detail)" throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
} }
$voiceApiAfter = Test-JsonHealth `
-Name "voice-api" `
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
-TimeoutSec 30
if (!$voiceApiAfter.Ok) {
throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)"
}
$webAfter = Test-JsonHealth ` $webAfter = Test-JsonHealth `
-Name "web-preview" ` -Name "web-preview" `
-Uri "http://127.0.0.1:$WebPort/" ` -Uri "http://127.0.0.1:$WebPort/" `
@ -343,6 +408,11 @@ if (!$engineAfter.Ok) {
throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)" throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)"
} }
$voiceSidecarsAfter = Test-VoiceSidecarStack
if (!$voiceSidecarsAfter.Ok) {
throw "Exact local voice sidecars still unhealthy after restart"
}
if (!$SkipPublicHealth) { if (!$SkipPublicHealth) {
$publicAfter = Test-JsonHealth ` $publicAfter = Test-JsonHealth `
-Name "public-api" ` -Name "public-api" `