From aaebe4450ee81bdd4475593a6d86f00d76d6b6d0 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Sun, 9 Aug 2026 21:57:07 +0900 Subject: [PATCH] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EB=9F=B0=ED=83=80?= =?UTF-8?q?=EC=9E=84=20=EB=A1=A4=EB=B0=B1=20=EA=B2=BD=EA=B3=84=20=EA=B0=95?= =?UTF-8?q?=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/boot-public-runtime.ps1 | 90 ++- scripts/install-public-runtime-task.ps1 | 10 +- scripts/register-boot-task.ps1 | 10 +- scripts/start-public-runtime.ps1 | 636 +++++++++++++++++- ...test_public_runtime_watchdog_provenance.py | 95 +++ scripts/test_start_public_runtime_contract.py | 388 ++++++++++- scripts/watch-public-runtime.ps1 | 72 +- 7 files changed, 1271 insertions(+), 30 deletions(-) diff --git a/scripts/boot-public-runtime.ps1 b/scripts/boot-public-runtime.ps1 index e149eb2..ff9b33c 100644 --- a/scripts/boot-public-runtime.ps1 +++ b/scripts/boot-public-runtime.ps1 @@ -28,6 +28,10 @@ param( [int]$DbTimeoutSec = 90, [int]$DbPort = 55432, [int]$ApiPort = 8001, + [int]$EnginePort = 9099, + [int]$WhisperPort = 9882, + [int]$MeloTtsPort = 9883, + [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$BootLog = "" ) @@ -36,6 +40,7 @@ $ErrorActionPreference = "Stop" $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" +$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" function Invoke-GitText { param([string[]]$Arguments) @@ -48,7 +53,7 @@ function Invoke-GitText { } function Assert-StableSourceProvenance { - foreach ($requiredScript in @($expectedBootScript, $startScript)) { + foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Pinned public runtime script not found at $requiredScript" } @@ -97,7 +102,8 @@ function Assert-StableSourceProvenance { } foreach ($relativePath in @( "scripts/boot-public-runtime.ps1", - "scripts/start-public-runtime.ps1" + "scripts/start-public-runtime.ps1", + "scripts/probe-public-voice-sidecars.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -159,7 +165,7 @@ function Test-ApiControlPlaneHealthy { $body = $reader.ReadToEnd() $reader.Close(); $resp.Close() $h = $body | ConvertFrom-Json - if ($h.environment -eq "prod" -and $h.db -eq $true) { return $true } + if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true } Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body) return $false } catch { @@ -172,13 +178,72 @@ function Test-ApiControlPlaneHealthy { function Test-EngineHealthy { try { - $response = Invoke-RestMethod -Uri "http://127.0.0.1:9099/health" -TimeoutSec 10 + $response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10 return $response.ok -eq $true } catch { return $false } } +function Test-VoiceApiReady { + param([object]$Health) + + return ( + $null -ne $Health -and + $Health.status -eq "ok" -and + $Health.available -eq $true -and + $Health.stt_available -eq $true -and + $Health.tts_available -eq $true -and + $Health.stt_provider -eq "local_whisper" -and + $Health.stt_model -eq "small" -and + $Health.tts_provider -eq "melotts" -and + $Health.tts_model -eq "melotts-korean" -and + $Health.limits.uvicorn_ws_max_queue -eq 4 + ) +} + +function Test-VoiceApiHealthy { + for ($i = 1; $i -le 3; $i++) { + try { + $req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health") + $req.Timeout = 5000 + $req.ReadWriteTimeout = 5000 + $req.Proxy = $null + $resp = $req.GetResponse() + $reader = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $body = $reader.ReadToEnd() + $reader.Close(); $resp.Close() + $health = $body | ConvertFrom-Json + if (Test-VoiceApiReady -Health $health) { return $true } + Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body) + return $false + } catch { + Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message) + Start-Sleep -Seconds 2 + } + } + return $false +} + +function Test-VoiceSidecarStack { + $probeArgs = @( + "-X", "utf8", "-B", $voiceSidecarProbe, + "--component", "all", + "--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen", + "--stt-provider", "local_whisper", + "--stt-model", "small", + "--stt-language", "ko", + "--stt-device", "cpu", + "--tts-url", "http://127.0.0.1:$MeloTtsPort", + "--tts-provider", "melotts", + "--tts-model", "melotts-korean", + "--tts-language", "KR", + "--timeout-seconds", "5" + ) + & $Python @probeArgs 1>$null 2>$null + return $LASTEXITCODE -eq 0 +} + Write-BootLog "================ boot start ================" # 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기) @@ -217,12 +282,16 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) { Write-BootLog "postgres 127.0.0.1:$DbPort up" # 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지) -if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) { - Write-BootLog "control plane and engine already healthy; skipping runtime restart" +if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) { + Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart" } else { Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart" $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript ` -Workspace $resolvedSourceRoot ` + -Python $Python ` + -EnginePort $EnginePort ` + -WhisperPort $WhisperPort ` + -MeloTtsPort $MeloTtsPort ` -SkipWebRestart 2>&1 $out | ForEach-Object { Write-BootLog (" pub> " + $_) } if ($LASTEXITCODE -ne 0) { @@ -233,12 +302,13 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) { # 4) 최종 확인 if (Test-ApiControlPlaneHealthy) { - if (Test-EngineHealthy) { - Write-BootLog "boot OK: control plane and engine healthy" + if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) { + Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy" + exit 0 } else { - Write-BootLog "boot OK: admin/auth control plane healthy; engine remains degraded" + Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded" + exit 2 } - exit 0 } else { Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log" exit 2 diff --git a/scripts/install-public-runtime-task.ps1 b/scripts/install-public-runtime-task.ps1 index 5411d2a..ec0be69 100644 --- a/scripts/install-public-runtime-task.ps1 +++ b/scripts/install-public-runtime-task.ps1 @@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime-task.ps1" $watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" +$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" function Invoke-GitText { param([string[]]$Arguments) @@ -30,7 +31,7 @@ function Invoke-GitText { return (@($value) -join [Environment]::NewLine).Trim() } -foreach ($requiredScript in @($installerScript, $watchScript, $startScript)) { +foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe)) { if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Public runtime script not found at $requiredScript" } @@ -69,9 +70,10 @@ if ($dirty) { throw "Stable source is not clean; refusing watchdog installation" } foreach ($relativePath in @( - "scripts/install-public-runtime-task.ps1", - "scripts/watch-public-runtime.ps1", - "scripts/start-public-runtime.ps1" + "scripts/install-public-runtime-task.ps1", + "scripts/watch-public-runtime.ps1", + "scripts/start-public-runtime.ps1", + "scripts/probe-public-voice-sidecars.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } diff --git a/scripts/register-boot-task.ps1 b/scripts/register-boot-task.ps1 index cbef271..c13e584 100644 --- a/scripts/register-boot-task.ps1 +++ b/scripts/register-boot-task.ps1 @@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $RegisterScript = Join-Path $resolvedSourceRoot "scripts\register-boot-task.ps1" $BootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1" $StartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" +$VoiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" $UserName = "$env:USERDOMAIN\$env:USERNAME" function Invoke-GitText { @@ -31,7 +32,7 @@ function Invoke-GitText { return (@($value) -join [Environment]::NewLine).Trim() } -foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript)) { +foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript, $VoiceSidecarProbe)) { if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Public runtime script not found: $requiredScript" } @@ -70,9 +71,10 @@ if ($dirty) { throw "Stable source is not clean; refusing boot task registration" } foreach ($relativePath in @( - "scripts/register-boot-task.ps1", - "scripts/boot-public-runtime.ps1", - "scripts/start-public-runtime.ps1" + "scripts/register-boot-task.ps1", + "scripts/boot-public-runtime.ps1", + "scripts/start-public-runtime.ps1", + "scripts/probe-public-voice-sidecars.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } diff --git a/scripts/start-public-runtime.ps1 b/scripts/start-public-runtime.ps1 index ebf6a9d..3f5c735 100644 --- a/scripts/start-public-runtime.ps1 +++ b/scripts/start-public-runtime.ps1 @@ -15,6 +15,7 @@ [switch]$RouteCloudflareDns, [string]$CloudflareTunnelName = "vignette", [switch]$SkipCloudflaredRestart, + [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health", [switch]$RequireFreshPublicProvenance, [string]$ExpectedSourceCommit = "", [string]$ExpectedSourceTree = "", @@ -51,6 +52,15 @@ $WhisperLanguage = "ko" $WhisperDevice = "cpu" $MeloTtsModel = "melotts-korean" $MeloTtsLanguage = "KR" +$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health" +$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health" +$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json" +$RequiredPublicApiPaths = @( + "/health", + "/voice/health", + "/voice/speech", + "/admin/voice-runtime" +) $PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net") $PublicWebHostnames = @("vnet.18ka.net") @@ -97,6 +107,18 @@ function Test-PortListener { return $null -ne $listener } +function Get-ListenerProcessIds { + param([int]$Port) + + return @( + Get-NetTCPConnection ` + -State Listen ` + -LocalPort $Port ` + -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty OwningProcess -Unique + ) +} + function Test-VoiceSidecarReady { param( [ValidateSet("stt", "tts")] @@ -156,6 +178,74 @@ function Test-VoiceApiReady { ) } +function Test-PriorVoiceApiReady { + param([object]$Health) + + return ( + $null -ne $Health -and + $Health.status -eq "ok" -and + $Health.available -eq $true -and + $Health.stt_available -eq $true -and + $Health.tts_available -eq $true -and + -not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and + -not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and + -not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and + -not [string]::IsNullOrWhiteSpace([string]$Health.tts_model) + ) +} + +function ConvertTo-SafeVoiceHealthContract { + param([object]$Health) + + return [ordered]@{ + status = [string]$Health.status + available = [bool]$Health.available + stt_available = [bool]$Health.stt_available + tts_available = [bool]$Health.tts_available + stt_provider = [string]$Health.stt_provider + stt_model = [string]$Health.stt_model + tts_provider = [string]$Health.tts_provider + tts_model = [string]$Health.tts_model + uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue + } +} + +function Test-VoiceHealthContract { + param( + [object]$Health, + [System.Collections.IDictionary]$Expected + ) + + if (-not (Test-PriorVoiceApiReady -Health $Health)) { + return $false + } + $actual = ConvertTo-SafeVoiceHealthContract -Health $Health + foreach ($name in $Expected.Keys) { + if ($actual[$name].ToString() -cne $Expected[$name].ToString()) { + return $false + } + } + return $true +} + +function Test-RequiredOpenApiPaths { + param( + [object]$Document, + [string[]]$RequiredPaths + ) + + if ($null -eq $Document -or $null -eq $Document.paths) { + return $false + } + $actualPaths = @($Document.paths.PSObject.Properties.Name) + foreach ($requiredPath in $RequiredPaths) { + if ($actualPaths -notcontains $requiredPath) { + return $false + } + } + return $true +} + function Test-EngineReady { param( [int]$Port, @@ -247,16 +337,15 @@ function Stop-ProcessesBounded { throw "Timed out stopping $Role process IDs: $($remaining -join ',')" } -function Stop-UvicornByPort { +function Get-UvicornProcessesByPort { param( [string]$AppImport, - [int]$Port, - [int]$TimeoutSec = 15 + [int]$Port ) # Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해 # 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다. - $processes = @( + return @( Get-CimInstance Win32_Process | Where-Object { $_.Name -like "python*" -and @@ -265,6 +354,16 @@ function Stop-UvicornByPort { $_.CommandLine -like "*--port $Port*" } ) +} + +function Stop-UvicornByPort { + param( + [string]$AppImport, + [int]$Port, + [int]$TimeoutSec = 15 + ) + + $processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port) return @( Stop-ProcessesBounded ` -Processes $processes ` @@ -298,7 +397,7 @@ function Wait-ProcessIdentity { param( [int]$ProcessId, [string]$Role, - [string]$ExpectedCwd, + [string]$ExpectedCwd = "", [int]$TimeoutSec = 15 ) @@ -314,20 +413,26 @@ function Wait-ProcessIdentity { ) { $identityProbeArgs = @( "-X", "utf8", "-c", - "import hashlib,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(p.cmdline()).encode('utf-8', errors='strict')).hexdigest())", + "import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))", "$ProcessId" ) $identityProbe = @(& $Python @identityProbeArgs) - if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 3) { + if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) { throw "Could not prove $Role psutil identity for PID $ProcessId" } $actualCwd = $identityProbe[0].Trim() $startedAtUtc = $identityProbe[1].Trim() $commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant() + $argumentList = @($identityProbe[3] | ConvertFrom-Json) + $environmentObject = $identityProbe[4] | ConvertFrom-Json + $processEnvironment = [ordered]@{} + foreach ($property in $environmentObject.PSObject.Properties) { + $processEnvironment[$property.Name] = [string]$property.Value + } if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") { throw "$Role psutil identity is incomplete for PID $ProcessId" } - if (-not [string]::Equals( + if ($ExpectedCwd -and -not [string]::Equals( [System.IO.Path]::GetFullPath($actualCwd), [System.IO.Path]::GetFullPath($ExpectedCwd), [System.StringComparison]::OrdinalIgnoreCase @@ -343,6 +448,8 @@ function Wait-ProcessIdentity { executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant() command_line = $process.CommandLine command_line_sha256 = $commandLineSha256 + argument_list = $argumentList + environment = $processEnvironment cwd = $actualCwd } } @@ -367,6 +474,274 @@ function ConvertTo-SafeProcessIdentity { } } +function Save-ManagedEnvironment { + param([string[]]$Names) + + $snapshot = [ordered]@{} + foreach ($name in $Names) { + $value = [System.Environment]::GetEnvironmentVariable( + $name, + [System.EnvironmentVariableTarget]::Process + ) + $snapshot[$name] = [ordered]@{ + present = $null -ne $value + value = $value + } + } + return $snapshot +} + +function Restore-ManagedEnvironment { + param([System.Collections.IDictionary]$Snapshot) + + foreach ($name in $Snapshot.Keys) { + $entry = $Snapshot[$name] + if ($entry.present) { + [System.Environment]::SetEnvironmentVariable( + $name, + [string]$entry.value, + [System.EnvironmentVariableTarget]::Process + ) + } else { + [System.Environment]::SetEnvironmentVariable( + $name, + $null, + [System.EnvironmentVariableTarget]::Process + ) + } + } +} + +function Save-CompleteProcessEnvironment { + $snapshot = [ordered]@{} + $environment = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + foreach ($name in $environment.Keys) { + $snapshot[[string]$name] = [string]$environment[$name] + } + return $snapshot +} + +function Set-CompleteProcessEnvironment { + param([System.Collections.IDictionary]$Environment) + + $current = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + foreach ($name in @($current.Keys)) { + [System.Environment]::SetEnvironmentVariable( + [string]$name, + $null, + [System.EnvironmentVariableTarget]::Process + ) + } + foreach ($name in $Environment.Keys) { + [System.Environment]::SetEnvironmentVariable( + [string]$name, + [string]$Environment[$name], + [System.EnvironmentVariableTarget]::Process + ) + } +} + +function ConvertTo-WindowsCommandLineArgument { + param([AllowEmptyString()][string]$Argument) + + if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') { + return $Argument + } + + $builder = New-Object System.Text.StringBuilder + $null = $builder.Append('"') + $backslashes = 0 + foreach ($character in $Argument.ToCharArray()) { + if ($character -eq '\') { + $backslashes++ + continue + } + if ($character -eq '"') { + $null = $builder.Append(('\' * (($backslashes * 2) + 1))) + $null = $builder.Append('"') + $backslashes = 0 + continue + } + if ($backslashes -gt 0) { + $null = $builder.Append(('\' * $backslashes)) + $backslashes = 0 + } + $null = $builder.Append($character) + } + if ($backslashes -gt 0) { + $null = $builder.Append(('\' * ($backslashes * 2))) + } + $null = $builder.Append('"') + return $builder.ToString() +} + +function Join-WindowsArgumentList { + param([object[]]$ArgumentList) + + return (@( + foreach ($argument in $ArgumentList) { + ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument) + } + ) -join ' ') +} + +function Start-PinnedPriorProcess { + param( + [System.Collections.IDictionary]$Identity, + [string]$Role, + [string]$StdoutLog, + [string]$StderrLog + ) + + if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) { + throw "Prior $Role executable is unavailable" + } + $actualExecutableSha256 = ( + Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256 + ).Hash.ToLowerInvariant() + if ($actualExecutableSha256 -ne $Identity.executable_sha256) { + throw "Prior $Role executable SHA256 drift" + } + if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) { + throw "Prior $Role working directory is unavailable" + } + if (@($Identity.argument_list).Count -eq 0) { + throw "Prior $Role argument list is unavailable" + } + if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) { + throw "Prior $Role environment is unavailable" + } + + $callerEnvironment = Save-CompleteProcessEnvironment + try { + Set-CompleteProcessEnvironment -Environment $Identity.environment + $argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list) + return Start-Process -WindowStyle Hidden ` + -FilePath $Identity.executable_path ` + -ArgumentList $argumentString ` + -WorkingDirectory $Identity.cwd ` + -RedirectStandardOutput $StdoutLog ` + -RedirectStandardError $StderrLog ` + -PassThru + } finally { + Set-CompleteProcessEnvironment -Environment $callerEnvironment + } +} + +function Restore-PriorPublicRuntime { + param( + [System.Collections.IDictionary]$PriorApi, + [System.Collections.IDictionary]$PriorCloudflared, + [System.Collections.IDictionary]$PriorLocalVoiceContract, + [System.Collections.IDictionary]$PriorPublicVoiceContract, + [System.Collections.IDictionary]$EnvironmentSnapshot, + [string]$ConfigPath, + [int]$ApiPortValue, + [string]$HealthUrl, + [string]$VoiceHealthUrl, + [int]$TimeoutSec + ) + + $null = @( + Stop-UvicornByPort ` + -AppImport "app.main:app" ` + -Port $ApiPortValue ` + -TimeoutSec $TimeoutSec + ) + Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot + $priorApiProcess = Start-PinnedPriorProcess ` + -Identity $PriorApi ` + -Role "api" ` + -StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") ` + -StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log") + $priorApiIdentity = Wait-ProcessIdentity ` + -ProcessId $priorApiProcess.Id ` + -Role "restored prior api" ` + -ExpectedCwd $PriorApi.cwd ` + -TimeoutSec $TimeoutSec + foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) { + if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) { + throw "Restored prior API identity drift: $field" + } + } + $null = Wait-JsonHealth ` + -Uri "http://127.0.0.1:$ApiPortValue/health" ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + } ` + -TimeoutSec 60 + $null = Wait-JsonHealth ` + -Uri "http://127.0.0.1:$ApiPortValue/voice/health" ` + -IsHealthy { + param($health) + Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract + } ` + -TimeoutSec 30 + if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) { + throw "Restored prior runtime does not have the exact local voice sidecars" + } + + $resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path + $currentCloudflared = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath $resolvedConfigPath ` + -ExactPath + ) + $null = @( + Stop-ProcessesBounded ` + -Processes $currentCloudflared ` + -TimeoutSec $TimeoutSec ` + -Role "failed fresh cloudflared" + ) + $priorCloudflaredProcess = Start-PinnedPriorProcess ` + -Identity $PriorCloudflared ` + -Role "cloudflared" ` + -StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") ` + -StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log") + $priorCloudflaredIdentity = Wait-ProcessIdentity ` + -ProcessId $priorCloudflaredProcess.Id ` + -Role "restored prior cloudflared" ` + -ExpectedCwd $PriorCloudflared.cwd ` + -TimeoutSec $TimeoutSec + foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) { + if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) { + throw "Restored prior cloudflared identity drift: $field" + } + } + $null = Wait-JsonHealth ` + -Uri $HealthUrl ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + } ` + -TimeoutSec 60 + $null = Wait-JsonHealth ` + -Uri $VoiceHealthUrl ` + -IsHealthy { + param($health) + Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract + } ` + -TimeoutSec 30 + + return [ordered]@{ + api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity + cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity + local_health = $true + local_voice_health = $true + public_health = $true + public_voice_health = $true + } +} + function Stop-NodeByPortHint { param([int]$Port) @@ -515,6 +890,39 @@ function Write-Utf8TextAtomically { } } +function Write-FailedFreshPromotionEvidence { + param( + [string]$OutputPath, + [string]$FailureStage, + [bool]$RollbackSucceeded, + [object]$RollbackResult, + [string]$SourceCommit, + [string]$SourceTree + ) + + # Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도 + # 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다. + $failedPath = "$OutputPath.failed.log" + $payload = [ordered]@{ + schema_version = "vignette.public-runtime-launch-failure.v1" + status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" } + captured_at_utc = (Get-Date).ToUniversalTime().ToString("o") + failure_stage = $FailureStage + source = [ordered]@{ + git_commit = $SourceCommit.ToLowerInvariant() + git_tree = $SourceTree.ToLowerInvariant() + } + rollback = [ordered]@{ + attempted = $true + succeeded = $RollbackSucceeded + result = $RollbackResult + } + } + $json = ConvertTo-Json -InputObject $payload -Depth 8 + Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine) + return $failedPath +} + function Assert-FreshPublicProvenanceContract { param( [string]$SourceRoot, @@ -534,6 +942,22 @@ function Assert-FreshPublicProvenanceContract { if ($SkipCloudflaredRestart) { throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart" } + if (-not $SkipEngineRestart) { + throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition" + } + if (-not $SkipWebRestart) { + throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction" + } + if ($RouteCloudflareDns) { + throw "-RequireFreshPublicProvenance forbids DNS route mutation" + } + if (-not [string]::Equals( + $PublicHealthUrl, + $CanonicalPublicHealthUrl, + [System.StringComparison]::OrdinalIgnoreCase + )) { + throw "Fresh public promotion requires the canonical HTTPS public health URL" + } foreach ($sourcePin in @($SourceCommit, $SourceTree)) { if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") { throw "Fresh public provenance requires exact source commit and tree pins" @@ -657,6 +1081,85 @@ function Set-CloudflaredIngress { } } +$freshMutationStarted = $false +$freshPromotionCommitted = $false +$freshFailureStage = "preflight" +$freshPriorApiIdentity = $null +$freshPriorCloudflaredIdentity = $null +$freshPriorLocalVoiceContract = $null +$freshPriorPublicVoiceContract = $null +$freshEnvironmentSnapshot = $null +$resolvedRuntimeProvenancePath = $null +$freshManagedEnvironmentNames = @( + "ENVIRONMENT", + "ENGINE_URL", + "ENGINE_MODE", + "VIGNETTE_LIVE_CLIENT_PROVIDER", + "AUTH_DEV_LOGIN_ENABLED", + "AUTO_SEED_PERSONAS", + "ALLOW_SEED_PERSONA_FALLBACK", + "VIGNETTE_VOICE_POC_SAMPLE_TTS", + "VIGNETTE_VOICE_STT_PROVIDER", + "VIGNETTE_LOCAL_WHISPER_STT_URL", + "VIGNETTE_LOCAL_WHISPER_STT_MODEL", + "VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE", + "VIGNETTE_VOICE_TTS_PROVIDER", + "VIGNETTE_MELOTTS_TTS_URL", + "FRONTEND_BASE_URL", + "CORS_ORIGINS", + "FRONTEND_ORIGIN_MAP" +) + +trap { + $caught = $_ + if ( + $RequireFreshPublicProvenance -and + $freshMutationStarted -and + -not $freshPromotionCommitted + ) { + $rollbackResult = $null + $rollbackSucceeded = $false + $rollbackFailureType = "none" + try { + $rollbackResult = Restore-PriorPublicRuntime ` + -PriorApi $freshPriorApiIdentity ` + -PriorCloudflared $freshPriorCloudflaredIdentity ` + -PriorLocalVoiceContract $freshPriorLocalVoiceContract ` + -PriorPublicVoiceContract $freshPriorPublicVoiceContract ` + -EnvironmentSnapshot $freshEnvironmentSnapshot ` + -ConfigPath $CloudflaredConfig ` + -ApiPortValue $ApiPort ` + -HealthUrl $PublicHealthUrl ` + -VoiceHealthUrl $CanonicalPublicVoiceHealthUrl ` + -TimeoutSec $ProcessStopTimeoutSeconds + $rollbackSucceeded = $true + } catch { + $rollbackFailureType = $_.Exception.GetType().Name + } + + $failedEvidencePath = "" + if ($resolvedRuntimeProvenancePath) { + try { + $failedEvidencePath = Write-FailedFreshPromotionEvidence ` + -OutputPath $resolvedRuntimeProvenancePath ` + -FailureStage $freshFailureStage ` + -RollbackSucceeded $rollbackSucceeded ` + -RollbackResult $rollbackResult ` + -SourceCommit $ExpectedSourceCommit ` + -SourceTree $ExpectedSourceTree + } catch { + $failedEvidencePath = "unavailable" + } + } + + if ($rollbackSucceeded) { + throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" + } + throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)" + } + throw $caught +} + if (!(Test-Path $Python)) { throw "Python 3.11 not found at $Python" } @@ -705,6 +1208,64 @@ if ($RequireFreshPublicProvenance) { -ApiPortValue $ApiPort ` -WebPortValue $WebPort ` -RequireUnchanged + + $priorApiProcesses = @( + Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort + ) + if ($priorApiProcesses.Count -ne 1) { + throw "Fresh public promotion requires exactly one prior API process for transactional rollback" + } + $priorCloudflaredProcesses = @( + Get-CloudflaredProcessesForConfig ` + -ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path ` + -ExactPath + ) + if ($priorCloudflaredProcesses.Count -ne 1) { + throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback" + } + $freshPriorApiIdentity = Wait-ProcessIdentity ` + -ProcessId $priorApiProcesses[0].ProcessId ` + -Role "prior api" ` + -TimeoutSec $ProcessStopTimeoutSeconds + $freshPriorCloudflaredIdentity = Wait-ProcessIdentity ` + -ProcessId $priorCloudflaredProcesses[0].ProcessId ` + -Role "prior cloudflared" ` + -TimeoutSec $ProcessStopTimeoutSeconds + $null = Wait-JsonHealth ` + -Uri "http://127.0.0.1:$ApiPort/health" ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + } ` + -TimeoutSec 30 + $priorLocalVoiceHealth = Wait-JsonHealth ` + -Uri "http://127.0.0.1:$ApiPort/voice/health" ` + -IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } ` + -TimeoutSec 30 + $freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract ` + -Health $priorLocalVoiceHealth + if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) { + throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition" + } + $null = Wait-JsonHealth ` + -Uri $CanonicalPublicHealthUrl ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + } ` + -TimeoutSec 30 + $priorPublicVoiceHealth = Wait-JsonHealth ` + -Uri $CanonicalPublicVoiceHealthUrl ` + -IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } ` + -TimeoutSec 30 + $freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract ` + -Health $priorPublicVoiceHealth + $freshEnvironmentSnapshot = Save-ManagedEnvironment ` + -Names $freshManagedEnvironmentNames } # 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. @@ -732,7 +1293,9 @@ if ($SkipEngineRestart) { $rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss" foreach ($logFile in @($EngineOutLog, $EngineErrLog)) { if (Test-Path $logFile) { - Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak" -Force -ErrorAction SilentlyContinue + # stable detached source의 provenance gate는 untracked 파일도 차단한다. + # suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다. + Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue } } @@ -781,6 +1344,9 @@ $env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{ # 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를 # 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다. if (-not (Test-VoiceSidecarReady -Component "stt")) { + if ($RequireFreshPublicProvenance) { + throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying" + } if (Test-PortListener -Port $WhisperPort) { throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol" } @@ -795,6 +1361,9 @@ if (-not (Test-VoiceSidecarReady -Component "stt")) { } if (-not (Test-VoiceSidecarReady -Component "tts")) { + if ($RequireFreshPublicProvenance) { + throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying" + } if (Test-PortListener -Port $MeloTtsPort) { throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract" } @@ -828,6 +1397,10 @@ $apiLaunchIdentity = $null if ($apiControlPlaneReady -and -not $ForceApiRestart) { Write-Output "Production API and exact local voice stack already healthy; skipping API restart" } else { + if ($RequireFreshPublicProvenance) { + $freshFailureStage = "api_cutover" + $freshMutationStarted = $true + } $apiStoppedProcessIds = @( Stop-UvicornByPort ` -AppImport "app.main:app" ` @@ -890,6 +1463,9 @@ if (-not (Test-VoiceApiReady -Health $voiceHealth)) { } if (!$SkipWebRestart) { + if ($RequireFreshPublicProvenance) { + $freshFailureStage = "web_preview" + } Stop-NodeByPortHint -Port $WebPort $build = Start-Process -FilePath "cmd.exe" ` @@ -916,6 +1492,9 @@ $cloudflaredProcess = $null $cloudflaredLaunchIdentity = $null $cloudflaredStoppedProcessIds = @() if (!$SkipCloudflaredRestart) { + if ($RequireFreshPublicProvenance) { + $freshFailureStage = "cloudflared_cutover" + } if (!(Test-Path $Cloudflared)) { throw "cloudflared not found at $Cloudflared" } @@ -986,6 +1565,7 @@ if (!$SkipCloudflaredRestart) { } if ($RequireFreshPublicProvenance) { + $freshFailureStage = "identity_revalidation" if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) { throw "Fresh public promotion did not produce both API and cloudflared identities" } @@ -1015,6 +1595,32 @@ if ($RequireFreshPublicProvenance) { if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) { throw "Pinned cloudflared config drifted before provenance receipt" } + $freshFailureStage = "public_health_validation" + $publicHealth = Wait-JsonHealth ` + -Uri $CanonicalPublicHealthUrl ` + -IsHealthy { + param($health) + $health.environment -eq "prod" -and + $health.db -eq $true -and + $health.engine -eq $true + } ` + -TimeoutSec 60 + $publicVoiceHealth = Wait-JsonHealth ` + -Uri $CanonicalPublicVoiceHealthUrl ` + -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` + -TimeoutSec 30 + $publicOpenApi = Wait-JsonHealth ` + -Uri $CanonicalPublicOpenApiUrl ` + -IsHealthy { + param($document) + Test-RequiredOpenApiPaths ` + -Document $document ` + -RequiredPaths $RequiredPublicApiPaths + } ` + -TimeoutSec 30 + if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) { + throw "Exact local voice sidecars changed before provenance receipt" + } $psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))") $psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim() if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) { @@ -1038,6 +1644,16 @@ if ($RequireFreshPublicProvenance) { path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path sha256 = $finalConfigSha256 } + public_validation = [ordered]@{ + health_url = $CanonicalPublicHealthUrl + health = $true + voice_health_url = $CanonicalPublicVoiceHealthUrl + voice_health = $true + openapi_url = $CanonicalPublicOpenApiUrl + required_openapi_paths = @($RequiredPublicApiPaths) + openapi = $true + local_voice_sidecars = $true + } replacement = [ordered]@{ api_stopped_pids = @($apiStoppedProcessIds) cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds) @@ -1068,6 +1684,7 @@ if ($RequireFreshPublicProvenance) { } } $provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8 + $freshFailureStage = "receipt_publish" try { Write-Utf8TextAtomically ` -OutputPath $resolvedRuntimeProvenancePath ` @@ -1077,6 +1694,7 @@ if ($RequireFreshPublicProvenance) { # 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다. throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)" } + $freshPromotionCommitted = $true Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath" } diff --git a/scripts/test_public_runtime_watchdog_provenance.py b/scripts/test_public_runtime_watchdog_provenance.py index e5862d5..fb98ad6 100644 --- a/scripts/test_public_runtime_watchdog_provenance.py +++ b/scripts/test_public_runtime_watchdog_provenance.py @@ -15,6 +15,7 @@ BOOT = SCRIPTS / "boot-public-runtime.ps1" BOOT_REGISTER = SCRIPTS / "register-boot-task.ps1" HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs" START = SCRIPTS / "start-public-runtime.ps1" +VOICE_PROBE = SCRIPTS / "probe-public-voice-sidecars.py" REPO_ROOT = SCRIPTS.parent RUNBOOK = REPO_ROOT / "docs" / "ops" / "public-runtime-watchdog.md" LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md" @@ -117,6 +118,97 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask") self.assertLess(dirty_gate, task_registration) + def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None: + for source in (WATCHDOG_SOURCE, BOOT_SOURCE): + for expected in ( + '"--component", "all"', + '"--stt-provider", "local_whisper"', + '"--stt-model", "small"', + '"--stt-device", "cpu"', + '"--tts-provider", "melotts"', + '"--tts-model", "melotts-korean"', + '"scripts\\probe-public-voice-sidecars.py"', + "Test-VoiceSidecarStack", + ): + with self.subTest(source=source[:32], expected=expected): + self.assertIn(expected, source) + self.assertIn("(Test-VoiceSidecarStack)", WATCHDOG_SOURCE) + self.assertIn("$voiceSidecarsAfter = Test-VoiceSidecarStack", WATCHDOG_SOURCE) + self.assertIn('-Name "voice-api"', WATCHDOG_SOURCE) + self.assertIn("$voiceApiAfter = Test-JsonHealth", WATCHDOG_SOURCE) + self.assertIn( + "(Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)", + BOOT_SOURCE, + ) + self.assertGreaterEqual(BOOT_SOURCE.count("Test-VoiceApiHealthy"), 3) + self.assertIn("-WhisperPort $WhisperPort", BOOT_SOURCE) + self.assertIn("-MeloTtsPort $MeloTtsPort", BOOT_SOURCE) + self.assertIn("scripts/probe-public-voice-sidecars.py", INSTALLER_SOURCE) + self.assertIn("scripts/probe-public-voice-sidecars.py", BOOT_REGISTER_SOURCE) + + def test_old_openai_api_is_not_healthy_when_exact_sidecars_are_ready(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + watchdog_contract = WATCHDOG_SOURCE[ + WATCHDOG_SOURCE.index("function Test-VoiceApiReady") : + WATCHDOG_SOURCE.index("function Test-VoiceSidecarStack") + ].strip() + boot_contract = BOOT_SOURCE[ + BOOT_SOURCE.index("function Test-VoiceApiReady") : + BOOT_SOURCE.index("function Test-VoiceApiHealthy") + ].strip() + self.assertEqual(watchdog_contract, boot_contract) + + with tempfile.TemporaryDirectory() as temporary_directory: + harness = Path(temporary_directory) / "voice-api-contract.ps1" + harness.write_text( + watchdog_contract + + r''' +$openAi = [pscustomobject]@{ + status='ok';available=$true;stt_available=$true;tts_available=$true + stt_provider='openai';stt_model='gpt-4o-mini-transcribe' + tts_provider='openai';tts_model='gpt-4o-mini-tts' + limits=[pscustomobject]@{uvicorn_ws_max_queue=4} +} +if (Test-VoiceApiReady -Health $openAi) { throw 'old OpenAI API was accepted' } +$local = [pscustomobject]@{ + status='ok';available=$true;stt_available=$true;tts_available=$true + stt_provider='local_whisper';stt_model='small' + tts_provider='melotts';tts_model='melotts-korean' + limits=[pscustomobject]@{uvicorn_ws_max_queue=4} +} +if (-not (Test-VoiceApiReady -Health $local)) { throw 'exact local API was rejected' } +$local.limits.uvicorn_ws_max_queue = 5 +if (Test-VoiceApiReady -Health $local) { throw 'wrong websocket queue was accepted' } +''', + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + def test_hidden_trigger_never_executes_a_workspace_script_directly(self) -> None: for expected in ( "VignettePublicRuntimeWatchdog", @@ -178,11 +270,13 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): copied_boot = scripts / BOOT.name copied_boot_register = scripts / BOOT_REGISTER.name copied_start = scripts / START.name + copied_voice_probe = scripts / VOICE_PROBE.name shutil.copy2(WATCHDOG, copied_watchdog) shutil.copy2(INSTALLER, copied_installer) shutil.copy2(BOOT, copied_boot) shutil.copy2(BOOT_REGISTER, copied_boot_register) shutil.copy2(START, copied_start) + shutil.copy2(VOICE_PROBE, copied_voice_probe) self._git(root, "init") self._git(root, "config", "user.name", "Watchdog Contract Test") @@ -195,6 +289,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): "scripts/boot-public-runtime.ps1", "scripts/register-boot-task.ps1", "scripts/start-public-runtime.ps1", + "scripts/probe-public-voice-sidecars.py", ) self._git(root, "commit", "-m", "watchdog fixture") self._git(root, "checkout", "--detach") diff --git a/scripts/test_start_public_runtime_contract.py b/scripts/test_start_public_runtime_contract.py index 5e3862d..b55442e 100644 --- a/scripts/test_start_public_runtime_contract.py +++ b/scripts/test_start_public_runtime_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import unittest import shutil import subprocess +import sys import tempfile from pathlib import Path @@ -83,6 +84,10 @@ class FreshPublicProvenanceContractTest(unittest.TestCase): "[switch]$RequireFreshPublicProvenance", 'throw "-RequireFreshPublicProvenance requires -ForceApiRestart"', 'throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"', + 'throw "-RequireFreshPublicProvenance requires -SkipEngineRestart;', + 'throw "-RequireFreshPublicProvenance requires -SkipWebRestart;', + 'throw "-RequireFreshPublicProvenance forbids DNS route mutation"', + "Fresh public promotion requires the canonical HTTPS public health URL", "ExpectedSourceCommit", "ExpectedSourceTree", "ExpectedPythonSha256", @@ -171,6 +176,18 @@ if ($receipt.status -ne 'passed') {{ throw 'atomic receipt content mismatch' }} Write-Utf8TextAtomically -OutputPath $resolved -Value '{{"status":"replaced"}}' $replacement = Get-Content -LiteralPath $resolved -Raw -Encoding UTF8 | ConvertFrom-Json if ($replacement.status -ne 'replaced') {{ throw 'atomic replacement mismatch' }} +$failedPath = Write-FailedFreshPromotionEvidence ` + -OutputPath $resolved ` + -FailureStage 'receipt_publish' ` + -RollbackSucceeded $true ` + -RollbackResult @{{local_health=$true;public_health=$true}} ` + -SourceCommit ('a' * 40) ` + -SourceTree ('b' * 40) +$failed = Get-Content -LiteralPath $failedPath -Raw -Encoding UTF8 | ConvertFrom-Json +if ((Split-Path -Leaf $failedPath) -notlike '*.failed.log') {{ throw 'failure evidence suffix mismatch' }} +if ($failed.status -ne 'failed_rolled_back') {{ throw 'failure evidence status mismatch' }} +if ($failed.failure_stage -ne 'receipt_publish') {{ throw 'failure evidence stage mismatch' }} +if (-not $failed.rollback.succeeded) {{ throw 'failure evidence rollback mismatch' }} if (@(Get-ChildItem -LiteralPath (Split-Path -Parent $resolved) -Filter '*.tmp').Count -ne 0) {{ throw 'temporary receipt files were not cleaned' }} @@ -271,7 +288,7 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri def test_receipt_projects_no_raw_command_or_config_contents(self) -> None: projection = PUBLIC_RUNTIME[ PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") : - PUBLIC_RUNTIME.index("function Stop-NodeByPortHint") + PUBLIC_RUNTIME.index("function Save-ManagedEnvironment") ] self.assertIn("command_line_sha256 =", projection) self.assertNotIn("command_line =", projection) @@ -297,9 +314,376 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") ] self.assertIn("datetime.fromtimestamp(p.create_time(), UTC)", identity) - self.assertIn("chr(0).join(p.cmdline())", identity) + self.assertIn("argv=p.cmdline()", identity) + self.assertIn("chr(0).join(argv)", identity) + self.assertIn("argument_list = $argumentList", identity) + self.assertIn("environment = $processEnvironment", identity) self.assertIn("command_line_sha256 = $commandLineSha256", identity) + def test_prior_process_argv_and_environment_round_trip_in_powershell_51(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + function_source = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Save-CompleteProcessEnvironment") : + PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") + ] + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + functions_path = root / "rollback-process-functions.ps1" + harness_path = root / "rollback-process-harness.ps1" + functions_path.write_text(function_source, encoding="utf-8-sig") + quoted_functions = str(functions_path).replace("'", "''") + quoted_root = str(root).replace("'", "''") + quoted_python = sys.executable.replace("'", "''") + harness_path.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quoted_functions}' +$root = '{quoted_root}' +$python = '{quoted_python}' +if ($null -eq (Get-Command Get-FileHash -ErrorAction SilentlyContinue)) {{ + function Get-FileHash {{ + param([string]$LiteralPath, [string]$Algorithm) + $stream = [IO.File]::OpenRead($LiteralPath) + try {{ + $hasher = [Security.Cryptography.SHA256]::Create() + try {{ $hash = $hasher.ComputeHash($stream) }} finally {{ $hasher.Dispose() }} + }} finally {{ + $stream.Dispose() + }} + [pscustomobject]@{{Hash = ([BitConverter]::ToString($hash)).Replace('-', '')}} + }} +}} +$pythonSha256 = (Get-FileHash -LiteralPath $python -Algorithm SHA256).Hash.ToLowerInvariant() +$originalEnvironment = Save-CompleteProcessEnvironment +try {{ + [Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'caller', 'Process') + [Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', 'caller-only', 'Process') + $callerEnvironment = Save-CompleteProcessEnvironment + [Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'prior', 'Process') + [Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', $null, 'Process') + $priorEnvironment = Save-CompleteProcessEnvironment + Set-CompleteProcessEnvironment -Environment $callerEnvironment + + $output = Join-Path $root 'argv-result.json' + $code = "import json,os,sys; open(sys.argv[1], 'w', encoding='utf-8').write(json.dumps({{'argv':sys.argv[2:],'marker':os.environ.get('VIGNETTE_ROLLBACK_ARGV_TEST'),'leak':os.environ.get('VIGNETTE_CALLER_ONLY')}}, ensure_ascii=True))" + $expected = @('plain', 'space value', 'quote"value', 'trailing\', '', 'slashes\\before"quote') + $identity = [ordered]@{{ + executable_path = $python + executable_sha256 = $pythonSha256 + cwd = $root + argument_list = @('-X', 'utf8', '-c', $code, $output) + $expected + environment = $priorEnvironment + }} + $process = Start-PinnedPriorProcess ` + -Identity $identity ` + -Role 'argv-roundtrip' ` + -StdoutLog (Join-Path $root 'child.out.log') ` + -StderrLog (Join-Path $root 'child.err.log') + $process.WaitForExit() + if (-not (Test-Path -LiteralPath $output -PathType Leaf)) {{ throw 'child did not write argv result' }} + $actual = Get-Content -LiteralPath $output -Raw -Encoding UTF8 | ConvertFrom-Json + if (@($actual.argv).Count -ne $expected.Count) {{ throw 'argument count mismatch' }} + for ($i = 0; $i -lt $expected.Count; $i++) {{ + if ($actual.argv[$i] -cne $expected[$i]) {{ throw "argument mismatch at $i" }} + }} + if ($actual.marker -cne 'prior') {{ throw 'prior environment was not inherited' }} + if ($null -ne $actual.leak) {{ throw 'caller-only environment leaked into prior process' }} + if ($env:VIGNETTE_ROLLBACK_ARGV_TEST -cne 'caller') {{ throw 'caller environment marker was not restored' }} + if ($env:VIGNETTE_CALLER_ONLY -cne 'caller-only') {{ throw 'caller-only environment was not restored' }} +}} finally {{ + Set-CompleteProcessEnvironment -Environment $originalEnvironment +}} +""", + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness_path), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_rollback_restores_prior_openai_voice_contract_in_powershell_51(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + voice_contract_source = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Test-PriorVoiceApiReady") : + PUBLIC_RUNTIME.index("function Test-RequiredOpenApiPaths") + ] + rollback_source = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") : + PUBLIC_RUNTIME.index("function Stop-NodeByPortHint") + ] + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + functions_path = root / "rollback-contract-functions.ps1" + harness_path = root / "rollback-contract-harness.ps1" + functions_path.write_text( + voice_contract_source + rollback_source, + encoding="utf-8-sig", + ) + quoted_functions = str(functions_path).replace("'", "''") + quoted_root = str(root).replace("'", "''") + harness_path.write_text( + f"""$ErrorActionPreference = 'Stop' +. '{quoted_functions}' +$events = New-Object System.Collections.ArrayList +$root = '{quoted_root}' +function Stop-UvicornByPort {{ param($AppImport,$Port,$TimeoutSec); $null=$events.Add('stop-api'); 11 }} +function Restore-ManagedEnvironment {{ param($Snapshot); $null=$events.Add('restore-env') }} +function Start-PinnedPriorProcess {{ + param($Identity,$Role,$StdoutLog,$StderrLog) + $null=$events.Add("start-$Role") + if ($Role -eq 'api') {{ [pscustomobject]@{{Id=101}} }} else {{ [pscustomobject]@{{Id=202}} }} +}} +function Wait-ProcessIdentity {{ + param($ProcessId,$Role,$ExpectedCwd,$TimeoutSec) + $null=$events.Add("identity-$Role") + [ordered]@{{ + pid=[int]$ProcessId; started_at_utc='2026-08-09T00:00:00Z'; + executable_name='runtime.exe'; executable_sha256=('a' * 64); + command_line_sha256=('b' * 64); cwd=$root + }} +}} +function Wait-JsonHealth {{ + param($Uri,$IsHealthy,$TimeoutSec) + $null=$events.Add("health-$Uri") + if ($Uri -like '*/voice/health') {{ + $payload=[pscustomobject]@{{ + status='ok';available=$true;stt_available=$true;tts_available=$true; + stt_provider='openai';stt_model='gpt-4o-mini-transcribe'; + tts_provider='openai';tts_model='gpt-4o-mini-tts'; + limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}} + }} + }} else {{ + $payload=[pscustomobject]@{{environment='prod';db=$true;engine=$true}} + }} + if (-not (& $IsHealthy $payload)) {{ throw "health predicate failed: $Uri" }} + $payload +}} +function Test-VoiceSidecarReady {{ param($Component); $true }} +function Get-CloudflaredProcessesForConfig {{ param($ConfigPath,[switch]$ExactPath); @([pscustomobject]@{{ProcessId=303}}) }} +function Stop-ProcessesBounded {{ param($Processes,$TimeoutSec,$Role); $null=$events.Add('stop-cloudflared'); 303 }} +function ConvertTo-SafeProcessIdentity {{ param($Identity); $Identity }} + +$configPath = Join-Path $root 'cloudflared.yml' +[IO.File]::WriteAllText($configPath, 'tunnel: test') +$priorVoice = [ordered]@{{ + status='ok';available=$true;stt_available=$true;tts_available=$true; + stt_provider='openai';stt_model='gpt-4o-mini-transcribe'; + tts_provider='openai';tts_model='gpt-4o-mini-tts';uvicorn_ws_max_queue=4 +}} +$priorApi=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$priorCloud=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}} +$result = Restore-PriorPublicRuntime ` + -PriorApi $priorApi ` + -PriorCloudflared $priorCloud ` + -PriorLocalVoiceContract $priorVoice ` + -PriorPublicVoiceContract $priorVoice ` + -EnvironmentSnapshot @{{}} ` + -ConfigPath $configPath ` + -ApiPortValue 8001 ` + -HealthUrl 'https://api-vignette.chanpaca.net/health' ` + -VoiceHealthUrl 'https://api-vignette.chanpaca.net/voice/health' ` + -TimeoutSec 5 +foreach ($field in @('local_health','local_voice_health','public_health','public_voice_health')) {{ + if (-not $result[$field]) {{ throw "rollback result missing $field" }} +}} +if ($events.IndexOf('stop-api') -ge $events.IndexOf('start-api')) {{ throw 'API restore ordering mismatch' }} +if ($events.IndexOf('stop-cloudflared') -ge $events.IndexOf('start-cloudflared')) {{ throw 'cloudflared restore ordering mismatch' }} +if ($events.IndexOf('start-api') -ge $events.IndexOf('health-http://127.0.0.1:8001/health')) {{ throw 'local health ordering mismatch' }} +if ($events.IndexOf('start-cloudflared') -ge $events.IndexOf('health-https://api-vignette.chanpaca.net/health')) {{ throw 'public health ordering mismatch' }} +$changed=[pscustomobject]@{{ + status='ok';available=$true;stt_available=$true;tts_available=$true; + stt_provider='openai';stt_model='changed-model';tts_provider='openai';tts_model='gpt-4o-mini-tts'; + limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}} +}} +if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'voice contract drift was accepted' }} +""", + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness_path), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_engine_log_rotation_cannot_dirty_the_stable_release(self) -> None: + self.assertIn('Destination "$logFile.$rotateStamp.bak.log"', PUBLIC_RUNTIME) + self.assertNotIn('Destination "$logFile.$rotateStamp.bak"', PUBLIC_RUNTIME) + + def test_failure_evidence_keeps_a_clean_detached_git_source(self) -> None: + self.assertIn('$failedPath = "$OutputPath.failed.log"', PUBLIC_RUNTIME) + self.assertNotIn('$failedPath = "$OutputPath.failed.json"', PUBLIC_RUNTIME) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + (root / ".gitignore").write_text("*.log\n", encoding="utf-8") + (root / "tracked.txt").write_text("release\n", encoding="utf-8") + commands = ( + ["git", "init", "--quiet"], + ["git", "config", "user.name", "Yun Chan"], + ["git", "config", "user.email", "yun.chan@example.invalid"], + ["git", "add", ".gitignore", "tracked.txt"], + ["git", "commit", "--quiet", "-m", "테스트 기준"], + ["git", "checkout", "--quiet", "--detach", "HEAD"], + ) + for command in commands: + subprocess.run( + command, + cwd=root, + capture_output=True, + check=True, + ) + + failure_evidence = root / "public-runtime-launch-provenance.log.failed.log" + failure_evidence.write_text('{"status":"failed_rolled_back"}\n', encoding="utf-8") + ignored = subprocess.run( + ["git", "check-ignore", "--quiet", failure_evidence.name], + cwd=root, + check=False, + ) + self.assertEqual(ignored.returncode, 0) + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=normal"], + cwd=root, + capture_output=True, + check=True, + ) + self.assertEqual(status.stdout, b"") + + def test_fresh_cutover_requires_and_restores_a_pinned_prior_runtime(self) -> None: + prior_capture = PUBLIC_RUNTIME.index("$priorApiProcesses = @(") + mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"') + self.assertLess(prior_capture, mutation) + for expected in ( + "requires exactly one prior API process for transactional rollback", + "requires exactly one prior cloudflared process for transactional rollback", + "$freshPriorApiIdentity = Wait-ProcessIdentity", + "$freshPriorCloudflaredIdentity = Wait-ProcessIdentity", + "$freshEnvironmentSnapshot = Save-ManagedEnvironment", + "Restore-PriorPublicRuntime `", + "Start-PinnedPriorProcess `", + "Restored prior API identity drift", + "Restored prior cloudflared identity drift", + "$freshPromotionCommitted = $true", + ): + with self.subTest(expected=expected): + self.assertIn(expected, PUBLIC_RUNTIME) + + def test_fresh_transaction_has_no_engine_web_sidecar_or_dns_mutation(self) -> None: + contract = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract") : + PUBLIC_RUNTIME.index("$freshMutationStarted = $false") + ] + for expected in ( + "if (-not $SkipEngineRestart)", + "if (-not $SkipWebRestart)", + "if ($RouteCloudflareDns)", + "$CanonicalPublicHealthUrl", + ): + with self.subTest(expected=expected): + self.assertIn(expected, contract) + + api_mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"') + preflight = PUBLIC_RUNTIME[:api_mutation] + self.assertIn("requires exact healthy voice sidecars as an unchanged precondition", preflight) + self.assertIn("-Uri $CanonicalPublicHealthUrl", preflight) + self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", preflight) + self.assertIn("$health.engine -eq $true", preflight) + + stt_start = PUBLIC_RUNTIME.index("& $WhisperStartScript `") + tts_start = PUBLIC_RUNTIME.index("& $MeloTtsStartScript `") + self.assertLess( + PUBLIC_RUNTIME.index("Fresh public promotion will not mutate local_whisper"), + stt_start, + ) + self.assertLess( + PUBLIC_RUNTIME.index("Fresh public promotion will not mutate MeloTTS"), + tts_start, + ) + + def test_success_and_rollback_require_canonical_public_engine_and_voice(self) -> None: + rollback = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") : + PUBLIC_RUNTIME.index("function Stop-NodeByPortHint") + ] + self.assertGreaterEqual(rollback.count("$health.engine -eq $true"), 2) + self.assertIn("Test-VoiceSidecarReady", rollback) + self.assertIn("Test-VoiceHealthContract", rollback) + + success_gate = PUBLIC_RUNTIME.index('$freshFailureStage = "public_health_validation"') + receipt = PUBLIC_RUNTIME.index("$provenance = [ordered]@{") + committed = PUBLIC_RUNTIME.index("$freshPromotionCommitted = $true") + self.assertLess(success_gate, receipt) + self.assertLess(receipt, committed) + success_section = PUBLIC_RUNTIME[success_gate:receipt] + self.assertIn("-Uri $CanonicalPublicHealthUrl", success_section) + self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", success_section) + self.assertIn("-Uri $CanonicalPublicOpenApiUrl", success_section) + self.assertIn("Test-RequiredOpenApiPaths", success_section) + self.assertIn('"/admin/voice-runtime"', PUBLIC_RUNTIME) + self.assertIn("$health.engine -eq $true", success_section) + self.assertIn("Test-VoiceSidecarReady", success_section) + + def test_failed_cutover_emits_metadata_only_rollback_evidence(self) -> None: + failure_writer = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Write-FailedFreshPromotionEvidence") : + PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract") + ] + for expected in ( + 'schema_version = "vignette.public-runtime-launch-failure.v1"', + '"failed_rolled_back"', + '"failed_rollback"', + "failure_stage = $FailureStage", + "attempted = $true", + "succeeded = $RollbackSucceeded", + "Write-Utf8TextAtomically", + ): + with self.subTest(expected=expected): + self.assertIn(expected, failure_writer) + self.assertNotIn("command_line", failure_writer) + self.assertNotIn("executable_path", failure_writer) + def test_receipt_is_direct_input_for_windows_topology_capture(self) -> None: receipt = PUBLIC_RUNTIME.index( 'schema_version = "vignette.public-runtime-launch-provenance.v1"' diff --git a/scripts/watch-public-runtime.ps1 b/scripts/watch-public-runtime.ps1 index bd6fe96..48e2a70 100644 --- a/scripts/watch-public-runtime.ps1 +++ b/scripts/watch-public-runtime.ps1 @@ -16,6 +16,8 @@ [int]$ApiPort = 8001, [int]$WebPort = 5174, [int]$EnginePort = 9099, + [int]$WhisperPort = 9882, + [int]$MeloTtsPort = 9883, [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", @@ -36,6 +38,7 @@ $ErrorActionPreference = "Stop" $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" +$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" function Invoke-GitText { param([string[]]$Arguments) @@ -54,6 +57,9 @@ function Assert-StableSourceProvenance { if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) { throw "Pinned start script not found at $startScript" } + if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) { + throw "Pinned voice sidecar probe not found at $voiceSidecarProbe" + } $runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path if (-not [string]::Equals( @@ -99,7 +105,8 @@ function Assert-StableSourceProvenance { foreach ($relativePath in @( "scripts/watch-public-runtime.ps1", - "scripts/start-public-runtime.ps1" + "scripts/start-public-runtime.ps1", + "scripts/probe-public-voice-sidecars.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -203,6 +210,47 @@ function Get-EngineHeaders { return $null } +function Test-VoiceApiReady { + param([object]$Health) + + return ( + $null -ne $Health -and + $Health.status -eq "ok" -and + $Health.available -eq $true -and + $Health.stt_available -eq $true -and + $Health.tts_available -eq $true -and + $Health.stt_provider -eq "local_whisper" -and + $Health.stt_model -eq "small" -and + $Health.tts_provider -eq "melotts" -and + $Health.tts_model -eq "melotts-korean" -and + $Health.limits.uvicorn_ws_max_queue -eq 4 + ) +} + +function Test-VoiceSidecarStack { + $probeArgs = @( + "-X", "utf8", "-B", $voiceSidecarProbe, + "--component", "all", + "--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen", + "--stt-provider", "local_whisper", + "--stt-model", "small", + "--stt-language", "ko", + "--stt-device", "cpu", + "--tts-url", "http://127.0.0.1:$MeloTtsPort", + "--tts-provider", "melotts", + "--tts-model", "melotts-korean", + "--tts-language", "KR", + "--timeout-seconds", "5" + ) + $output = @(& $Python @probeArgs 2>$null) + $probeExit = $LASTEXITCODE + return [pscustomobject]@{ + Name = "voice-sidecars" + Ok = $probeExit -eq 0 + Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" } + } +} + function Test-CloudflaredProcess { if ($SkipCloudflaredRestart) { return [pscustomobject]@{ @@ -245,10 +293,16 @@ $checks = @( -Uri "http://127.0.0.1:$ApiPort/health" ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` -TimeoutSec 60), + (Test-JsonHealth ` + -Name "voice-api" ` + -Uri "http://127.0.0.1:$ApiPort/voice/health" ` + -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` + -TimeoutSec 30), (Test-JsonHealth ` -Name "web-preview" ` -Uri "http://127.0.0.1:$WebPort/" ` -IsHealthy { param($body) $true }), + (Test-VoiceSidecarStack), (Test-CloudflaredProcess) ) @@ -294,6 +348,8 @@ $startArgs = @{ ApiPort = $ApiPort WebPort = $WebPort EnginePort = $EnginePort + WhisperPort = $WhisperPort + MeloTtsPort = $MeloTtsPort Python = $Python Cloudflared = $Cloudflared CloudflaredConfig = $CloudflaredConfig @@ -324,6 +380,15 @@ if (!$apiAfter.Ok) { throw "Public API still unhealthy after restart: $($apiAfter.Detail)" } +$voiceApiAfter = Test-JsonHealth ` + -Name "voice-api" ` + -Uri "http://127.0.0.1:$ApiPort/voice/health" ` + -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` + -TimeoutSec 30 +if (!$voiceApiAfter.Ok) { + throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)" +} + $webAfter = Test-JsonHealth ` -Name "web-preview" ` -Uri "http://127.0.0.1:$WebPort/" ` @@ -343,6 +408,11 @@ if (!$engineAfter.Ok) { throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)" } +$voiceSidecarsAfter = Test-VoiceSidecarStack +if (!$voiceSidecarsAfter.Ok) { + throw "Exact local voice sidecars still unhealthy after restart" +} + if (!$SkipPublicHealth) { $publicAfter = Test-JsonHealth ` -Name "public-api" `