공개 런타임 롤백 경계 강화
This commit is contained in:
Yun Chan
parent
ff3c79dfc2
commit
aaebe4450e
7 changed files with 1271 additions and 30 deletions
|
|
@ -28,6 +28,10 @@ param(
|
|||
[int]$DbTimeoutSec = 90,
|
||||
[int]$DbPort = 55432,
|
||||
[int]$ApiPort = 8001,
|
||||
[int]$EnginePort = 9099,
|
||||
[int]$WhisperPort = 9882,
|
||||
[int]$MeloTtsPort = 9883,
|
||||
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
||||
[string]$BootLog = ""
|
||||
)
|
||||
|
||||
|
|
@ -36,6 +40,7 @@ $ErrorActionPreference = "Stop"
|
|||
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
||||
$expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
|
||||
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
|
||||
function Invoke-GitText {
|
||||
param([string[]]$Arguments)
|
||||
|
|
@ -48,7 +53,7 @@ function Invoke-GitText {
|
|||
}
|
||||
|
||||
function Assert-StableSourceProvenance {
|
||||
foreach ($requiredScript in @($expectedBootScript, $startScript)) {
|
||||
foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) {
|
||||
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
|
||||
throw "Pinned public runtime script not found at $requiredScript"
|
||||
}
|
||||
|
|
@ -97,7 +102,8 @@ function Assert-StableSourceProvenance {
|
|||
}
|
||||
foreach ($relativePath in @(
|
||||
"scripts/boot-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1"
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
@ -159,7 +165,7 @@ function Test-ApiControlPlaneHealthy {
|
|||
$body = $reader.ReadToEnd()
|
||||
$reader.Close(); $resp.Close()
|
||||
$h = $body | ConvertFrom-Json
|
||||
if ($h.environment -eq "prod" -and $h.db -eq $true) { return $true }
|
||||
if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true }
|
||||
Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body)
|
||||
return $false
|
||||
} catch {
|
||||
|
|
@ -172,13 +178,72 @@ function Test-ApiControlPlaneHealthy {
|
|||
|
||||
function Test-EngineHealthy {
|
||||
try {
|
||||
$response = Invoke-RestMethod -Uri "http://127.0.0.1:9099/health" -TimeoutSec 10
|
||||
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10
|
||||
return $response.ok -eq $true
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Test-VoiceApiReady {
|
||||
param([object]$Health)
|
||||
|
||||
return (
|
||||
$null -ne $Health -and
|
||||
$Health.status -eq "ok" -and
|
||||
$Health.available -eq $true -and
|
||||
$Health.stt_available -eq $true -and
|
||||
$Health.tts_available -eq $true -and
|
||||
$Health.stt_provider -eq "local_whisper" -and
|
||||
$Health.stt_model -eq "small" -and
|
||||
$Health.tts_provider -eq "melotts" -and
|
||||
$Health.tts_model -eq "melotts-korean" -and
|
||||
$Health.limits.uvicorn_ws_max_queue -eq 4
|
||||
)
|
||||
}
|
||||
|
||||
function Test-VoiceApiHealthy {
|
||||
for ($i = 1; $i -le 3; $i++) {
|
||||
try {
|
||||
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health")
|
||||
$req.Timeout = 5000
|
||||
$req.ReadWriteTimeout = 5000
|
||||
$req.Proxy = $null
|
||||
$resp = $req.GetResponse()
|
||||
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
|
||||
$body = $reader.ReadToEnd()
|
||||
$reader.Close(); $resp.Close()
|
||||
$health = $body | ConvertFrom-Json
|
||||
if (Test-VoiceApiReady -Health $health) { return $true }
|
||||
Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body)
|
||||
return $false
|
||||
} catch {
|
||||
Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Test-VoiceSidecarStack {
|
||||
$probeArgs = @(
|
||||
"-X", "utf8", "-B", $voiceSidecarProbe,
|
||||
"--component", "all",
|
||||
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
||||
"--stt-provider", "local_whisper",
|
||||
"--stt-model", "small",
|
||||
"--stt-language", "ko",
|
||||
"--stt-device", "cpu",
|
||||
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
||||
"--tts-provider", "melotts",
|
||||
"--tts-model", "melotts-korean",
|
||||
"--tts-language", "KR",
|
||||
"--timeout-seconds", "5"
|
||||
)
|
||||
& $Python @probeArgs 1>$null 2>$null
|
||||
return $LASTEXITCODE -eq 0
|
||||
}
|
||||
|
||||
Write-BootLog "================ boot start ================"
|
||||
|
||||
# 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기)
|
||||
|
|
@ -217,12 +282,16 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) {
|
|||
Write-BootLog "postgres 127.0.0.1:$DbPort up"
|
||||
|
||||
# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지)
|
||||
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
|
||||
Write-BootLog "control plane and engine already healthy; skipping runtime restart"
|
||||
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
||||
Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart"
|
||||
} else {
|
||||
Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart"
|
||||
$out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
|
||||
-Workspace $resolvedSourceRoot `
|
||||
-Python $Python `
|
||||
-EnginePort $EnginePort `
|
||||
-WhisperPort $WhisperPort `
|
||||
-MeloTtsPort $MeloTtsPort `
|
||||
-SkipWebRestart 2>&1
|
||||
$out | ForEach-Object { Write-BootLog (" pub> " + $_) }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
|
|
@ -233,12 +302,13 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
|
|||
|
||||
# 4) 최종 확인
|
||||
if (Test-ApiControlPlaneHealthy) {
|
||||
if (Test-EngineHealthy) {
|
||||
Write-BootLog "boot OK: control plane and engine healthy"
|
||||
} else {
|
||||
Write-BootLog "boot OK: admin/auth control plane healthy; engine remains degraded"
|
||||
}
|
||||
if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
||||
Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy"
|
||||
exit 0
|
||||
} else {
|
||||
Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded"
|
||||
exit 2
|
||||
}
|
||||
} else {
|
||||
Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log"
|
||||
exit 2
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|||
$installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime-task.ps1"
|
||||
$watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
|
||||
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
|
||||
function Invoke-GitText {
|
||||
param([string[]]$Arguments)
|
||||
|
|
@ -30,7 +31,7 @@ function Invoke-GitText {
|
|||
return (@($value) -join [Environment]::NewLine).Trim()
|
||||
}
|
||||
|
||||
foreach ($requiredScript in @($installerScript, $watchScript, $startScript)) {
|
||||
foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe)) {
|
||||
if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
|
||||
throw "Public runtime script not found at $requiredScript"
|
||||
}
|
||||
|
|
@ -71,7 +72,8 @@ if ($dirty) {
|
|||
foreach ($relativePath in @(
|
||||
"scripts/install-public-runtime-task.ps1",
|
||||
"scripts/watch-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1"
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|||
$RegisterScript = Join-Path $resolvedSourceRoot "scripts\register-boot-task.ps1"
|
||||
$BootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
|
||||
$StartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$VoiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
$UserName = "$env:USERDOMAIN\$env:USERNAME"
|
||||
|
||||
function Invoke-GitText {
|
||||
|
|
@ -31,7 +32,7 @@ function Invoke-GitText {
|
|||
return (@($value) -join [Environment]::NewLine).Trim()
|
||||
}
|
||||
|
||||
foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript)) {
|
||||
foreach ($requiredScript in @($RegisterScript, $BootScript, $StartScript, $VoiceSidecarProbe)) {
|
||||
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
|
||||
throw "Public runtime script not found: $requiredScript"
|
||||
}
|
||||
|
|
@ -72,7 +73,8 @@ if ($dirty) {
|
|||
foreach ($relativePath in @(
|
||||
"scripts/register-boot-task.ps1",
|
||||
"scripts/boot-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1"
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
[switch]$RouteCloudflareDns,
|
||||
[string]$CloudflareTunnelName = "vignette",
|
||||
[switch]$SkipCloudflaredRestart,
|
||||
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
||||
[switch]$RequireFreshPublicProvenance,
|
||||
[string]$ExpectedSourceCommit = "",
|
||||
[string]$ExpectedSourceTree = "",
|
||||
|
|
@ -51,6 +52,15 @@ $WhisperLanguage = "ko"
|
|||
$WhisperDevice = "cpu"
|
||||
$MeloTtsModel = "melotts-korean"
|
||||
$MeloTtsLanguage = "KR"
|
||||
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
|
||||
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
|
||||
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
|
||||
$RequiredPublicApiPaths = @(
|
||||
"/health",
|
||||
"/voice/health",
|
||||
"/voice/speech",
|
||||
"/admin/voice-runtime"
|
||||
)
|
||||
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
|
||||
$PublicWebHostnames = @("vnet.18ka.net")
|
||||
|
||||
|
|
@ -97,6 +107,18 @@ function Test-PortListener {
|
|||
return $null -ne $listener
|
||||
}
|
||||
|
||||
function Get-ListenerProcessIds {
|
||||
param([int]$Port)
|
||||
|
||||
return @(
|
||||
Get-NetTCPConnection `
|
||||
-State Listen `
|
||||
-LocalPort $Port `
|
||||
-ErrorAction SilentlyContinue |
|
||||
Select-Object -ExpandProperty OwningProcess -Unique
|
||||
)
|
||||
}
|
||||
|
||||
function Test-VoiceSidecarReady {
|
||||
param(
|
||||
[ValidateSet("stt", "tts")]
|
||||
|
|
@ -156,6 +178,74 @@ function Test-VoiceApiReady {
|
|||
)
|
||||
}
|
||||
|
||||
function Test-PriorVoiceApiReady {
|
||||
param([object]$Health)
|
||||
|
||||
return (
|
||||
$null -ne $Health -and
|
||||
$Health.status -eq "ok" -and
|
||||
$Health.available -eq $true -and
|
||||
$Health.stt_available -eq $true -and
|
||||
$Health.tts_available -eq $true -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
|
||||
)
|
||||
}
|
||||
|
||||
function ConvertTo-SafeVoiceHealthContract {
|
||||
param([object]$Health)
|
||||
|
||||
return [ordered]@{
|
||||
status = [string]$Health.status
|
||||
available = [bool]$Health.available
|
||||
stt_available = [bool]$Health.stt_available
|
||||
tts_available = [bool]$Health.tts_available
|
||||
stt_provider = [string]$Health.stt_provider
|
||||
stt_model = [string]$Health.stt_model
|
||||
tts_provider = [string]$Health.tts_provider
|
||||
tts_model = [string]$Health.tts_model
|
||||
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
|
||||
}
|
||||
}
|
||||
|
||||
function Test-VoiceHealthContract {
|
||||
param(
|
||||
[object]$Health,
|
||||
[System.Collections.IDictionary]$Expected
|
||||
)
|
||||
|
||||
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
|
||||
return $false
|
||||
}
|
||||
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
|
||||
foreach ($name in $Expected.Keys) {
|
||||
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-RequiredOpenApiPaths {
|
||||
param(
|
||||
[object]$Document,
|
||||
[string[]]$RequiredPaths
|
||||
)
|
||||
|
||||
if ($null -eq $Document -or $null -eq $Document.paths) {
|
||||
return $false
|
||||
}
|
||||
$actualPaths = @($Document.paths.PSObject.Properties.Name)
|
||||
foreach ($requiredPath in $RequiredPaths) {
|
||||
if ($actualPaths -notcontains $requiredPath) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-EngineReady {
|
||||
param(
|
||||
[int]$Port,
|
||||
|
|
@ -247,16 +337,15 @@ function Stop-ProcessesBounded {
|
|||
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
||||
}
|
||||
|
||||
function Stop-UvicornByPort {
|
||||
function Get-UvicornProcessesByPort {
|
||||
param(
|
||||
[string]$AppImport,
|
||||
[int]$Port,
|
||||
[int]$TimeoutSec = 15
|
||||
[int]$Port
|
||||
)
|
||||
|
||||
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
|
||||
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
|
||||
$processes = @(
|
||||
return @(
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object {
|
||||
$_.Name -like "python*" -and
|
||||
|
|
@ -265,6 +354,16 @@ function Stop-UvicornByPort {
|
|||
$_.CommandLine -like "*--port $Port*"
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function Stop-UvicornByPort {
|
||||
param(
|
||||
[string]$AppImport,
|
||||
[int]$Port,
|
||||
[int]$TimeoutSec = 15
|
||||
)
|
||||
|
||||
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
|
||||
return @(
|
||||
Stop-ProcessesBounded `
|
||||
-Processes $processes `
|
||||
|
|
@ -298,7 +397,7 @@ function Wait-ProcessIdentity {
|
|||
param(
|
||||
[int]$ProcessId,
|
||||
[string]$Role,
|
||||
[string]$ExpectedCwd,
|
||||
[string]$ExpectedCwd = "",
|
||||
[int]$TimeoutSec = 15
|
||||
)
|
||||
|
||||
|
|
@ -314,20 +413,26 @@ function Wait-ProcessIdentity {
|
|||
) {
|
||||
$identityProbeArgs = @(
|
||||
"-X", "utf8", "-c",
|
||||
"import hashlib,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(p.cmdline()).encode('utf-8', errors='strict')).hexdigest())",
|
||||
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
|
||||
"$ProcessId"
|
||||
)
|
||||
$identityProbe = @(& $Python @identityProbeArgs)
|
||||
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 3) {
|
||||
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
|
||||
throw "Could not prove $Role psutil identity for PID $ProcessId"
|
||||
}
|
||||
$actualCwd = $identityProbe[0].Trim()
|
||||
$startedAtUtc = $identityProbe[1].Trim()
|
||||
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
|
||||
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
|
||||
$environmentObject = $identityProbe[4] | ConvertFrom-Json
|
||||
$processEnvironment = [ordered]@{}
|
||||
foreach ($property in $environmentObject.PSObject.Properties) {
|
||||
$processEnvironment[$property.Name] = [string]$property.Value
|
||||
}
|
||||
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
||||
throw "$Role psutil identity is incomplete for PID $ProcessId"
|
||||
}
|
||||
if (-not [string]::Equals(
|
||||
if ($ExpectedCwd -and -not [string]::Equals(
|
||||
[System.IO.Path]::GetFullPath($actualCwd),
|
||||
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
|
|
@ -343,6 +448,8 @@ function Wait-ProcessIdentity {
|
|||
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
command_line = $process.CommandLine
|
||||
command_line_sha256 = $commandLineSha256
|
||||
argument_list = $argumentList
|
||||
environment = $processEnvironment
|
||||
cwd = $actualCwd
|
||||
}
|
||||
}
|
||||
|
|
@ -367,6 +474,274 @@ function ConvertTo-SafeProcessIdentity {
|
|||
}
|
||||
}
|
||||
|
||||
function Save-ManagedEnvironment {
|
||||
param([string[]]$Names)
|
||||
|
||||
$snapshot = [ordered]@{}
|
||||
foreach ($name in $Names) {
|
||||
$value = [System.Environment]::GetEnvironmentVariable(
|
||||
$name,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
$snapshot[$name] = [ordered]@{
|
||||
present = $null -ne $value
|
||||
value = $value
|
||||
}
|
||||
}
|
||||
return $snapshot
|
||||
}
|
||||
|
||||
function Restore-ManagedEnvironment {
|
||||
param([System.Collections.IDictionary]$Snapshot)
|
||||
|
||||
foreach ($name in $Snapshot.Keys) {
|
||||
$entry = $Snapshot[$name]
|
||||
if ($entry.present) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
[string]$entry.value,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
} else {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
$null,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Save-CompleteProcessEnvironment {
|
||||
$snapshot = [ordered]@{}
|
||||
$environment = [System.Environment]::GetEnvironmentVariables(
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
foreach ($name in $environment.Keys) {
|
||||
$snapshot[[string]$name] = [string]$environment[$name]
|
||||
}
|
||||
return $snapshot
|
||||
}
|
||||
|
||||
function Set-CompleteProcessEnvironment {
|
||||
param([System.Collections.IDictionary]$Environment)
|
||||
|
||||
$current = [System.Environment]::GetEnvironmentVariables(
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
foreach ($name in @($current.Keys)) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
[string]$name,
|
||||
$null,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
foreach ($name in $Environment.Keys) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
[string]$name,
|
||||
[string]$Environment[$name],
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertTo-WindowsCommandLineArgument {
|
||||
param([AllowEmptyString()][string]$Argument)
|
||||
|
||||
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
||||
return $Argument
|
||||
}
|
||||
|
||||
$builder = New-Object System.Text.StringBuilder
|
||||
$null = $builder.Append('"')
|
||||
$backslashes = 0
|
||||
foreach ($character in $Argument.ToCharArray()) {
|
||||
if ($character -eq '\') {
|
||||
$backslashes++
|
||||
continue
|
||||
}
|
||||
if ($character -eq '"') {
|
||||
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
||||
$null = $builder.Append('"')
|
||||
$backslashes = 0
|
||||
continue
|
||||
}
|
||||
if ($backslashes -gt 0) {
|
||||
$null = $builder.Append(('\' * $backslashes))
|
||||
$backslashes = 0
|
||||
}
|
||||
$null = $builder.Append($character)
|
||||
}
|
||||
if ($backslashes -gt 0) {
|
||||
$null = $builder.Append(('\' * ($backslashes * 2)))
|
||||
}
|
||||
$null = $builder.Append('"')
|
||||
return $builder.ToString()
|
||||
}
|
||||
|
||||
function Join-WindowsArgumentList {
|
||||
param([object[]]$ArgumentList)
|
||||
|
||||
return (@(
|
||||
foreach ($argument in $ArgumentList) {
|
||||
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
||||
}
|
||||
) -join ' ')
|
||||
}
|
||||
|
||||
function Start-PinnedPriorProcess {
|
||||
param(
|
||||
[System.Collections.IDictionary]$Identity,
|
||||
[string]$Role,
|
||||
[string]$StdoutLog,
|
||||
[string]$StderrLog
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
|
||||
throw "Prior $Role executable is unavailable"
|
||||
}
|
||||
$actualExecutableSha256 = (
|
||||
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
|
||||
throw "Prior $Role executable SHA256 drift"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
||||
throw "Prior $Role working directory is unavailable"
|
||||
}
|
||||
if (@($Identity.argument_list).Count -eq 0) {
|
||||
throw "Prior $Role argument list is unavailable"
|
||||
}
|
||||
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
|
||||
throw "Prior $Role environment is unavailable"
|
||||
}
|
||||
|
||||
$callerEnvironment = Save-CompleteProcessEnvironment
|
||||
try {
|
||||
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
||||
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
||||
return Start-Process -WindowStyle Hidden `
|
||||
-FilePath $Identity.executable_path `
|
||||
-ArgumentList $argumentString `
|
||||
-WorkingDirectory $Identity.cwd `
|
||||
-RedirectStandardOutput $StdoutLog `
|
||||
-RedirectStandardError $StderrLog `
|
||||
-PassThru
|
||||
} finally {
|
||||
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
||||
}
|
||||
}
|
||||
|
||||
function Restore-PriorPublicRuntime {
|
||||
param(
|
||||
[System.Collections.IDictionary]$PriorApi,
|
||||
[System.Collections.IDictionary]$PriorCloudflared,
|
||||
[System.Collections.IDictionary]$PriorLocalVoiceContract,
|
||||
[System.Collections.IDictionary]$PriorPublicVoiceContract,
|
||||
[System.Collections.IDictionary]$EnvironmentSnapshot,
|
||||
[string]$ConfigPath,
|
||||
[int]$ApiPortValue,
|
||||
[string]$HealthUrl,
|
||||
[string]$VoiceHealthUrl,
|
||||
[int]$TimeoutSec
|
||||
)
|
||||
|
||||
$null = @(
|
||||
Stop-UvicornByPort `
|
||||
-AppImport "app.main:app" `
|
||||
-Port $ApiPortValue `
|
||||
-TimeoutSec $TimeoutSec
|
||||
)
|
||||
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
|
||||
$priorApiProcess = Start-PinnedPriorProcess `
|
||||
-Identity $PriorApi `
|
||||
-Role "api" `
|
||||
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
|
||||
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
|
||||
$priorApiIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorApiProcess.Id `
|
||||
-Role "restored prior api" `
|
||||
-ExpectedCwd $PriorApi.cwd `
|
||||
-TimeoutSec $TimeoutSec
|
||||
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
||||
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
|
||||
throw "Restored prior API identity drift: $field"
|
||||
}
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPortValue/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Restored prior runtime does not have the exact local voice sidecars"
|
||||
}
|
||||
|
||||
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
|
||||
$currentCloudflared = @(
|
||||
Get-CloudflaredProcessesForConfig `
|
||||
-ConfigPath $resolvedConfigPath `
|
||||
-ExactPath
|
||||
)
|
||||
$null = @(
|
||||
Stop-ProcessesBounded `
|
||||
-Processes $currentCloudflared `
|
||||
-TimeoutSec $TimeoutSec `
|
||||
-Role "failed fresh cloudflared"
|
||||
)
|
||||
$priorCloudflaredProcess = Start-PinnedPriorProcess `
|
||||
-Identity $PriorCloudflared `
|
||||
-Role "cloudflared" `
|
||||
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
|
||||
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
|
||||
$priorCloudflaredIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorCloudflaredProcess.Id `
|
||||
-Role "restored prior cloudflared" `
|
||||
-ExpectedCwd $PriorCloudflared.cwd `
|
||||
-TimeoutSec $TimeoutSec
|
||||
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
||||
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
|
||||
throw "Restored prior cloudflared identity drift: $field"
|
||||
}
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $HealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $VoiceHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
|
||||
return [ordered]@{
|
||||
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
||||
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
|
||||
local_health = $true
|
||||
local_voice_health = $true
|
||||
public_health = $true
|
||||
public_voice_health = $true
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-NodeByPortHint {
|
||||
param([int]$Port)
|
||||
|
||||
|
|
@ -515,6 +890,39 @@ function Write-Utf8TextAtomically {
|
|||
}
|
||||
}
|
||||
|
||||
function Write-FailedFreshPromotionEvidence {
|
||||
param(
|
||||
[string]$OutputPath,
|
||||
[string]$FailureStage,
|
||||
[bool]$RollbackSucceeded,
|
||||
[object]$RollbackResult,
|
||||
[string]$SourceCommit,
|
||||
[string]$SourceTree
|
||||
)
|
||||
|
||||
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
|
||||
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
|
||||
$failedPath = "$OutputPath.failed.log"
|
||||
$payload = [ordered]@{
|
||||
schema_version = "vignette.public-runtime-launch-failure.v1"
|
||||
status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" }
|
||||
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
||||
failure_stage = $FailureStage
|
||||
source = [ordered]@{
|
||||
git_commit = $SourceCommit.ToLowerInvariant()
|
||||
git_tree = $SourceTree.ToLowerInvariant()
|
||||
}
|
||||
rollback = [ordered]@{
|
||||
attempted = $true
|
||||
succeeded = $RollbackSucceeded
|
||||
result = $RollbackResult
|
||||
}
|
||||
}
|
||||
$json = ConvertTo-Json -InputObject $payload -Depth 8
|
||||
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
|
||||
return $failedPath
|
||||
}
|
||||
|
||||
function Assert-FreshPublicProvenanceContract {
|
||||
param(
|
||||
[string]$SourceRoot,
|
||||
|
|
@ -534,6 +942,22 @@ function Assert-FreshPublicProvenanceContract {
|
|||
if ($SkipCloudflaredRestart) {
|
||||
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
|
||||
}
|
||||
if (-not $SkipEngineRestart) {
|
||||
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
|
||||
}
|
||||
if (-not $SkipWebRestart) {
|
||||
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
|
||||
}
|
||||
if ($RouteCloudflareDns) {
|
||||
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
|
||||
}
|
||||
if (-not [string]::Equals(
|
||||
$PublicHealthUrl,
|
||||
$CanonicalPublicHealthUrl,
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
)) {
|
||||
throw "Fresh public promotion requires the canonical HTTPS public health URL"
|
||||
}
|
||||
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
||||
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
|
||||
throw "Fresh public provenance requires exact source commit and tree pins"
|
||||
|
|
@ -657,6 +1081,85 @@ function Set-CloudflaredIngress {
|
|||
}
|
||||
}
|
||||
|
||||
$freshMutationStarted = $false
|
||||
$freshPromotionCommitted = $false
|
||||
$freshFailureStage = "preflight"
|
||||
$freshPriorApiIdentity = $null
|
||||
$freshPriorCloudflaredIdentity = $null
|
||||
$freshPriorLocalVoiceContract = $null
|
||||
$freshPriorPublicVoiceContract = $null
|
||||
$freshEnvironmentSnapshot = $null
|
||||
$resolvedRuntimeProvenancePath = $null
|
||||
$freshManagedEnvironmentNames = @(
|
||||
"ENVIRONMENT",
|
||||
"ENGINE_URL",
|
||||
"ENGINE_MODE",
|
||||
"VIGNETTE_LIVE_CLIENT_PROVIDER",
|
||||
"AUTH_DEV_LOGIN_ENABLED",
|
||||
"AUTO_SEED_PERSONAS",
|
||||
"ALLOW_SEED_PERSONA_FALLBACK",
|
||||
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
|
||||
"VIGNETTE_VOICE_STT_PROVIDER",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_URL",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
|
||||
"VIGNETTE_VOICE_TTS_PROVIDER",
|
||||
"VIGNETTE_MELOTTS_TTS_URL",
|
||||
"FRONTEND_BASE_URL",
|
||||
"CORS_ORIGINS",
|
||||
"FRONTEND_ORIGIN_MAP"
|
||||
)
|
||||
|
||||
trap {
|
||||
$caught = $_
|
||||
if (
|
||||
$RequireFreshPublicProvenance -and
|
||||
$freshMutationStarted -and
|
||||
-not $freshPromotionCommitted
|
||||
) {
|
||||
$rollbackResult = $null
|
||||
$rollbackSucceeded = $false
|
||||
$rollbackFailureType = "none"
|
||||
try {
|
||||
$rollbackResult = Restore-PriorPublicRuntime `
|
||||
-PriorApi $freshPriorApiIdentity `
|
||||
-PriorCloudflared $freshPriorCloudflaredIdentity `
|
||||
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
|
||||
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
|
||||
-EnvironmentSnapshot $freshEnvironmentSnapshot `
|
||||
-ConfigPath $CloudflaredConfig `
|
||||
-ApiPortValue $ApiPort `
|
||||
-HealthUrl $PublicHealthUrl `
|
||||
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$rollbackSucceeded = $true
|
||||
} catch {
|
||||
$rollbackFailureType = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$failedEvidencePath = ""
|
||||
if ($resolvedRuntimeProvenancePath) {
|
||||
try {
|
||||
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
|
||||
-OutputPath $resolvedRuntimeProvenancePath `
|
||||
-FailureStage $freshFailureStage `
|
||||
-RollbackSucceeded $rollbackSucceeded `
|
||||
-RollbackResult $rollbackResult `
|
||||
-SourceCommit $ExpectedSourceCommit `
|
||||
-SourceTree $ExpectedSourceTree
|
||||
} catch {
|
||||
$failedEvidencePath = "unavailable"
|
||||
}
|
||||
}
|
||||
|
||||
if ($rollbackSucceeded) {
|
||||
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
||||
}
|
||||
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
||||
}
|
||||
throw $caught
|
||||
}
|
||||
|
||||
if (!(Test-Path $Python)) {
|
||||
throw "Python 3.11 not found at $Python"
|
||||
}
|
||||
|
|
@ -705,6 +1208,64 @@ if ($RequireFreshPublicProvenance) {
|
|||
-ApiPortValue $ApiPort `
|
||||
-WebPortValue $WebPort `
|
||||
-RequireUnchanged
|
||||
|
||||
$priorApiProcesses = @(
|
||||
Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort
|
||||
)
|
||||
if ($priorApiProcesses.Count -ne 1) {
|
||||
throw "Fresh public promotion requires exactly one prior API process for transactional rollback"
|
||||
}
|
||||
$priorCloudflaredProcesses = @(
|
||||
Get-CloudflaredProcessesForConfig `
|
||||
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
|
||||
-ExactPath
|
||||
)
|
||||
if ($priorCloudflaredProcesses.Count -ne 1) {
|
||||
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
|
||||
}
|
||||
$freshPriorApiIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorApiProcesses[0].ProcessId `
|
||||
-Role "prior api" `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
|
||||
-Role "prior cloudflared" `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
$priorLocalVoiceHealth = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
||||
-Health $priorLocalVoiceHealth
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
$priorPublicVoiceHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicVoiceHealthUrl `
|
||||
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
||||
-Health $priorPublicVoiceHealth
|
||||
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
|
||||
-Names $freshManagedEnvironmentNames
|
||||
}
|
||||
|
||||
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
||||
|
|
@ -732,7 +1293,9 @@ if ($SkipEngineRestart) {
|
|||
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
|
||||
if (Test-Path $logFile) {
|
||||
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak" -Force -ErrorAction SilentlyContinue
|
||||
# stable detached source의 provenance gate는 untracked 파일도 차단한다.
|
||||
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
|
||||
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -781,6 +1344,9 @@ $env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
|
|||
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
|
||||
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying"
|
||||
}
|
||||
if (Test-PortListener -Port $WhisperPort) {
|
||||
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
|
||||
}
|
||||
|
|
@ -795,6 +1361,9 @@ if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
|||
}
|
||||
|
||||
if (-not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
|
||||
}
|
||||
if (Test-PortListener -Port $MeloTtsPort) {
|
||||
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
|
||||
}
|
||||
|
|
@ -828,6 +1397,10 @@ $apiLaunchIdentity = $null
|
|||
if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
||||
Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
|
||||
} else {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "api_cutover"
|
||||
$freshMutationStarted = $true
|
||||
}
|
||||
$apiStoppedProcessIds = @(
|
||||
Stop-UvicornByPort `
|
||||
-AppImport "app.main:app" `
|
||||
|
|
@ -890,6 +1463,9 @@ if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
|
|||
}
|
||||
|
||||
if (!$SkipWebRestart) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "web_preview"
|
||||
}
|
||||
Stop-NodeByPortHint -Port $WebPort
|
||||
|
||||
$build = Start-Process -FilePath "cmd.exe" `
|
||||
|
|
@ -916,6 +1492,9 @@ $cloudflaredProcess = $null
|
|||
$cloudflaredLaunchIdentity = $null
|
||||
$cloudflaredStoppedProcessIds = @()
|
||||
if (!$SkipCloudflaredRestart) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "cloudflared_cutover"
|
||||
}
|
||||
if (!(Test-Path $Cloudflared)) {
|
||||
throw "cloudflared not found at $Cloudflared"
|
||||
}
|
||||
|
|
@ -986,6 +1565,7 @@ if (!$SkipCloudflaredRestart) {
|
|||
}
|
||||
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "identity_revalidation"
|
||||
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
|
||||
throw "Fresh public promotion did not produce both API and cloudflared identities"
|
||||
}
|
||||
|
|
@ -1015,6 +1595,32 @@ if ($RequireFreshPublicProvenance) {
|
|||
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
|
||||
throw "Pinned cloudflared config drifted before provenance receipt"
|
||||
}
|
||||
$freshFailureStage = "public_health_validation"
|
||||
$publicHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$publicVoiceHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicVoiceHealthUrl `
|
||||
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$publicOpenApi = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicOpenApiUrl `
|
||||
-IsHealthy {
|
||||
param($document)
|
||||
Test-RequiredOpenApiPaths `
|
||||
-Document $document `
|
||||
-RequiredPaths $RequiredPublicApiPaths
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Exact local voice sidecars changed before provenance receipt"
|
||||
}
|
||||
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
|
||||
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
|
||||
|
|
@ -1038,6 +1644,16 @@ if ($RequireFreshPublicProvenance) {
|
|||
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
||||
sha256 = $finalConfigSha256
|
||||
}
|
||||
public_validation = [ordered]@{
|
||||
health_url = $CanonicalPublicHealthUrl
|
||||
health = $true
|
||||
voice_health_url = $CanonicalPublicVoiceHealthUrl
|
||||
voice_health = $true
|
||||
openapi_url = $CanonicalPublicOpenApiUrl
|
||||
required_openapi_paths = @($RequiredPublicApiPaths)
|
||||
openapi = $true
|
||||
local_voice_sidecars = $true
|
||||
}
|
||||
replacement = [ordered]@{
|
||||
api_stopped_pids = @($apiStoppedProcessIds)
|
||||
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
|
||||
|
|
@ -1068,6 +1684,7 @@ if ($RequireFreshPublicProvenance) {
|
|||
}
|
||||
}
|
||||
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
|
||||
$freshFailureStage = "receipt_publish"
|
||||
try {
|
||||
Write-Utf8TextAtomically `
|
||||
-OutputPath $resolvedRuntimeProvenancePath `
|
||||
|
|
@ -1077,6 +1694,7 @@ if ($RequireFreshPublicProvenance) {
|
|||
# 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다.
|
||||
throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)"
|
||||
}
|
||||
$freshPromotionCommitted = $true
|
||||
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ BOOT = SCRIPTS / "boot-public-runtime.ps1"
|
|||
BOOT_REGISTER = SCRIPTS / "register-boot-task.ps1"
|
||||
HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs"
|
||||
START = SCRIPTS / "start-public-runtime.ps1"
|
||||
VOICE_PROBE = SCRIPTS / "probe-public-voice-sidecars.py"
|
||||
REPO_ROOT = SCRIPTS.parent
|
||||
RUNBOOK = REPO_ROOT / "docs" / "ops" / "public-runtime-watchdog.md"
|
||||
LOCAL_DEVELOPMENT = REPO_ROOT / "docs" / "guides" / "local-development.md"
|
||||
|
|
@ -117,6 +118,97 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
|
|||
task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask")
|
||||
self.assertLess(dirty_gate, task_registration)
|
||||
|
||||
def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None:
|
||||
for source in (WATCHDOG_SOURCE, BOOT_SOURCE):
|
||||
for expected in (
|
||||
'"--component", "all"',
|
||||
'"--stt-provider", "local_whisper"',
|
||||
'"--stt-model", "small"',
|
||||
'"--stt-device", "cpu"',
|
||||
'"--tts-provider", "melotts"',
|
||||
'"--tts-model", "melotts-korean"',
|
||||
'"scripts\\probe-public-voice-sidecars.py"',
|
||||
"Test-VoiceSidecarStack",
|
||||
):
|
||||
with self.subTest(source=source[:32], expected=expected):
|
||||
self.assertIn(expected, source)
|
||||
self.assertIn("(Test-VoiceSidecarStack)", WATCHDOG_SOURCE)
|
||||
self.assertIn("$voiceSidecarsAfter = Test-VoiceSidecarStack", WATCHDOG_SOURCE)
|
||||
self.assertIn('-Name "voice-api"', WATCHDOG_SOURCE)
|
||||
self.assertIn("$voiceApiAfter = Test-JsonHealth", WATCHDOG_SOURCE)
|
||||
self.assertIn(
|
||||
"(Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)",
|
||||
BOOT_SOURCE,
|
||||
)
|
||||
self.assertGreaterEqual(BOOT_SOURCE.count("Test-VoiceApiHealthy"), 3)
|
||||
self.assertIn("-WhisperPort $WhisperPort", BOOT_SOURCE)
|
||||
self.assertIn("-MeloTtsPort $MeloTtsPort", BOOT_SOURCE)
|
||||
self.assertIn("scripts/probe-public-voice-sidecars.py", INSTALLER_SOURCE)
|
||||
self.assertIn("scripts/probe-public-voice-sidecars.py", BOOT_REGISTER_SOURCE)
|
||||
|
||||
def test_old_openai_api_is_not_healthy_when_exact_sidecars_are_ready(self) -> None:
|
||||
powershell = shutil.which("powershell.exe")
|
||||
if powershell is None:
|
||||
self.skipTest("Windows PowerShell 5.1 is not available")
|
||||
|
||||
watchdog_contract = WATCHDOG_SOURCE[
|
||||
WATCHDOG_SOURCE.index("function Test-VoiceApiReady") :
|
||||
WATCHDOG_SOURCE.index("function Test-VoiceSidecarStack")
|
||||
].strip()
|
||||
boot_contract = BOOT_SOURCE[
|
||||
BOOT_SOURCE.index("function Test-VoiceApiReady") :
|
||||
BOOT_SOURCE.index("function Test-VoiceApiHealthy")
|
||||
].strip()
|
||||
self.assertEqual(watchdog_contract, boot_contract)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
harness = Path(temporary_directory) / "voice-api-contract.ps1"
|
||||
harness.write_text(
|
||||
watchdog_contract
|
||||
+ r'''
|
||||
$openAi = [pscustomobject]@{
|
||||
status='ok';available=$true;stt_available=$true;tts_available=$true
|
||||
stt_provider='openai';stt_model='gpt-4o-mini-transcribe'
|
||||
tts_provider='openai';tts_model='gpt-4o-mini-tts'
|
||||
limits=[pscustomobject]@{uvicorn_ws_max_queue=4}
|
||||
}
|
||||
if (Test-VoiceApiReady -Health $openAi) { throw 'old OpenAI API was accepted' }
|
||||
$local = [pscustomobject]@{
|
||||
status='ok';available=$true;stt_available=$true;tts_available=$true
|
||||
stt_provider='local_whisper';stt_model='small'
|
||||
tts_provider='melotts';tts_model='melotts-korean'
|
||||
limits=[pscustomobject]@{uvicorn_ws_max_queue=4}
|
||||
}
|
||||
if (-not (Test-VoiceApiReady -Health $local)) { throw 'exact local API was rejected' }
|
||||
$local.limits.uvicorn_ws_max_queue = 5
|
||||
if (Test-VoiceApiReady -Health $local) { throw 'wrong websocket queue was accepted' }
|
||||
''',
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[
|
||||
powershell,
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(harness),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode,
|
||||
0,
|
||||
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
||||
)
|
||||
|
||||
def test_hidden_trigger_never_executes_a_workspace_script_directly(self) -> None:
|
||||
for expected in (
|
||||
"VignettePublicRuntimeWatchdog",
|
||||
|
|
@ -178,11 +270,13 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
|
|||
copied_boot = scripts / BOOT.name
|
||||
copied_boot_register = scripts / BOOT_REGISTER.name
|
||||
copied_start = scripts / START.name
|
||||
copied_voice_probe = scripts / VOICE_PROBE.name
|
||||
shutil.copy2(WATCHDOG, copied_watchdog)
|
||||
shutil.copy2(INSTALLER, copied_installer)
|
||||
shutil.copy2(BOOT, copied_boot)
|
||||
shutil.copy2(BOOT_REGISTER, copied_boot_register)
|
||||
shutil.copy2(START, copied_start)
|
||||
shutil.copy2(VOICE_PROBE, copied_voice_probe)
|
||||
|
||||
self._git(root, "init")
|
||||
self._git(root, "config", "user.name", "Watchdog Contract Test")
|
||||
|
|
@ -195,6 +289,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
|
|||
"scripts/boot-public-runtime.ps1",
|
||||
"scripts/register-boot-task.ps1",
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py",
|
||||
)
|
||||
self._git(root, "commit", "-m", "watchdog fixture")
|
||||
self._git(root, "checkout", "--detach")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import unittest
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -83,6 +84,10 @@ class FreshPublicProvenanceContractTest(unittest.TestCase):
|
|||
"[switch]$RequireFreshPublicProvenance",
|
||||
'throw "-RequireFreshPublicProvenance requires -ForceApiRestart"',
|
||||
'throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"',
|
||||
'throw "-RequireFreshPublicProvenance requires -SkipEngineRestart;',
|
||||
'throw "-RequireFreshPublicProvenance requires -SkipWebRestart;',
|
||||
'throw "-RequireFreshPublicProvenance forbids DNS route mutation"',
|
||||
"Fresh public promotion requires the canonical HTTPS public health URL",
|
||||
"ExpectedSourceCommit",
|
||||
"ExpectedSourceTree",
|
||||
"ExpectedPythonSha256",
|
||||
|
|
@ -171,6 +176,18 @@ if ($receipt.status -ne 'passed') {{ throw 'atomic receipt content mismatch' }}
|
|||
Write-Utf8TextAtomically -OutputPath $resolved -Value '{{"status":"replaced"}}'
|
||||
$replacement = Get-Content -LiteralPath $resolved -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
if ($replacement.status -ne 'replaced') {{ throw 'atomic replacement mismatch' }}
|
||||
$failedPath = Write-FailedFreshPromotionEvidence `
|
||||
-OutputPath $resolved `
|
||||
-FailureStage 'receipt_publish' `
|
||||
-RollbackSucceeded $true `
|
||||
-RollbackResult @{{local_health=$true;public_health=$true}} `
|
||||
-SourceCommit ('a' * 40) `
|
||||
-SourceTree ('b' * 40)
|
||||
$failed = Get-Content -LiteralPath $failedPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
if ((Split-Path -Leaf $failedPath) -notlike '*.failed.log') {{ throw 'failure evidence suffix mismatch' }}
|
||||
if ($failed.status -ne 'failed_rolled_back') {{ throw 'failure evidence status mismatch' }}
|
||||
if ($failed.failure_stage -ne 'receipt_publish') {{ throw 'failure evidence stage mismatch' }}
|
||||
if (-not $failed.rollback.succeeded) {{ throw 'failure evidence rollback mismatch' }}
|
||||
if (@(Get-ChildItem -LiteralPath (Split-Path -Parent $resolved) -Filter '*.tmp').Count -ne 0) {{
|
||||
throw 'temporary receipt files were not cleaned'
|
||||
}}
|
||||
|
|
@ -271,7 +288,7 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri
|
|||
def test_receipt_projects_no_raw_command_or_config_contents(self) -> None:
|
||||
projection = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity") :
|
||||
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint")
|
||||
PUBLIC_RUNTIME.index("function Save-ManagedEnvironment")
|
||||
]
|
||||
self.assertIn("command_line_sha256 =", projection)
|
||||
self.assertNotIn("command_line =", projection)
|
||||
|
|
@ -297,9 +314,376 @@ if ($preserved.status -ne 'replaced') {{ throw 'failed preflight changed the pri
|
|||
PUBLIC_RUNTIME.index("function ConvertTo-SafeProcessIdentity")
|
||||
]
|
||||
self.assertIn("datetime.fromtimestamp(p.create_time(), UTC)", identity)
|
||||
self.assertIn("chr(0).join(p.cmdline())", identity)
|
||||
self.assertIn("argv=p.cmdline()", identity)
|
||||
self.assertIn("chr(0).join(argv)", identity)
|
||||
self.assertIn("argument_list = $argumentList", identity)
|
||||
self.assertIn("environment = $processEnvironment", identity)
|
||||
self.assertIn("command_line_sha256 = $commandLineSha256", identity)
|
||||
|
||||
def test_prior_process_argv_and_environment_round_trip_in_powershell_51(self) -> None:
|
||||
powershell = shutil.which("powershell.exe")
|
||||
if powershell is None:
|
||||
self.skipTest("Windows PowerShell 5.1 is not available")
|
||||
|
||||
function_source = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Save-CompleteProcessEnvironment") :
|
||||
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime")
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
functions_path = root / "rollback-process-functions.ps1"
|
||||
harness_path = root / "rollback-process-harness.ps1"
|
||||
functions_path.write_text(function_source, encoding="utf-8-sig")
|
||||
quoted_functions = str(functions_path).replace("'", "''")
|
||||
quoted_root = str(root).replace("'", "''")
|
||||
quoted_python = sys.executable.replace("'", "''")
|
||||
harness_path.write_text(
|
||||
f"""$ErrorActionPreference = 'Stop'
|
||||
. '{quoted_functions}'
|
||||
$root = '{quoted_root}'
|
||||
$python = '{quoted_python}'
|
||||
if ($null -eq (Get-Command Get-FileHash -ErrorAction SilentlyContinue)) {{
|
||||
function Get-FileHash {{
|
||||
param([string]$LiteralPath, [string]$Algorithm)
|
||||
$stream = [IO.File]::OpenRead($LiteralPath)
|
||||
try {{
|
||||
$hasher = [Security.Cryptography.SHA256]::Create()
|
||||
try {{ $hash = $hasher.ComputeHash($stream) }} finally {{ $hasher.Dispose() }}
|
||||
}} finally {{
|
||||
$stream.Dispose()
|
||||
}}
|
||||
[pscustomobject]@{{Hash = ([BitConverter]::ToString($hash)).Replace('-', '')}}
|
||||
}}
|
||||
}}
|
||||
$pythonSha256 = (Get-FileHash -LiteralPath $python -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$originalEnvironment = Save-CompleteProcessEnvironment
|
||||
try {{
|
||||
[Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'caller', 'Process')
|
||||
[Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', 'caller-only', 'Process')
|
||||
$callerEnvironment = Save-CompleteProcessEnvironment
|
||||
[Environment]::SetEnvironmentVariable('VIGNETTE_ROLLBACK_ARGV_TEST', 'prior', 'Process')
|
||||
[Environment]::SetEnvironmentVariable('VIGNETTE_CALLER_ONLY', $null, 'Process')
|
||||
$priorEnvironment = Save-CompleteProcessEnvironment
|
||||
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
||||
|
||||
$output = Join-Path $root 'argv-result.json'
|
||||
$code = "import json,os,sys; open(sys.argv[1], 'w', encoding='utf-8').write(json.dumps({{'argv':sys.argv[2:],'marker':os.environ.get('VIGNETTE_ROLLBACK_ARGV_TEST'),'leak':os.environ.get('VIGNETTE_CALLER_ONLY')}}, ensure_ascii=True))"
|
||||
$expected = @('plain', 'space value', 'quote"value', 'trailing\', '', 'slashes\\before"quote')
|
||||
$identity = [ordered]@{{
|
||||
executable_path = $python
|
||||
executable_sha256 = $pythonSha256
|
||||
cwd = $root
|
||||
argument_list = @('-X', 'utf8', '-c', $code, $output) + $expected
|
||||
environment = $priorEnvironment
|
||||
}}
|
||||
$process = Start-PinnedPriorProcess `
|
||||
-Identity $identity `
|
||||
-Role 'argv-roundtrip' `
|
||||
-StdoutLog (Join-Path $root 'child.out.log') `
|
||||
-StderrLog (Join-Path $root 'child.err.log')
|
||||
$process.WaitForExit()
|
||||
if (-not (Test-Path -LiteralPath $output -PathType Leaf)) {{ throw 'child did not write argv result' }}
|
||||
$actual = Get-Content -LiteralPath $output -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
if (@($actual.argv).Count -ne $expected.Count) {{ throw 'argument count mismatch' }}
|
||||
for ($i = 0; $i -lt $expected.Count; $i++) {{
|
||||
if ($actual.argv[$i] -cne $expected[$i]) {{ throw "argument mismatch at $i" }}
|
||||
}}
|
||||
if ($actual.marker -cne 'prior') {{ throw 'prior environment was not inherited' }}
|
||||
if ($null -ne $actual.leak) {{ throw 'caller-only environment leaked into prior process' }}
|
||||
if ($env:VIGNETTE_ROLLBACK_ARGV_TEST -cne 'caller') {{ throw 'caller environment marker was not restored' }}
|
||||
if ($env:VIGNETTE_CALLER_ONLY -cne 'caller-only') {{ throw 'caller-only environment was not restored' }}
|
||||
}} finally {{
|
||||
Set-CompleteProcessEnvironment -Environment $originalEnvironment
|
||||
}}
|
||||
""",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[
|
||||
powershell,
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(harness_path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode,
|
||||
0,
|
||||
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
||||
)
|
||||
|
||||
def test_rollback_restores_prior_openai_voice_contract_in_powershell_51(self) -> None:
|
||||
powershell = shutil.which("powershell.exe")
|
||||
if powershell is None:
|
||||
self.skipTest("Windows PowerShell 5.1 is not available")
|
||||
|
||||
voice_contract_source = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Test-PriorVoiceApiReady") :
|
||||
PUBLIC_RUNTIME.index("function Test-RequiredOpenApiPaths")
|
||||
]
|
||||
rollback_source = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") :
|
||||
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint")
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
functions_path = root / "rollback-contract-functions.ps1"
|
||||
harness_path = root / "rollback-contract-harness.ps1"
|
||||
functions_path.write_text(
|
||||
voice_contract_source + rollback_source,
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
quoted_functions = str(functions_path).replace("'", "''")
|
||||
quoted_root = str(root).replace("'", "''")
|
||||
harness_path.write_text(
|
||||
f"""$ErrorActionPreference = 'Stop'
|
||||
. '{quoted_functions}'
|
||||
$events = New-Object System.Collections.ArrayList
|
||||
$root = '{quoted_root}'
|
||||
function Stop-UvicornByPort {{ param($AppImport,$Port,$TimeoutSec); $null=$events.Add('stop-api'); 11 }}
|
||||
function Restore-ManagedEnvironment {{ param($Snapshot); $null=$events.Add('restore-env') }}
|
||||
function Start-PinnedPriorProcess {{
|
||||
param($Identity,$Role,$StdoutLog,$StderrLog)
|
||||
$null=$events.Add("start-$Role")
|
||||
if ($Role -eq 'api') {{ [pscustomobject]@{{Id=101}} }} else {{ [pscustomobject]@{{Id=202}} }}
|
||||
}}
|
||||
function Wait-ProcessIdentity {{
|
||||
param($ProcessId,$Role,$ExpectedCwd,$TimeoutSec)
|
||||
$null=$events.Add("identity-$Role")
|
||||
[ordered]@{{
|
||||
pid=[int]$ProcessId; started_at_utc='2026-08-09T00:00:00Z';
|
||||
executable_name='runtime.exe'; executable_sha256=('a' * 64);
|
||||
command_line_sha256=('b' * 64); cwd=$root
|
||||
}}
|
||||
}}
|
||||
function Wait-JsonHealth {{
|
||||
param($Uri,$IsHealthy,$TimeoutSec)
|
||||
$null=$events.Add("health-$Uri")
|
||||
if ($Uri -like '*/voice/health') {{
|
||||
$payload=[pscustomobject]@{{
|
||||
status='ok';available=$true;stt_available=$true;tts_available=$true;
|
||||
stt_provider='openai';stt_model='gpt-4o-mini-transcribe';
|
||||
tts_provider='openai';tts_model='gpt-4o-mini-tts';
|
||||
limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}}
|
||||
}}
|
||||
}} else {{
|
||||
$payload=[pscustomobject]@{{environment='prod';db=$true;engine=$true}}
|
||||
}}
|
||||
if (-not (& $IsHealthy $payload)) {{ throw "health predicate failed: $Uri" }}
|
||||
$payload
|
||||
}}
|
||||
function Test-VoiceSidecarReady {{ param($Component); $true }}
|
||||
function Get-CloudflaredProcessesForConfig {{ param($ConfigPath,[switch]$ExactPath); @([pscustomobject]@{{ProcessId=303}}) }}
|
||||
function Stop-ProcessesBounded {{ param($Processes,$TimeoutSec,$Role); $null=$events.Add('stop-cloudflared'); 303 }}
|
||||
function ConvertTo-SafeProcessIdentity {{ param($Identity); $Identity }}
|
||||
|
||||
$configPath = Join-Path $root 'cloudflared.yml'
|
||||
[IO.File]::WriteAllText($configPath, 'tunnel: test')
|
||||
$priorVoice = [ordered]@{{
|
||||
status='ok';available=$true;stt_available=$true;tts_available=$true;
|
||||
stt_provider='openai';stt_model='gpt-4o-mini-transcribe';
|
||||
tts_provider='openai';tts_model='gpt-4o-mini-tts';uvicorn_ws_max_queue=4
|
||||
}}
|
||||
$priorApi=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}}
|
||||
$priorCloud=[ordered]@{{executable_sha256=('a' * 64);command_line_sha256=('b' * 64);cwd=$root}}
|
||||
$result = Restore-PriorPublicRuntime `
|
||||
-PriorApi $priorApi `
|
||||
-PriorCloudflared $priorCloud `
|
||||
-PriorLocalVoiceContract $priorVoice `
|
||||
-PriorPublicVoiceContract $priorVoice `
|
||||
-EnvironmentSnapshot @{{}} `
|
||||
-ConfigPath $configPath `
|
||||
-ApiPortValue 8001 `
|
||||
-HealthUrl 'https://api-vignette.chanpaca.net/health' `
|
||||
-VoiceHealthUrl 'https://api-vignette.chanpaca.net/voice/health' `
|
||||
-TimeoutSec 5
|
||||
foreach ($field in @('local_health','local_voice_health','public_health','public_voice_health')) {{
|
||||
if (-not $result[$field]) {{ throw "rollback result missing $field" }}
|
||||
}}
|
||||
if ($events.IndexOf('stop-api') -ge $events.IndexOf('start-api')) {{ throw 'API restore ordering mismatch' }}
|
||||
if ($events.IndexOf('stop-cloudflared') -ge $events.IndexOf('start-cloudflared')) {{ throw 'cloudflared restore ordering mismatch' }}
|
||||
if ($events.IndexOf('start-api') -ge $events.IndexOf('health-http://127.0.0.1:8001/health')) {{ throw 'local health ordering mismatch' }}
|
||||
if ($events.IndexOf('start-cloudflared') -ge $events.IndexOf('health-https://api-vignette.chanpaca.net/health')) {{ throw 'public health ordering mismatch' }}
|
||||
$changed=[pscustomobject]@{{
|
||||
status='ok';available=$true;stt_available=$true;tts_available=$true;
|
||||
stt_provider='openai';stt_model='changed-model';tts_provider='openai';tts_model='gpt-4o-mini-tts';
|
||||
limits=[pscustomobject]@{{uvicorn_ws_max_queue=4}}
|
||||
}}
|
||||
if (Test-VoiceHealthContract -Health $changed -Expected $priorVoice) {{ throw 'voice contract drift was accepted' }}
|
||||
""",
|
||||
encoding="utf-8-sig",
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[
|
||||
powershell,
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(harness_path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode,
|
||||
0,
|
||||
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
||||
)
|
||||
|
||||
def test_engine_log_rotation_cannot_dirty_the_stable_release(self) -> None:
|
||||
self.assertIn('Destination "$logFile.$rotateStamp.bak.log"', PUBLIC_RUNTIME)
|
||||
self.assertNotIn('Destination "$logFile.$rotateStamp.bak"', PUBLIC_RUNTIME)
|
||||
|
||||
def test_failure_evidence_keeps_a_clean_detached_git_source(self) -> None:
|
||||
self.assertIn('$failedPath = "$OutputPath.failed.log"', PUBLIC_RUNTIME)
|
||||
self.assertNotIn('$failedPath = "$OutputPath.failed.json"', PUBLIC_RUNTIME)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
(root / ".gitignore").write_text("*.log\n", encoding="utf-8")
|
||||
(root / "tracked.txt").write_text("release\n", encoding="utf-8")
|
||||
commands = (
|
||||
["git", "init", "--quiet"],
|
||||
["git", "config", "user.name", "Yun Chan"],
|
||||
["git", "config", "user.email", "yun.chan@example.invalid"],
|
||||
["git", "add", ".gitignore", "tracked.txt"],
|
||||
["git", "commit", "--quiet", "-m", "테스트 기준"],
|
||||
["git", "checkout", "--quiet", "--detach", "HEAD"],
|
||||
)
|
||||
for command in commands:
|
||||
subprocess.run(
|
||||
command,
|
||||
cwd=root,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
failure_evidence = root / "public-runtime-launch-provenance.log.failed.log"
|
||||
failure_evidence.write_text('{"status":"failed_rolled_back"}\n', encoding="utf-8")
|
||||
ignored = subprocess.run(
|
||||
["git", "check-ignore", "--quiet", failure_evidence.name],
|
||||
cwd=root,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(ignored.returncode, 0)
|
||||
status = subprocess.run(
|
||||
["git", "status", "--porcelain", "--untracked-files=normal"],
|
||||
cwd=root,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
self.assertEqual(status.stdout, b"")
|
||||
|
||||
def test_fresh_cutover_requires_and_restores_a_pinned_prior_runtime(self) -> None:
|
||||
prior_capture = PUBLIC_RUNTIME.index("$priorApiProcesses = @(")
|
||||
mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"')
|
||||
self.assertLess(prior_capture, mutation)
|
||||
for expected in (
|
||||
"requires exactly one prior API process for transactional rollback",
|
||||
"requires exactly one prior cloudflared process for transactional rollback",
|
||||
"$freshPriorApiIdentity = Wait-ProcessIdentity",
|
||||
"$freshPriorCloudflaredIdentity = Wait-ProcessIdentity",
|
||||
"$freshEnvironmentSnapshot = Save-ManagedEnvironment",
|
||||
"Restore-PriorPublicRuntime `",
|
||||
"Start-PinnedPriorProcess `",
|
||||
"Restored prior API identity drift",
|
||||
"Restored prior cloudflared identity drift",
|
||||
"$freshPromotionCommitted = $true",
|
||||
):
|
||||
with self.subTest(expected=expected):
|
||||
self.assertIn(expected, PUBLIC_RUNTIME)
|
||||
|
||||
def test_fresh_transaction_has_no_engine_web_sidecar_or_dns_mutation(self) -> None:
|
||||
contract = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract") :
|
||||
PUBLIC_RUNTIME.index("$freshMutationStarted = $false")
|
||||
]
|
||||
for expected in (
|
||||
"if (-not $SkipEngineRestart)",
|
||||
"if (-not $SkipWebRestart)",
|
||||
"if ($RouteCloudflareDns)",
|
||||
"$CanonicalPublicHealthUrl",
|
||||
):
|
||||
with self.subTest(expected=expected):
|
||||
self.assertIn(expected, contract)
|
||||
|
||||
api_mutation = PUBLIC_RUNTIME.index('$freshFailureStage = "api_cutover"')
|
||||
preflight = PUBLIC_RUNTIME[:api_mutation]
|
||||
self.assertIn("requires exact healthy voice sidecars as an unchanged precondition", preflight)
|
||||
self.assertIn("-Uri $CanonicalPublicHealthUrl", preflight)
|
||||
self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", preflight)
|
||||
self.assertIn("$health.engine -eq $true", preflight)
|
||||
|
||||
stt_start = PUBLIC_RUNTIME.index("& $WhisperStartScript `")
|
||||
tts_start = PUBLIC_RUNTIME.index("& $MeloTtsStartScript `")
|
||||
self.assertLess(
|
||||
PUBLIC_RUNTIME.index("Fresh public promotion will not mutate local_whisper"),
|
||||
stt_start,
|
||||
)
|
||||
self.assertLess(
|
||||
PUBLIC_RUNTIME.index("Fresh public promotion will not mutate MeloTTS"),
|
||||
tts_start,
|
||||
)
|
||||
|
||||
def test_success_and_rollback_require_canonical_public_engine_and_voice(self) -> None:
|
||||
rollback = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Restore-PriorPublicRuntime") :
|
||||
PUBLIC_RUNTIME.index("function Stop-NodeByPortHint")
|
||||
]
|
||||
self.assertGreaterEqual(rollback.count("$health.engine -eq $true"), 2)
|
||||
self.assertIn("Test-VoiceSidecarReady", rollback)
|
||||
self.assertIn("Test-VoiceHealthContract", rollback)
|
||||
|
||||
success_gate = PUBLIC_RUNTIME.index('$freshFailureStage = "public_health_validation"')
|
||||
receipt = PUBLIC_RUNTIME.index("$provenance = [ordered]@{")
|
||||
committed = PUBLIC_RUNTIME.index("$freshPromotionCommitted = $true")
|
||||
self.assertLess(success_gate, receipt)
|
||||
self.assertLess(receipt, committed)
|
||||
success_section = PUBLIC_RUNTIME[success_gate:receipt]
|
||||
self.assertIn("-Uri $CanonicalPublicHealthUrl", success_section)
|
||||
self.assertIn("-Uri $CanonicalPublicVoiceHealthUrl", success_section)
|
||||
self.assertIn("-Uri $CanonicalPublicOpenApiUrl", success_section)
|
||||
self.assertIn("Test-RequiredOpenApiPaths", success_section)
|
||||
self.assertIn('"/admin/voice-runtime"', PUBLIC_RUNTIME)
|
||||
self.assertIn("$health.engine -eq $true", success_section)
|
||||
self.assertIn("Test-VoiceSidecarReady", success_section)
|
||||
|
||||
def test_failed_cutover_emits_metadata_only_rollback_evidence(self) -> None:
|
||||
failure_writer = PUBLIC_RUNTIME[
|
||||
PUBLIC_RUNTIME.index("function Write-FailedFreshPromotionEvidence") :
|
||||
PUBLIC_RUNTIME.index("function Assert-FreshPublicProvenanceContract")
|
||||
]
|
||||
for expected in (
|
||||
'schema_version = "vignette.public-runtime-launch-failure.v1"',
|
||||
'"failed_rolled_back"',
|
||||
'"failed_rollback"',
|
||||
"failure_stage = $FailureStage",
|
||||
"attempted = $true",
|
||||
"succeeded = $RollbackSucceeded",
|
||||
"Write-Utf8TextAtomically",
|
||||
):
|
||||
with self.subTest(expected=expected):
|
||||
self.assertIn(expected, failure_writer)
|
||||
self.assertNotIn("command_line", failure_writer)
|
||||
self.assertNotIn("executable_path", failure_writer)
|
||||
|
||||
def test_receipt_is_direct_input_for_windows_topology_capture(self) -> None:
|
||||
receipt = PUBLIC_RUNTIME.index(
|
||||
'schema_version = "vignette.public-runtime-launch-provenance.v1"'
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
[int]$ApiPort = 8001,
|
||||
[int]$WebPort = 5174,
|
||||
[int]$EnginePort = 9099,
|
||||
[int]$WhisperPort = 9882,
|
||||
[int]$MeloTtsPort = 9883,
|
||||
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
||||
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
||||
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
||||
|
|
@ -36,6 +38,7 @@ $ErrorActionPreference = "Stop"
|
|||
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
||||
$expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
|
||||
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
|
||||
function Invoke-GitText {
|
||||
param([string[]]$Arguments)
|
||||
|
|
@ -54,6 +57,9 @@ function Assert-StableSourceProvenance {
|
|||
if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) {
|
||||
throw "Pinned start script not found at $startScript"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) {
|
||||
throw "Pinned voice sidecar probe not found at $voiceSidecarProbe"
|
||||
}
|
||||
|
||||
$runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
||||
if (-not [string]::Equals(
|
||||
|
|
@ -99,7 +105,8 @@ function Assert-StableSourceProvenance {
|
|||
|
||||
foreach ($relativePath in @(
|
||||
"scripts/watch-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1"
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
@ -203,6 +210,47 @@ function Get-EngineHeaders {
|
|||
return $null
|
||||
}
|
||||
|
||||
function Test-VoiceApiReady {
|
||||
param([object]$Health)
|
||||
|
||||
return (
|
||||
$null -ne $Health -and
|
||||
$Health.status -eq "ok" -and
|
||||
$Health.available -eq $true -and
|
||||
$Health.stt_available -eq $true -and
|
||||
$Health.tts_available -eq $true -and
|
||||
$Health.stt_provider -eq "local_whisper" -and
|
||||
$Health.stt_model -eq "small" -and
|
||||
$Health.tts_provider -eq "melotts" -and
|
||||
$Health.tts_model -eq "melotts-korean" -and
|
||||
$Health.limits.uvicorn_ws_max_queue -eq 4
|
||||
)
|
||||
}
|
||||
|
||||
function Test-VoiceSidecarStack {
|
||||
$probeArgs = @(
|
||||
"-X", "utf8", "-B", $voiceSidecarProbe,
|
||||
"--component", "all",
|
||||
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
||||
"--stt-provider", "local_whisper",
|
||||
"--stt-model", "small",
|
||||
"--stt-language", "ko",
|
||||
"--stt-device", "cpu",
|
||||
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
||||
"--tts-provider", "melotts",
|
||||
"--tts-model", "melotts-korean",
|
||||
"--tts-language", "KR",
|
||||
"--timeout-seconds", "5"
|
||||
)
|
||||
$output = @(& $Python @probeArgs 2>$null)
|
||||
$probeExit = $LASTEXITCODE
|
||||
return [pscustomobject]@{
|
||||
Name = "voice-sidecars"
|
||||
Ok = $probeExit -eq 0
|
||||
Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" }
|
||||
}
|
||||
}
|
||||
|
||||
function Test-CloudflaredProcess {
|
||||
if ($SkipCloudflaredRestart) {
|
||||
return [pscustomobject]@{
|
||||
|
|
@ -245,10 +293,16 @@ $checks = @(
|
|||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
||||
-TimeoutSec 60),
|
||||
(Test-JsonHealth `
|
||||
-Name "voice-api" `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30),
|
||||
(Test-JsonHealth `
|
||||
-Name "web-preview" `
|
||||
-Uri "http://127.0.0.1:$WebPort/" `
|
||||
-IsHealthy { param($body) $true }),
|
||||
(Test-VoiceSidecarStack),
|
||||
(Test-CloudflaredProcess)
|
||||
)
|
||||
|
||||
|
|
@ -294,6 +348,8 @@ $startArgs = @{
|
|||
ApiPort = $ApiPort
|
||||
WebPort = $WebPort
|
||||
EnginePort = $EnginePort
|
||||
WhisperPort = $WhisperPort
|
||||
MeloTtsPort = $MeloTtsPort
|
||||
Python = $Python
|
||||
Cloudflared = $Cloudflared
|
||||
CloudflaredConfig = $CloudflaredConfig
|
||||
|
|
@ -324,6 +380,15 @@ if (!$apiAfter.Ok) {
|
|||
throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
|
||||
}
|
||||
|
||||
$voiceApiAfter = Test-JsonHealth `
|
||||
-Name "voice-api" `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
if (!$voiceApiAfter.Ok) {
|
||||
throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)"
|
||||
}
|
||||
|
||||
$webAfter = Test-JsonHealth `
|
||||
-Name "web-preview" `
|
||||
-Uri "http://127.0.0.1:$WebPort/" `
|
||||
|
|
@ -343,6 +408,11 @@ if (!$engineAfter.Ok) {
|
|||
throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)"
|
||||
}
|
||||
|
||||
$voiceSidecarsAfter = Test-VoiceSidecarStack
|
||||
if (!$voiceSidecarsAfter.Ok) {
|
||||
throw "Exact local voice sidecars still unhealthy after restart"
|
||||
}
|
||||
|
||||
if (!$SkipPublicHealth) {
|
||||
$publicAfter = Test-JsonHealth `
|
||||
-Name "public-api" `
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue