공개 런타임 롤백 경계 강화

This commit is contained in:
Yun Chan 2026-08-09 21:57:07 +09:00
parent ff3c79dfc2
commit aaebe4450e
7 changed files with 1271 additions and 30 deletions

View file

@ -28,6 +28,10 @@ param(
[int]$DbTimeoutSec = 90,
[int]$DbPort = 55432,
[int]$ApiPort = 8001,
[int]$EnginePort = 9099,
[int]$WhisperPort = 9882,
[int]$MeloTtsPort = 9883,
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
[string]$BootLog = ""
)
@ -36,6 +40,7 @@ $ErrorActionPreference = "Stop"
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
$expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
function Invoke-GitText {
param([string[]]$Arguments)
@ -48,7 +53,7 @@ function Invoke-GitText {
}
function Assert-StableSourceProvenance {
foreach ($requiredScript in @($expectedBootScript, $startScript)) {
foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) {
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
throw "Pinned public runtime script not found at $requiredScript"
}
@ -97,7 +102,8 @@ function Assert-StableSourceProvenance {
}
foreach ($relativePath in @(
"scripts/boot-public-runtime.ps1",
"scripts/start-public-runtime.ps1"
"scripts/start-public-runtime.ps1",
"scripts/probe-public-voice-sidecars.py"
)) {
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
}
@ -159,7 +165,7 @@ function Test-ApiControlPlaneHealthy {
$body = $reader.ReadToEnd()
$reader.Close(); $resp.Close()
$h = $body | ConvertFrom-Json
if ($h.environment -eq "prod" -and $h.db -eq $true) { return $true }
if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true }
Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body)
return $false
} catch {
@ -172,13 +178,72 @@ function Test-ApiControlPlaneHealthy {
function Test-EngineHealthy {
try {
$response = Invoke-RestMethod -Uri "http://127.0.0.1:9099/health" -TimeoutSec 10
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10
return $response.ok -eq $true
} catch {
return $false
}
}
function Test-VoiceApiReady {
param([object]$Health)
return (
$null -ne $Health -and
$Health.status -eq "ok" -and
$Health.available -eq $true -and
$Health.stt_available -eq $true -and
$Health.tts_available -eq $true -and
$Health.stt_provider -eq "local_whisper" -and
$Health.stt_model -eq "small" -and
$Health.tts_provider -eq "melotts" -and
$Health.tts_model -eq "melotts-korean" -and
$Health.limits.uvicorn_ws_max_queue -eq 4
)
}
function Test-VoiceApiHealthy {
for ($i = 1; $i -le 3; $i++) {
try {
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health")
$req.Timeout = 5000
$req.ReadWriteTimeout = 5000
$req.Proxy = $null
$resp = $req.GetResponse()
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
$body = $reader.ReadToEnd()
$reader.Close(); $resp.Close()
$health = $body | ConvertFrom-Json
if (Test-VoiceApiReady -Health $health) { return $true }
Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body)
return $false
} catch {
Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
Start-Sleep -Seconds 2
}
}
return $false
}
function Test-VoiceSidecarStack {
$probeArgs = @(
"-X", "utf8", "-B", $voiceSidecarProbe,
"--component", "all",
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
"--stt-provider", "local_whisper",
"--stt-model", "small",
"--stt-language", "ko",
"--stt-device", "cpu",
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
"--tts-provider", "melotts",
"--tts-model", "melotts-korean",
"--tts-language", "KR",
"--timeout-seconds", "5"
)
& $Python @probeArgs 1>$null 2>$null
return $LASTEXITCODE -eq 0
}
Write-BootLog "================ boot start ================"
# 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기)
@ -217,12 +282,16 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) {
Write-BootLog "postgres 127.0.0.1:$DbPort up"
# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지)
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
Write-BootLog "control plane and engine already healthy; skipping runtime restart"
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart"
} else {
Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart"
$out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
-Workspace $resolvedSourceRoot `
-Python $Python `
-EnginePort $EnginePort `
-WhisperPort $WhisperPort `
-MeloTtsPort $MeloTtsPort `
-SkipWebRestart 2>&1
$out | ForEach-Object { Write-BootLog (" pub> " + $_) }
if ($LASTEXITCODE -ne 0) {
@ -233,12 +302,13 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
# 4) 최종 확인
if (Test-ApiControlPlaneHealthy) {
if (Test-EngineHealthy) {
Write-BootLog "boot OK: control plane and engine healthy"
if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy"
exit 0
} else {
Write-BootLog "boot OK: admin/auth control plane healthy; engine remains degraded"
Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded"
exit 2
}
exit 0
} else {
Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log"
exit 2