공개 런타임 롤백 경계 강화
This commit is contained in:
parent
ff3c79dfc2
commit
aaebe4450e
7 changed files with 1271 additions and 30 deletions
|
|
@ -28,6 +28,10 @@ param(
|
|||
[int]$DbTimeoutSec = 90,
|
||||
[int]$DbPort = 55432,
|
||||
[int]$ApiPort = 8001,
|
||||
[int]$EnginePort = 9099,
|
||||
[int]$WhisperPort = 9882,
|
||||
[int]$MeloTtsPort = 9883,
|
||||
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
||||
[string]$BootLog = ""
|
||||
)
|
||||
|
||||
|
|
@ -36,6 +40,7 @@ $ErrorActionPreference = "Stop"
|
|||
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
||||
$expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
|
||||
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
|
||||
function Invoke-GitText {
|
||||
param([string[]]$Arguments)
|
||||
|
|
@ -48,7 +53,7 @@ function Invoke-GitText {
|
|||
}
|
||||
|
||||
function Assert-StableSourceProvenance {
|
||||
foreach ($requiredScript in @($expectedBootScript, $startScript)) {
|
||||
foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) {
|
||||
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
|
||||
throw "Pinned public runtime script not found at $requiredScript"
|
||||
}
|
||||
|
|
@ -97,7 +102,8 @@ function Assert-StableSourceProvenance {
|
|||
}
|
||||
foreach ($relativePath in @(
|
||||
"scripts/boot-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1"
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
@ -159,7 +165,7 @@ function Test-ApiControlPlaneHealthy {
|
|||
$body = $reader.ReadToEnd()
|
||||
$reader.Close(); $resp.Close()
|
||||
$h = $body | ConvertFrom-Json
|
||||
if ($h.environment -eq "prod" -and $h.db -eq $true) { return $true }
|
||||
if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true }
|
||||
Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body)
|
||||
return $false
|
||||
} catch {
|
||||
|
|
@ -172,13 +178,72 @@ function Test-ApiControlPlaneHealthy {
|
|||
|
||||
function Test-EngineHealthy {
|
||||
try {
|
||||
$response = Invoke-RestMethod -Uri "http://127.0.0.1:9099/health" -TimeoutSec 10
|
||||
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10
|
||||
return $response.ok -eq $true
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Test-VoiceApiReady {
|
||||
param([object]$Health)
|
||||
|
||||
return (
|
||||
$null -ne $Health -and
|
||||
$Health.status -eq "ok" -and
|
||||
$Health.available -eq $true -and
|
||||
$Health.stt_available -eq $true -and
|
||||
$Health.tts_available -eq $true -and
|
||||
$Health.stt_provider -eq "local_whisper" -and
|
||||
$Health.stt_model -eq "small" -and
|
||||
$Health.tts_provider -eq "melotts" -and
|
||||
$Health.tts_model -eq "melotts-korean" -and
|
||||
$Health.limits.uvicorn_ws_max_queue -eq 4
|
||||
)
|
||||
}
|
||||
|
||||
function Test-VoiceApiHealthy {
|
||||
for ($i = 1; $i -le 3; $i++) {
|
||||
try {
|
||||
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health")
|
||||
$req.Timeout = 5000
|
||||
$req.ReadWriteTimeout = 5000
|
||||
$req.Proxy = $null
|
||||
$resp = $req.GetResponse()
|
||||
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
|
||||
$body = $reader.ReadToEnd()
|
||||
$reader.Close(); $resp.Close()
|
||||
$health = $body | ConvertFrom-Json
|
||||
if (Test-VoiceApiReady -Health $health) { return $true }
|
||||
Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body)
|
||||
return $false
|
||||
} catch {
|
||||
Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Test-VoiceSidecarStack {
|
||||
$probeArgs = @(
|
||||
"-X", "utf8", "-B", $voiceSidecarProbe,
|
||||
"--component", "all",
|
||||
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
||||
"--stt-provider", "local_whisper",
|
||||
"--stt-model", "small",
|
||||
"--stt-language", "ko",
|
||||
"--stt-device", "cpu",
|
||||
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
||||
"--tts-provider", "melotts",
|
||||
"--tts-model", "melotts-korean",
|
||||
"--tts-language", "KR",
|
||||
"--timeout-seconds", "5"
|
||||
)
|
||||
& $Python @probeArgs 1>$null 2>$null
|
||||
return $LASTEXITCODE -eq 0
|
||||
}
|
||||
|
||||
Write-BootLog "================ boot start ================"
|
||||
|
||||
# 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기)
|
||||
|
|
@ -217,12 +282,16 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) {
|
|||
Write-BootLog "postgres 127.0.0.1:$DbPort up"
|
||||
|
||||
# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지)
|
||||
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
|
||||
Write-BootLog "control plane and engine already healthy; skipping runtime restart"
|
||||
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
||||
Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart"
|
||||
} else {
|
||||
Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart"
|
||||
$out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
|
||||
-Workspace $resolvedSourceRoot `
|
||||
-Python $Python `
|
||||
-EnginePort $EnginePort `
|
||||
-WhisperPort $WhisperPort `
|
||||
-MeloTtsPort $MeloTtsPort `
|
||||
-SkipWebRestart 2>&1
|
||||
$out | ForEach-Object { Write-BootLog (" pub> " + $_) }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
|
|
@ -233,12 +302,13 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy)) {
|
|||
|
||||
# 4) 최종 확인
|
||||
if (Test-ApiControlPlaneHealthy) {
|
||||
if (Test-EngineHealthy) {
|
||||
Write-BootLog "boot OK: control plane and engine healthy"
|
||||
if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
||||
Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy"
|
||||
exit 0
|
||||
} else {
|
||||
Write-BootLog "boot OK: admin/auth control plane healthy; engine remains degraded"
|
||||
Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded"
|
||||
exit 2
|
||||
}
|
||||
exit 0
|
||||
} else {
|
||||
Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log"
|
||||
exit 2
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue