527 lines
18 KiB
PowerShell
527 lines
18 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$StableSourceRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceCommit,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceTree,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedWatchdogSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedStartScriptSha256,
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[int]$DbPort = 55432,
|
|
[string]$DbContainer = "vignette-dev-db",
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[string[]]$AdditionalPublicHealthUrls = @(),
|
|
# 게이트웨이가 shared secret으로 떠 있으면 /ready는 인증이 필요하다(/health만 면제).
|
|
# 토큰이 없으면 401을 장애로 오판해 재시작 폭풍이 난다.
|
|
[string]$EngineToken = $env:ENGINE_GATEWAY_SHARED_SECRET,
|
|
[string]$LogPath = "",
|
|
[int]$FailuresBeforeRestart = 3,
|
|
[switch]$CheckOnly,
|
|
[switch]$SkipPublicHealth,
|
|
[switch]$SkipCloudflaredRestart
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|
$expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
|
|
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
|
|
|
function Invoke-GitText {
|
|
param([string[]]$Arguments)
|
|
|
|
$value = & git.exe -C $resolvedSourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Assert-StableSourceProvenance {
|
|
if (-not (Test-Path -LiteralPath $expectedWatchdogPath -PathType Leaf)) {
|
|
throw "Pinned watchdog script not found at $expectedWatchdogPath"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) {
|
|
throw "Pinned start script not found at $startScript"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) {
|
|
throw "Pinned voice sidecar probe not found at $voiceSidecarProbe"
|
|
}
|
|
|
|
$runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningWatchdogPath,
|
|
(Resolve-Path -LiteralPath $expectedWatchdogPath).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Watchdog is not executing from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-GitText -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Stable source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Stable source must be a detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $ExpectedSourceCommit.ToLowerInvariant()) {
|
|
throw "Stable source commit drift: expected=$ExpectedSourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $ExpectedSourceTree.ToLowerInvariant()) {
|
|
throw "Stable source tree drift: expected=$ExpectedSourceTree actual=$actualTree"
|
|
}
|
|
|
|
$dirty = Invoke-GitText -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Stable source is not clean; refusing runtime recovery"
|
|
}
|
|
|
|
foreach ($relativePath in @(
|
|
"scripts/watch-public-runtime.ps1",
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/probe-public-voice-sidecars.py"
|
|
)) {
|
|
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
|
}
|
|
|
|
$actualWatchdogSha256 = (Get-FileHash -LiteralPath $expectedWatchdogPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$actualStartScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualWatchdogSha256 -ne $ExpectedWatchdogSha256.ToLowerInvariant()) {
|
|
throw "Pinned watchdog SHA256 drift"
|
|
}
|
|
if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) {
|
|
throw "Pinned start script SHA256 drift"
|
|
}
|
|
}
|
|
|
|
# health probe, failcount 기록, 프로세스 재기동보다 먼저 source provenance를 닫는다.
|
|
# 검증 실패는 운영 프로세스를 그대로 보존한 채 non-zero로 끝난다.
|
|
Assert-StableSourceProvenance
|
|
|
|
if (!$LogPath) {
|
|
$LogPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.log"
|
|
}
|
|
|
|
# 연속 실패 카운터(재시작 debounce용). 워치독은 매 실행마다 새 프로세스라 파일로 유지한다.
|
|
$FailCountPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.failcount"
|
|
|
|
function Get-FailCount {
|
|
if (Test-Path $FailCountPath) {
|
|
$raw = (Get-Content -Raw -Path $FailCountPath -ErrorAction SilentlyContinue)
|
|
$n = 0
|
|
if ([int]::TryParse(($raw -replace '\s', ''), [ref]$n)) { return $n }
|
|
}
|
|
return 0
|
|
}
|
|
|
|
function Set-FailCount {
|
|
param([int]$Value)
|
|
Set-Content -Path $FailCountPath -Value $Value -Encoding ascii
|
|
}
|
|
|
|
function Write-WatchdogLog {
|
|
param([string]$Message)
|
|
|
|
$line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-ddTHH:mm:ssK"), $Message
|
|
Write-Output $line
|
|
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
|
}
|
|
|
|
function Test-JsonHealth {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 30,
|
|
[hashtable]$Headers = $null
|
|
)
|
|
|
|
try {
|
|
if ($Headers) {
|
|
$response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec -Headers $Headers
|
|
} else {
|
|
$response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
}
|
|
$ok = [bool](& $IsHealthy $response)
|
|
$detail = $response | ConvertTo-Json -Compress -Depth 5
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $ok
|
|
Detail = $detail
|
|
}
|
|
} catch {
|
|
# 게이트웨이 /ready는 실패를 503 + JSON 본문으로 알린다. 본문을 버리면 로그에
|
|
# "(503)"만 남아 원인(만료 인증·플래그 오류)을 잃는다.
|
|
$detail = $_.Exception.Message
|
|
$body = ""
|
|
if ($_.ErrorDetails -and $_.ErrorDetails.Message) {
|
|
$body = $_.ErrorDetails.Message
|
|
} elseif ($_.Exception.Response) {
|
|
try {
|
|
$reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream())
|
|
$body = $reader.ReadToEnd()
|
|
$reader.Dispose()
|
|
} catch {
|
|
$body = ""
|
|
}
|
|
}
|
|
if ($body) {
|
|
$detail = "$detail :: $body"
|
|
}
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $false
|
|
Detail = $detail
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-EngineHeaders {
|
|
if ($EngineToken) {
|
|
return @{ "X-Vignette-Engine-Token" = $EngineToken }
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq "small" -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq "melotts-korean" -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-VoiceSidecarStack {
|
|
$probeArgs = @(
|
|
"-X", "utf8", "-B", $voiceSidecarProbe,
|
|
"--component", "all",
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", "small",
|
|
"--stt-language", "ko",
|
|
"--stt-device", "cpu",
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", "melotts-korean",
|
|
"--tts-language", "KR",
|
|
"--timeout-seconds", "5"
|
|
)
|
|
# Windows PowerShell 5.1은 native stderr를 ErrorRecord로 승격한다. 스크립트
|
|
# 전역의 Stop 정책을 그대로 두면 sidecar 장애를 복구 신호로 반환하기 전에
|
|
# NativeCommandError로 종료하므로, 이 단일 probe 경계에서만 Continue로 낮춘다.
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
$output = @(& $Python @probeArgs 2>$null)
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return [pscustomobject]@{
|
|
Name = "voice-sidecars"
|
|
Ok = $probeExit -eq 0
|
|
Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" }
|
|
}
|
|
}
|
|
|
|
function Test-CloudflaredProcess {
|
|
if ($SkipCloudflaredRestart) {
|
|
return [pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $true
|
|
Detail = "skipped"
|
|
}
|
|
}
|
|
|
|
$configLeaf = Split-Path -Leaf $CloudflaredConfig
|
|
$process = Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*$configLeaf*"
|
|
} |
|
|
Select-Object -First 1
|
|
|
|
[pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $null -ne $process
|
|
Detail = if ($process) { "pid=$($process.ProcessId)" } else { "not running" }
|
|
}
|
|
}
|
|
|
|
function Test-DatabaseListening {
|
|
$ok = $false
|
|
try {
|
|
$client = New-Object System.Net.Sockets.TcpClient
|
|
$async = $client.BeginConnect("127.0.0.1", $DbPort, $null, $null)
|
|
$ok = $async.AsyncWaitHandle.WaitOne(3000) -and $client.Connected
|
|
$client.Close()
|
|
} catch {
|
|
$ok = $false
|
|
}
|
|
|
|
[pscustomobject]@{
|
|
Name = "db"
|
|
Ok = $ok
|
|
Detail = if ($ok) { "127.0.0.1:$DbPort" } else { "not listening on 127.0.0.1:$DbPort" }
|
|
}
|
|
}
|
|
|
|
# postgres(docker: vignette-dev-db)는 start-public-runtime.ps1의 복구 범위 밖이다(boot 담당).
|
|
# DB가 내려간 채로 start만 반복 호출하면 API가 매번 startup에서 죽어 워치독은 영원히
|
|
# 실패만 기록한다 — 2026-08-12 PC 비정상 종료 후 5분마다 실패 창이 뜬 사건의 구조다.
|
|
# 재시작 전에 컨테이너를 직접 되살린다.
|
|
function Restore-DatabaseContainer {
|
|
try {
|
|
$output = & docker.exe start $DbContainer 2>$null
|
|
$output | ForEach-Object { Write-WatchdogLog " docker> $_" }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-WatchdogLog "db restore: docker start exit=$LASTEXITCODE"
|
|
return $false
|
|
}
|
|
} catch {
|
|
Write-WatchdogLog "db restore failed: $($_.Exception.Message)"
|
|
return $false
|
|
}
|
|
|
|
for ($i = 1; $i -le 30; $i++) {
|
|
if ((Test-DatabaseListening).Ok) { return $true }
|
|
Start-Sleep -Seconds 2
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Test-PublicRuntimeHardDown {
|
|
param([string[]]$FailedNames)
|
|
|
|
$allLocalSurfacesDown = (
|
|
($FailedNames -contains "api") -and
|
|
($FailedNames -contains "web-preview") -and
|
|
($FailedNames -contains "voice-sidecars")
|
|
)
|
|
return ($FailedNames -contains "cloudflared") -or $allLocalSurfacesDown
|
|
}
|
|
|
|
# engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
|
# /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은"
|
|
# 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0).
|
|
# /ready는 게이트웨이 readiness 캐시(ENGINE_READY_TTL_SECONDS)를 그대로 쓰므로
|
|
# 매 주기 LLM 호출로 이어지지 않는다. 콜드 스폰 여유로 타임아웃만 넉넉히 준다.
|
|
$checks = @(
|
|
(Test-JsonHealth `
|
|
-Name "engine" `
|
|
-Uri "http://127.0.0.1:$EnginePort/ready" `
|
|
-IsHealthy { param($health) $health.ok -eq $true } `
|
|
-TimeoutSec 45 `
|
|
-Headers (Get-EngineHeaders)),
|
|
(Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 60),
|
|
(Test-JsonHealth `
|
|
-Name "voice-api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30),
|
|
(Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true }),
|
|
(Test-VoiceSidecarStack),
|
|
(Test-CloudflaredProcess),
|
|
(Test-DatabaseListening)
|
|
)
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
}
|
|
}
|
|
|
|
$failed = @($checks | Where-Object { -not $_.Ok })
|
|
if ($CheckOnly) {
|
|
if ($failed.Count -eq 0) {
|
|
Write-Output "healthy: $($checks.Name -join ', ')"
|
|
exit 0
|
|
}
|
|
Write-Output "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
|
|
exit 1
|
|
}
|
|
if ($failed.Count -eq 0) {
|
|
Set-FailCount 0
|
|
Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
|
|
exit 0
|
|
}
|
|
|
|
$failCount = (Get-FailCount) + 1
|
|
Set-FailCount $failCount
|
|
Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
|
|
|
|
# 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라
|
|
# 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다.
|
|
# 다만 cloudflared가 사라졌거나 API·web·voice sidecar가 함께 죽은 경우는 transient
|
|
# readiness가 아니라 공개 서비스 전체 소실이므로 첫 감지에서 즉시 복구한다.
|
|
$failedNames = @($failed | ForEach-Object { $_.Name })
|
|
$hardDown = Test-PublicRuntimeHardDown -FailedNames $failedNames
|
|
if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown) {
|
|
Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)"
|
|
exit 1
|
|
}
|
|
if ($hardDown -and $failCount -lt $FailuresBeforeRestart) {
|
|
Write-WatchdogLog "immediate restart: public runtime hard-down detected"
|
|
}
|
|
|
|
# DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당).
|
|
# 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에
|
|
# start를 반복 호출하는 것이 5분 주기 실패 로그의 정체였다.
|
|
if (-not (@($checks | Where-Object { $_.Name -eq "db" })[0]).Ok) {
|
|
Write-WatchdogLog "db down; restoring container '$DbContainer' before runtime restart"
|
|
if (Restore-DatabaseContainer) {
|
|
Write-WatchdogLog "db restored: 127.0.0.1:$DbPort"
|
|
} else {
|
|
Write-WatchdogLog "ERROR: db restore failed; skipping runtime restart (start script cannot fix a missing DB)"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
$startArgs = @{
|
|
Workspace = $resolvedSourceRoot
|
|
ApiPort = $ApiPort
|
|
WebPort = $WebPort
|
|
EnginePort = $EnginePort
|
|
WhisperPort = $WhisperPort
|
|
MeloTtsPort = $MeloTtsPort
|
|
Python = $Python
|
|
Cloudflared = $Cloudflared
|
|
CloudflaredConfig = $CloudflaredConfig
|
|
}
|
|
if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) {
|
|
$startArgs["SkipWebRestart"] = $true
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
$startArgs["SkipCloudflaredRestart"] = $true
|
|
}
|
|
|
|
try {
|
|
& $startScript @startArgs | ForEach-Object {
|
|
Write-WatchdogLog "$_"
|
|
}
|
|
} catch {
|
|
Write-WatchdogLog "restart failed: $($_.Exception.Message)"
|
|
throw
|
|
}
|
|
|
|
# 재시작 직후 첫 health는 readiness 콜드 스폰(10~20초)을 포함하므로 여유를 준다.
|
|
$apiAfter = Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 90
|
|
if (!$apiAfter.Ok) {
|
|
throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
|
|
}
|
|
|
|
$voiceApiAfter = Test-JsonHealth `
|
|
-Name "voice-api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
if (!$voiceApiAfter.Ok) {
|
|
throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)"
|
|
}
|
|
|
|
$webAfter = Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true } `
|
|
-TimeoutSec 20
|
|
if (!$webAfter.Ok) {
|
|
throw "Public web preview still unhealthy after restart: $($webAfter.Detail)"
|
|
}
|
|
|
|
$engineAfter = Test-JsonHealth `
|
|
-Name "engine" `
|
|
-Uri "http://127.0.0.1:$EnginePort/ready" `
|
|
-IsHealthy { param($health) $health.ok -eq $true } `
|
|
-TimeoutSec 60 `
|
|
-Headers (Get-EngineHeaders)
|
|
if (!$engineAfter.Ok) {
|
|
throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)"
|
|
}
|
|
|
|
$voiceSidecarsAfter = Test-VoiceSidecarStack
|
|
if (!$voiceSidecarsAfter.Ok) {
|
|
throw "Exact local voice sidecars still unhealthy after restart"
|
|
}
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$publicAfter = Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
if (!$publicAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)"
|
|
}
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$publicExtraAfter = Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
if (!$publicExtraAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)"
|
|
}
|
|
}
|
|
}
|
|
|
|
Set-FailCount 0
|
|
Write-WatchdogLog "restart verified"
|