438 lines
14 KiB
PowerShell
438 lines
14 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$StableSourceRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceCommit,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceTree,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedWatchdogSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedStartScriptSha256,
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[string[]]$AdditionalPublicHealthUrls = @(),
|
|
# 게이트웨이가 shared secret으로 떠 있으면 /ready는 인증이 필요하다(/health만 면제).
|
|
# 토큰이 없으면 401을 장애로 오판해 재시작 폭풍이 난다.
|
|
[string]$EngineToken = $env:ENGINE_GATEWAY_SHARED_SECRET,
|
|
[string]$LogPath = "",
|
|
[int]$FailuresBeforeRestart = 3,
|
|
[switch]$CheckOnly,
|
|
[switch]$SkipPublicHealth,
|
|
[switch]$SkipCloudflaredRestart
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|
$expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
|
|
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
|
|
|
function Invoke-GitText {
|
|
param([string[]]$Arguments)
|
|
|
|
$value = & git.exe -C $resolvedSourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Assert-StableSourceProvenance {
|
|
if (-not (Test-Path -LiteralPath $expectedWatchdogPath -PathType Leaf)) {
|
|
throw "Pinned watchdog script not found at $expectedWatchdogPath"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) {
|
|
throw "Pinned start script not found at $startScript"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) {
|
|
throw "Pinned voice sidecar probe not found at $voiceSidecarProbe"
|
|
}
|
|
|
|
$runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningWatchdogPath,
|
|
(Resolve-Path -LiteralPath $expectedWatchdogPath).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Watchdog is not executing from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-GitText -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Stable source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Stable source must be a detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $ExpectedSourceCommit.ToLowerInvariant()) {
|
|
throw "Stable source commit drift: expected=$ExpectedSourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $ExpectedSourceTree.ToLowerInvariant()) {
|
|
throw "Stable source tree drift: expected=$ExpectedSourceTree actual=$actualTree"
|
|
}
|
|
|
|
$dirty = Invoke-GitText -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Stable source is not clean; refusing runtime recovery"
|
|
}
|
|
|
|
foreach ($relativePath in @(
|
|
"scripts/watch-public-runtime.ps1",
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/probe-public-voice-sidecars.py"
|
|
)) {
|
|
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
|
}
|
|
|
|
$actualWatchdogSha256 = (Get-FileHash -LiteralPath $expectedWatchdogPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$actualStartScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualWatchdogSha256 -ne $ExpectedWatchdogSha256.ToLowerInvariant()) {
|
|
throw "Pinned watchdog SHA256 drift"
|
|
}
|
|
if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) {
|
|
throw "Pinned start script SHA256 drift"
|
|
}
|
|
}
|
|
|
|
# health probe, failcount 기록, 프로세스 재기동보다 먼저 source provenance를 닫는다.
|
|
# 검증 실패는 운영 프로세스를 그대로 보존한 채 non-zero로 끝난다.
|
|
Assert-StableSourceProvenance
|
|
|
|
if (!$LogPath) {
|
|
$LogPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.log"
|
|
}
|
|
|
|
# 연속 실패 카운터(재시작 debounce용). 워치독은 매 실행마다 새 프로세스라 파일로 유지한다.
|
|
$FailCountPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.failcount"
|
|
|
|
function Get-FailCount {
|
|
if (Test-Path $FailCountPath) {
|
|
$raw = (Get-Content -Raw -Path $FailCountPath -ErrorAction SilentlyContinue)
|
|
$n = 0
|
|
if ([int]::TryParse(($raw -replace '\s', ''), [ref]$n)) { return $n }
|
|
}
|
|
return 0
|
|
}
|
|
|
|
function Set-FailCount {
|
|
param([int]$Value)
|
|
Set-Content -Path $FailCountPath -Value $Value -Encoding ascii
|
|
}
|
|
|
|
function Write-WatchdogLog {
|
|
param([string]$Message)
|
|
|
|
$line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-ddTHH:mm:ssK"), $Message
|
|
Write-Output $line
|
|
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
|
}
|
|
|
|
function Test-JsonHealth {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 30,
|
|
[hashtable]$Headers = $null
|
|
)
|
|
|
|
try {
|
|
if ($Headers) {
|
|
$response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec -Headers $Headers
|
|
} else {
|
|
$response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
}
|
|
$ok = [bool](& $IsHealthy $response)
|
|
$detail = $response | ConvertTo-Json -Compress -Depth 5
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $ok
|
|
Detail = $detail
|
|
}
|
|
} catch {
|
|
# 게이트웨이 /ready는 실패를 503 + JSON 본문으로 알린다. 본문을 버리면 로그에
|
|
# "(503)"만 남아 원인(만료 인증·플래그 오류)을 잃는다.
|
|
$detail = $_.Exception.Message
|
|
$body = ""
|
|
if ($_.ErrorDetails -and $_.ErrorDetails.Message) {
|
|
$body = $_.ErrorDetails.Message
|
|
} elseif ($_.Exception.Response) {
|
|
try {
|
|
$reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream())
|
|
$body = $reader.ReadToEnd()
|
|
$reader.Dispose()
|
|
} catch {
|
|
$body = ""
|
|
}
|
|
}
|
|
if ($body) {
|
|
$detail = "$detail :: $body"
|
|
}
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $false
|
|
Detail = $detail
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-EngineHeaders {
|
|
if ($EngineToken) {
|
|
return @{ "X-Vignette-Engine-Token" = $EngineToken }
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq "small" -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq "melotts-korean" -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-VoiceSidecarStack {
|
|
$probeArgs = @(
|
|
"-X", "utf8", "-B", $voiceSidecarProbe,
|
|
"--component", "all",
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", "small",
|
|
"--stt-language", "ko",
|
|
"--stt-device", "cpu",
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", "melotts-korean",
|
|
"--tts-language", "KR",
|
|
"--timeout-seconds", "5"
|
|
)
|
|
$output = @(& $Python @probeArgs 2>$null)
|
|
$probeExit = $LASTEXITCODE
|
|
return [pscustomobject]@{
|
|
Name = "voice-sidecars"
|
|
Ok = $probeExit -eq 0
|
|
Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" }
|
|
}
|
|
}
|
|
|
|
function Test-CloudflaredProcess {
|
|
if ($SkipCloudflaredRestart) {
|
|
return [pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $true
|
|
Detail = "skipped"
|
|
}
|
|
}
|
|
|
|
$configLeaf = Split-Path -Leaf $CloudflaredConfig
|
|
$process = Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*$configLeaf*"
|
|
} |
|
|
Select-Object -First 1
|
|
|
|
[pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $null -ne $process
|
|
Detail = if ($process) { "pid=$($process.ProcessId)" } else { "not running" }
|
|
}
|
|
}
|
|
|
|
# engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
|
# /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은"
|
|
# 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0).
|
|
# /ready는 게이트웨이 readiness 캐시(ENGINE_READY_TTL_SECONDS)를 그대로 쓰므로
|
|
# 매 주기 LLM 호출로 이어지지 않는다. 콜드 스폰 여유로 타임아웃만 넉넉히 준다.
|
|
$checks = @(
|
|
(Test-JsonHealth `
|
|
-Name "engine" `
|
|
-Uri "http://127.0.0.1:$EnginePort/ready" `
|
|
-IsHealthy { param($health) $health.ok -eq $true } `
|
|
-TimeoutSec 45 `
|
|
-Headers (Get-EngineHeaders)),
|
|
(Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 60),
|
|
(Test-JsonHealth `
|
|
-Name "voice-api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30),
|
|
(Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true }),
|
|
(Test-VoiceSidecarStack),
|
|
(Test-CloudflaredProcess)
|
|
)
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
}
|
|
}
|
|
|
|
$failed = @($checks | Where-Object { -not $_.Ok })
|
|
if ($failed.Count -eq 0) {
|
|
Set-FailCount 0
|
|
Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
|
|
exit 0
|
|
}
|
|
|
|
$failCount = (Get-FailCount) + 1
|
|
Set-FailCount $failCount
|
|
Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
|
|
if ($CheckOnly) {
|
|
exit 1
|
|
}
|
|
|
|
# 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라
|
|
# 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다.
|
|
# 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다.
|
|
if ($failCount -lt $FailuresBeforeRestart) {
|
|
Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)"
|
|
exit 1
|
|
}
|
|
|
|
$startArgs = @{
|
|
Workspace = $resolvedSourceRoot
|
|
ApiPort = $ApiPort
|
|
WebPort = $WebPort
|
|
EnginePort = $EnginePort
|
|
WhisperPort = $WhisperPort
|
|
MeloTtsPort = $MeloTtsPort
|
|
Python = $Python
|
|
Cloudflared = $Cloudflared
|
|
CloudflaredConfig = $CloudflaredConfig
|
|
}
|
|
if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) {
|
|
$startArgs["SkipWebRestart"] = $true
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
$startArgs["SkipCloudflaredRestart"] = $true
|
|
}
|
|
|
|
try {
|
|
& $startScript @startArgs 2>&1 | ForEach-Object {
|
|
Write-WatchdogLog "$_"
|
|
}
|
|
} catch {
|
|
Write-WatchdogLog "restart failed: $($_.Exception.Message)"
|
|
throw
|
|
}
|
|
|
|
# 재시작 직후 첫 health는 readiness 콜드 스폰(10~20초)을 포함하므로 여유를 준다.
|
|
$apiAfter = Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 90
|
|
if (!$apiAfter.Ok) {
|
|
throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
|
|
}
|
|
|
|
$voiceApiAfter = Test-JsonHealth `
|
|
-Name "voice-api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
if (!$voiceApiAfter.Ok) {
|
|
throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)"
|
|
}
|
|
|
|
$webAfter = Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true } `
|
|
-TimeoutSec 20
|
|
if (!$webAfter.Ok) {
|
|
throw "Public web preview still unhealthy after restart: $($webAfter.Detail)"
|
|
}
|
|
|
|
$engineAfter = Test-JsonHealth `
|
|
-Name "engine" `
|
|
-Uri "http://127.0.0.1:$EnginePort/ready" `
|
|
-IsHealthy { param($health) $health.ok -eq $true } `
|
|
-TimeoutSec 60 `
|
|
-Headers (Get-EngineHeaders)
|
|
if (!$engineAfter.Ok) {
|
|
throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)"
|
|
}
|
|
|
|
$voiceSidecarsAfter = Test-VoiceSidecarStack
|
|
if (!$voiceSidecarsAfter.Ok) {
|
|
throw "Exact local voice sidecars still unhealthy after restart"
|
|
}
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$publicAfter = Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
if (!$publicAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)"
|
|
}
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$publicExtraAfter = Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
|
|
-TimeoutSec 30
|
|
if (!$publicExtraAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)"
|
|
}
|
|
}
|
|
}
|
|
|
|
Set-FailCount 0
|
|
Write-WatchdogLog "restart verified"
|