param( [Parameter(Mandatory = $true)] [string]$StableSourceRoot, [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{40}$")] [string]$ExpectedSourceCommit, [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{40}$")] [string]$ExpectedSourceTree, [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{64}$")] [string]$ExpectedWatchdogSha256, [Parameter(Mandatory = $true)] [ValidatePattern("^[0-9a-fA-F]{64}$")] [string]$ExpectedStartScriptSha256, [int]$ApiPort = 8001, [int]$WebPort = 5174, [int]$EnginePort = 9099, [int]$WhisperPort = 9882, [int]$MeloTtsPort = 9883, [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health", [string[]]$AdditionalPublicHealthUrls = @(), # 게이트웨이가 shared secret으로 떠 있으면 /ready는 인증이 필요하다(/health만 면제). # 토큰이 없으면 401을 장애로 오판해 재시작 폭풍이 난다. [string]$EngineToken = $env:ENGINE_GATEWAY_SHARED_SECRET, [string]$LogPath = "", [int]$FailuresBeforeRestart = 3, [switch]$CheckOnly, [switch]$SkipPublicHealth, [switch]$SkipCloudflaredRestart ) $ErrorActionPreference = "Stop" $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path $expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" function Invoke-GitText { param([string[]]$Arguments) $value = & git.exe -C $resolvedSourceRoot @Arguments if ($LASTEXITCODE -ne 0) { throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')" } return (@($value) -join [Environment]::NewLine).Trim() } function Assert-StableSourceProvenance { if (-not (Test-Path -LiteralPath $expectedWatchdogPath -PathType Leaf)) { throw "Pinned watchdog script not found at $expectedWatchdogPath" } if (-not (Test-Path -LiteralPath $startScript -PathType Leaf)) { throw "Pinned start script not found at $startScript" } if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) { throw "Pinned voice sidecar probe not found at $voiceSidecarProbe" } $runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path if (-not [string]::Equals( $runningWatchdogPath, (Resolve-Path -LiteralPath $expectedWatchdogPath).Path, [System.StringComparison]::OrdinalIgnoreCase )) { throw "Watchdog is not executing from the pinned stable source root" } $gitRoot = Invoke-GitText -Arguments @("rev-parse", "--show-toplevel") $resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path if (-not [string]::Equals( $resolvedGitRoot, $resolvedSourceRoot, [System.StringComparison]::OrdinalIgnoreCase )) { throw "Stable source root does not match its Git toplevel" } $symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD $symbolicHeadExit = $LASTEXITCODE if ($symbolicHeadExit -eq 0) { throw "Stable source must be a detached HEAD, not branch $symbolicHead" } if ($symbolicHeadExit -ne 1) { throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)" } $actualCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD") $actualTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}") if ($actualCommit -ne $ExpectedSourceCommit.ToLowerInvariant()) { throw "Stable source commit drift: expected=$ExpectedSourceCommit actual=$actualCommit" } if ($actualTree -ne $ExpectedSourceTree.ToLowerInvariant()) { throw "Stable source tree drift: expected=$ExpectedSourceTree actual=$actualTree" } $dirty = Invoke-GitText -Arguments @("status", "--porcelain=v1", "--untracked-files=normal") if ($dirty) { throw "Stable source is not clean; refusing runtime recovery" } foreach ($relativePath in @( "scripts/watch-public-runtime.ps1", "scripts/start-public-runtime.ps1", "scripts/probe-public-voice-sidecars.py" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } $actualWatchdogSha256 = (Get-FileHash -LiteralPath $expectedWatchdogPath -Algorithm SHA256).Hash.ToLowerInvariant() $actualStartScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant() if ($actualWatchdogSha256 -ne $ExpectedWatchdogSha256.ToLowerInvariant()) { throw "Pinned watchdog SHA256 drift" } if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) { throw "Pinned start script SHA256 drift" } } # health probe, failcount 기록, 프로세스 재기동보다 먼저 source provenance를 닫는다. # 검증 실패는 운영 프로세스를 그대로 보존한 채 non-zero로 끝난다. Assert-StableSourceProvenance if (!$LogPath) { $LogPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.log" } # 연속 실패 카운터(재시작 debounce용). 워치독은 매 실행마다 새 프로세스라 파일로 유지한다. $FailCountPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.failcount" function Get-FailCount { if (Test-Path $FailCountPath) { $raw = (Get-Content -Raw -Path $FailCountPath -ErrorAction SilentlyContinue) $n = 0 if ([int]::TryParse(($raw -replace '\s', ''), [ref]$n)) { return $n } } return 0 } function Set-FailCount { param([int]$Value) Set-Content -Path $FailCountPath -Value $Value -Encoding ascii } function Write-WatchdogLog { param([string]$Message) $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-ddTHH:mm:ssK"), $Message Write-Output $line Add-Content -Path $LogPath -Value $line -Encoding UTF8 } function Test-JsonHealth { param( [string]$Name, [string]$Uri, [scriptblock]$IsHealthy, [int]$TimeoutSec = 30, [hashtable]$Headers = $null ) try { if ($Headers) { $response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec -Headers $Headers } else { $response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec } $ok = [bool](& $IsHealthy $response) $detail = $response | ConvertTo-Json -Compress -Depth 5 [pscustomobject]@{ Name = $Name Ok = $ok Detail = $detail } } catch { # 게이트웨이 /ready는 실패를 503 + JSON 본문으로 알린다. 본문을 버리면 로그에 # "(503)"만 남아 원인(만료 인증·플래그 오류)을 잃는다. $detail = $_.Exception.Message $body = "" if ($_.ErrorDetails -and $_.ErrorDetails.Message) { $body = $_.ErrorDetails.Message } elseif ($_.Exception.Response) { try { $reader = New-Object System.IO.StreamReader($_.Exception.Response.GetResponseStream()) $body = $reader.ReadToEnd() $reader.Dispose() } catch { $body = "" } } if ($body) { $detail = "$detail :: $body" } [pscustomobject]@{ Name = $Name Ok = $false Detail = $detail } } } function Get-EngineHeaders { if ($EngineToken) { return @{ "X-Vignette-Engine-Token" = $EngineToken } } return $null } function Test-VoiceApiReady { param([object]$Health) return ( $null -ne $Health -and $Health.status -eq "ok" -and $Health.available -eq $true -and $Health.stt_available -eq $true -and $Health.tts_available -eq $true -and $Health.stt_provider -eq "local_whisper" -and $Health.stt_model -eq "small" -and $Health.tts_provider -eq "melotts" -and $Health.tts_model -eq "melotts-korean" -and $Health.limits.uvicorn_ws_max_queue -eq 4 ) } function Test-VoiceSidecarStack { $probeArgs = @( "-X", "utf8", "-B", $voiceSidecarProbe, "--component", "all", "--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen", "--stt-provider", "local_whisper", "--stt-model", "small", "--stt-language", "ko", "--stt-device", "cpu", "--tts-url", "http://127.0.0.1:$MeloTtsPort", "--tts-provider", "melotts", "--tts-model", "melotts-korean", "--tts-language", "KR", "--timeout-seconds", "5" ) $output = @(& $Python @probeArgs 2>$null) $probeExit = $LASTEXITCODE return [pscustomobject]@{ Name = "voice-sidecars" Ok = $probeExit -eq 0 Detail = if ($probeExit -eq 0) { (@($output) -join "") } else { "exact readiness probe failed" } } } function Test-CloudflaredProcess { if ($SkipCloudflaredRestart) { return [pscustomobject]@{ Name = "cloudflared" Ok = $true Detail = "skipped" } } $configLeaf = Split-Path -Leaf $CloudflaredConfig $process = Get-CimInstance Win32_Process | Where-Object { $_.Name -eq "cloudflared.exe" -and $_.CommandLine -and $_.CommandLine -like "*$configLeaf*" } | Select-Object -First 1 [pscustomobject]@{ Name = "cloudflared" Ok = $null -ne $process Detail = if ($process) { "pid=$($process.ProcessId)" } else { "not running" } } } # engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. # /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은" # 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0). # /ready는 게이트웨이 readiness 캐시(ENGINE_READY_TTL_SECONDS)를 그대로 쓰므로 # 매 주기 LLM 호출로 이어지지 않는다. 콜드 스폰 여유로 타임아웃만 넉넉히 준다. $checks = @( (Test-JsonHealth ` -Name "engine" ` -Uri "http://127.0.0.1:$EnginePort/ready" ` -IsHealthy { param($health) $health.ok -eq $true } ` -TimeoutSec 45 ` -Headers (Get-EngineHeaders)), (Test-JsonHealth ` -Name "api" ` -Uri "http://127.0.0.1:$ApiPort/health" ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` -TimeoutSec 60), (Test-JsonHealth ` -Name "voice-api" ` -Uri "http://127.0.0.1:$ApiPort/voice/health" ` -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` -TimeoutSec 30), (Test-JsonHealth ` -Name "web-preview" ` -Uri "http://127.0.0.1:$WebPort/" ` -IsHealthy { param($body) $true }), (Test-VoiceSidecarStack), (Test-CloudflaredProcess) ) if (!$SkipPublicHealth) { $checks += Test-JsonHealth ` -Name "public-api" ` -Uri $PublicHealthUrl ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } ` -TimeoutSec 30 foreach ($url in $AdditionalPublicHealthUrls) { $checks += Test-JsonHealth ` -Name "public-api:$url" ` -Uri $url ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } ` -TimeoutSec 30 } } $failed = @($checks | Where-Object { -not $_.Ok }) if ($failed.Count -eq 0) { Set-FailCount 0 Write-WatchdogLog "healthy: $($checks.Name -join ', ')" exit 0 } $failCount = (Get-FailCount) + 1 Set-FailCount $failCount Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))" if ($CheckOnly) { exit 1 } # 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라 # 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다. # 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다. if ($failCount -lt $FailuresBeforeRestart) { Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)" exit 1 } $startArgs = @{ Workspace = $resolvedSourceRoot ApiPort = $ApiPort WebPort = $WebPort EnginePort = $EnginePort WhisperPort = $WhisperPort MeloTtsPort = $MeloTtsPort Python = $Python Cloudflared = $Cloudflared CloudflaredConfig = $CloudflaredConfig } if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) { $startArgs["SkipWebRestart"] = $true } if ($SkipCloudflaredRestart) { $startArgs["SkipCloudflaredRestart"] = $true } try { & $startScript @startArgs 2>&1 | ForEach-Object { Write-WatchdogLog "$_" } } catch { Write-WatchdogLog "restart failed: $($_.Exception.Message)" throw } # 재시작 직후 첫 health는 readiness 콜드 스폰(10~20초)을 포함하므로 여유를 준다. $apiAfter = Test-JsonHealth ` -Name "api" ` -Uri "http://127.0.0.1:$ApiPort/health" ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } ` -TimeoutSec 90 if (!$apiAfter.Ok) { throw "Public API still unhealthy after restart: $($apiAfter.Detail)" } $voiceApiAfter = Test-JsonHealth ` -Name "voice-api" ` -Uri "http://127.0.0.1:$ApiPort/voice/health" ` -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` -TimeoutSec 30 if (!$voiceApiAfter.Ok) { throw "Public voice API still does not expose the exact local provider/model contract after restart: $($voiceApiAfter.Detail)" } $webAfter = Test-JsonHealth ` -Name "web-preview" ` -Uri "http://127.0.0.1:$WebPort/" ` -IsHealthy { param($body) $true } ` -TimeoutSec 20 if (!$webAfter.Ok) { throw "Public web preview still unhealthy after restart: $($webAfter.Detail)" } $engineAfter = Test-JsonHealth ` -Name "engine" ` -Uri "http://127.0.0.1:$EnginePort/ready" ` -IsHealthy { param($health) $health.ok -eq $true } ` -TimeoutSec 60 ` -Headers (Get-EngineHeaders) if (!$engineAfter.Ok) { throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)" } $voiceSidecarsAfter = Test-VoiceSidecarStack if (!$voiceSidecarsAfter.Ok) { throw "Exact local voice sidecars still unhealthy after restart" } if (!$SkipPublicHealth) { $publicAfter = Test-JsonHealth ` -Name "public-api" ` -Uri $PublicHealthUrl ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } ` -TimeoutSec 30 if (!$publicAfter.Ok) { throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)" } foreach ($url in $AdditionalPublicHealthUrls) { $publicExtraAfter = Test-JsonHealth ` -Name "public-api:$url" ` -Uri $url ` -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } ` -TimeoutSec 30 if (!$publicExtraAfter.Ok) { throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)" } } } Set-FailCount 0 Write-WatchdogLog "restart verified"