전 저장소 리팩터링과 SSOT 정비

scripts/watch-public-runtime.ps1

@@ -9,6 +9,7 @@
   [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
   [string[]]$AdditionalPublicHealthUrls = @(),
   [string]$LogPath = "",
+  [int]$FailuresBeforeRestart = 3,
   [switch]$CheckOnly,
   [switch]$SkipPublicHealth,
   [switch]$SkipCloudflaredRestart
@@ -20,6 +21,23 @@ if (!$LogPath) {
   $LogPath = Join-Path $Workspace "public-runtime-watchdog.log"
 }
 
+# 연속 실패 카운터(재시작 debounce용). 워치독은 매 실행마다 새 프로세스라 파일로 유지한다.
+$FailCountPath = Join-Path $Workspace "public-runtime-watchdog.failcount"
+
+function Get-FailCount {
+  if (Test-Path $FailCountPath) {
+    $raw = (Get-Content -Raw -Path $FailCountPath -ErrorAction SilentlyContinue)
+    $n = 0
+    if ([int]::TryParse(($raw -replace '\s', ''), [ref]$n)) { return $n }
+  }
+  return 0
+}
+
+function Set-FailCount {
+  param([int]$Value)
+  Set-Content -Path $FailCountPath -Value $Value -Encoding ascii
+}
+
 function Write-WatchdogLog {
   param([string]$Message)
 
@@ -33,7 +51,7 @@ function Test-JsonHealth {
     [string]$Name,
     [string]$Uri,
     [scriptblock]$IsHealthy,
-    [int]$TimeoutSec = 10
+    [int]$TimeoutSec = 30
   )
 
   try {
@@ -105,27 +123,38 @@ if (!$SkipPublicHealth) {
     -Name "public-api" `
     -Uri $PublicHealthUrl `
     -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
-    -TimeoutSec 20
+    -TimeoutSec 30
   foreach ($url in $AdditionalPublicHealthUrls) {
     $checks += Test-JsonHealth `
       -Name "public-api:$url" `
       -Uri $url `
       -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
-      -TimeoutSec 20
+      -TimeoutSec 30
   }
 }
 
 $failed = @($checks | Where-Object { -not $_.Ok })
 if ($failed.Count -eq 0) {
+  Set-FailCount 0
   Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
   exit 0
 }
 
-Write-WatchdogLog "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
+$failCount = (Get-FailCount) + 1
+Set-FailCount $failCount
+Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
 if ($CheckOnly) {
   exit 1
 }
 
+# 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라
+# 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다.
+# 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다.
+if ($failCount -lt $FailuresBeforeRestart) {
+  Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)"
+  exit 1
+}
+
 $startArgs = @{
   Workspace = $Workspace
   ApiPort = $ApiPort
@@ -171,7 +200,7 @@ if (!$SkipPublicHealth) {
     -Name "public-api" `
     -Uri $PublicHealthUrl `
     -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
-    -TimeoutSec 20
+    -TimeoutSec 30
   if (!$publicAfter.Ok) {
     throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)"
   }
@@ -180,11 +209,12 @@ if (!$SkipPublicHealth) {
       -Name "public-api:$url" `
       -Uri $url `
       -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
-      -TimeoutSec 20
+      -TimeoutSec 30
     if (!$publicExtraAfter.Ok) {
       throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)"
     }
   }
 }
 
+Set-FailCount 0
 Write-WatchdogLog "restart verified"