vignette/scripts/watch-public-runtime.ps1

param(
  [string]$Workspace = "D:\workspace\vignette",
  [int]$ApiPort = 8001,
  [int]$WebPort = 5174,
  [int]$EnginePort = 9099,
  [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
  [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
  [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
  [string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
  [string[]]$AdditionalPublicHealthUrls = @(),
  [string]$LogPath = "",
  [int]$FailuresBeforeRestart = 3,
  [switch]$CheckOnly,
  [switch]$SkipPublicHealth,
  [switch]$SkipCloudflaredRestart
)

$ErrorActionPreference = "Stop"

if (!$LogPath) {
  $LogPath = Join-Path $Workspace "public-runtime-watchdog.log"
}

# 연속 실패 카운터(재시작 debounce용). 워치독은 매 실행마다 새 프로세스라 파일로 유지한다.
$FailCountPath = Join-Path $Workspace "public-runtime-watchdog.failcount"

function Get-FailCount {
  if (Test-Path $FailCountPath) {
    $raw = (Get-Content -Raw -Path $FailCountPath -ErrorAction SilentlyContinue)
    $n = 0
    if ([int]::TryParse(($raw -replace '\s', ''), [ref]$n)) { return $n }
  }
  return 0
}

function Set-FailCount {
  param([int]$Value)
  Set-Content -Path $FailCountPath -Value $Value -Encoding ascii
}

function Write-WatchdogLog {
  param([string]$Message)

  $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-ddTHH:mm:ssK"), $Message
  Write-Output $line
  Add-Content -Path $LogPath -Value $line -Encoding UTF8
}

function Test-JsonHealth {
  param(
    [string]$Name,
    [string]$Uri,
    [scriptblock]$IsHealthy,
    [int]$TimeoutSec = 30
  )

  try {
    $response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
    $ok = [bool](& $IsHealthy $response)
    $detail = $response | ConvertTo-Json -Compress -Depth 5
    [pscustomobject]@{
      Name = $Name
      Ok = $ok
      Detail = $detail
    }
  } catch {
    [pscustomobject]@{
      Name = $Name
      Ok = $false
      Detail = $_.Exception.Message
    }
  }
}

function Test-CloudflaredProcess {
  if ($SkipCloudflaredRestart) {
    return [pscustomobject]@{
      Name = "cloudflared"
      Ok = $true
      Detail = "skipped"
    }
  }

  $configLeaf = Split-Path -Leaf $CloudflaredConfig
  $process = Get-CimInstance Win32_Process |
    Where-Object {
      $_.Name -eq "cloudflared.exe" -and
      $_.CommandLine -and
      $_.CommandLine -like "*$configLeaf*"
    } |
    Select-Object -First 1

  [pscustomobject]@{
    Name = "cloudflared"
    Ok = $null -ne $process
    Detail = if ($process) { "pid=$($process.ProcessId)" } else { "not running" }
  }
}

$startScript = Join-Path $PSScriptRoot "start-public-runtime.ps1"
if (!(Test-Path $startScript)) {
  throw "Start script not found at $startScript"
}

$checks = @(
  (Test-JsonHealth `
    -Name "engine" `
    -Uri "http://127.0.0.1:$EnginePort/health" `
    -IsHealthy { param($health) $health.ok -eq $true }),
  (Test-JsonHealth `
    -Name "api" `
    -Uri "http://127.0.0.1:$ApiPort/health" `
    -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db }),
  (Test-JsonHealth `
    -Name "web-preview" `
    -Uri "http://127.0.0.1:$WebPort/" `
    -IsHealthy { param($body) $true }),
  (Test-CloudflaredProcess)
)

if (!$SkipPublicHealth) {
  $checks += Test-JsonHealth `
    -Name "public-api" `
    -Uri $PublicHealthUrl `
    -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
    -TimeoutSec 30
  foreach ($url in $AdditionalPublicHealthUrls) {
    $checks += Test-JsonHealth `
      -Name "public-api:$url" `
      -Uri $url `
      -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
      -TimeoutSec 30
  }
}

$failed = @($checks | Where-Object { -not $_.Ok })
if ($failed.Count -eq 0) {
  Set-FailCount 0
  Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
  exit 0
}

$failCount = (Get-FailCount) + 1
Set-FailCount $failCount
Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
if ($CheckOnly) {
  exit 1
}

# 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라
# 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다.
# 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다.
if ($failCount -lt $FailuresBeforeRestart) {
  Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)"
  exit 1
}

$startArgs = @{
  Workspace = $Workspace
  ApiPort = $ApiPort
  WebPort = $WebPort
  EnginePort = $EnginePort
  Python = $Python
  Cloudflared = $Cloudflared
  CloudflaredConfig = $CloudflaredConfig
}
if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) {
  $startArgs["SkipWebRestart"] = $true
}
if ($SkipCloudflaredRestart) {
  $startArgs["SkipCloudflaredRestart"] = $true
}

try {
  & $startScript @startArgs 2>&1 | ForEach-Object {
    Write-WatchdogLog "$_"
  }
} catch {
  Write-WatchdogLog "restart failed: $($_.Exception.Message)"
  throw
}

$apiAfter = Test-JsonHealth `
  -Name "api" `
  -Uri "http://127.0.0.1:$ApiPort/health" `
  -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
  -TimeoutSec 20
if (!$apiAfter.Ok) {
  throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
}

$webAfter = Test-JsonHealth `
  -Name "web-preview" `
  -Uri "http://127.0.0.1:$WebPort/" `
  -IsHealthy { param($body) $true } `
  -TimeoutSec 20
if (!$webAfter.Ok) {
  throw "Public web preview still unhealthy after restart: $($webAfter.Detail)"
}

$engineAfter = Test-JsonHealth `
  -Name "engine" `
  -Uri "http://127.0.0.1:$EnginePort/health" `
  -IsHealthy { param($health) $health.ok -eq $true } `
  -TimeoutSec 20
if (!$engineAfter.Ok) {
  throw "Engine gateway still unhealthy after isolated restart: $($engineAfter.Detail)"
}

if (!$SkipPublicHealth) {
  $publicAfter = Test-JsonHealth `
    -Name "public-api" `
    -Uri $PublicHealthUrl `
    -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
    -TimeoutSec 30
  if (!$publicAfter.Ok) {
    throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)"
  }
  foreach ($url in $AdditionalPublicHealthUrls) {
    $publicExtraAfter = Test-JsonHealth `
      -Name "public-api:$url" `
      -Uri $url `
      -IsHealthy { param($health) $health.environment -eq "prod" -and $health.db } `
      -TimeoutSec 30
    if (!$publicExtraAfter.Ok) {
      throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)"
    }
  }
}

Set-FailCount 0
Write-WatchdogLog "restart verified"