190 lines
5.2 KiB
PowerShell
190 lines
5.2 KiB
PowerShell
param(
|
|
[string]$Workspace = "D:\workspace\vignette",
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[string[]]$AdditionalPublicHealthUrls = @("https://api-vnet.18ka.net/health"),
|
|
[string]$LogPath = "",
|
|
[switch]$CheckOnly,
|
|
[switch]$SkipPublicHealth,
|
|
[switch]$SkipCloudflaredRestart
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
if (!$LogPath) {
|
|
$LogPath = Join-Path $Workspace "public-runtime-watchdog.log"
|
|
}
|
|
|
|
function Write-WatchdogLog {
|
|
param([string]$Message)
|
|
|
|
$line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-ddTHH:mm:ssK"), $Message
|
|
Write-Output $line
|
|
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
|
}
|
|
|
|
function Test-JsonHealth {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 10
|
|
)
|
|
|
|
try {
|
|
$response = Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
$ok = [bool](& $IsHealthy $response)
|
|
$detail = $response | ConvertTo-Json -Compress -Depth 5
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $ok
|
|
Detail = $detail
|
|
}
|
|
} catch {
|
|
[pscustomobject]@{
|
|
Name = $Name
|
|
Ok = $false
|
|
Detail = $_.Exception.Message
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-CloudflaredProcess {
|
|
if ($SkipCloudflaredRestart) {
|
|
return [pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $true
|
|
Detail = "skipped"
|
|
}
|
|
}
|
|
|
|
$configLeaf = Split-Path -Leaf $CloudflaredConfig
|
|
$process = Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*$configLeaf*"
|
|
} |
|
|
Select-Object -First 1
|
|
|
|
[pscustomobject]@{
|
|
Name = "cloudflared"
|
|
Ok = $null -ne $process
|
|
Detail = if ($process) { "pid=$($process.ProcessId)" } else { "not running" }
|
|
}
|
|
}
|
|
|
|
$startScript = Join-Path $PSScriptRoot "start-public-runtime.ps1"
|
|
if (!(Test-Path $startScript)) {
|
|
throw "Start script not found at $startScript"
|
|
}
|
|
|
|
$checks = @(
|
|
(Test-JsonHealth `
|
|
-Name "engine" `
|
|
-Uri "http://127.0.0.1:$EnginePort/health" `
|
|
-IsHealthy { param($health) $health.ok -eq $true }),
|
|
(Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine }),
|
|
(Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true }),
|
|
(Test-CloudflaredProcess)
|
|
)
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 20
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$checks += Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 20
|
|
}
|
|
}
|
|
|
|
$failed = @($checks | Where-Object { -not $_.Ok })
|
|
if ($failed.Count -eq 0) {
|
|
Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
|
|
exit 0
|
|
}
|
|
|
|
Write-WatchdogLog "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
|
|
if ($CheckOnly) {
|
|
exit 1
|
|
}
|
|
|
|
$startArgs = @{
|
|
Workspace = $Workspace
|
|
ApiPort = $ApiPort
|
|
WebPort = $WebPort
|
|
EnginePort = $EnginePort
|
|
Python = $Python
|
|
Cloudflared = $Cloudflared
|
|
CloudflaredConfig = $CloudflaredConfig
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
$startArgs["SkipCloudflaredRestart"] = $true
|
|
}
|
|
|
|
try {
|
|
& $startScript @startArgs 2>&1 | ForEach-Object {
|
|
Write-WatchdogLog "$_"
|
|
}
|
|
} catch {
|
|
Write-WatchdogLog "restart failed: $($_.Exception.Message)"
|
|
throw
|
|
}
|
|
|
|
$apiAfter = Test-JsonHealth `
|
|
-Name "api" `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 20
|
|
if (!$apiAfter.Ok) {
|
|
throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
|
|
}
|
|
|
|
$webAfter = Test-JsonHealth `
|
|
-Name "web-preview" `
|
|
-Uri "http://127.0.0.1:$WebPort/" `
|
|
-IsHealthy { param($body) $true } `
|
|
-TimeoutSec 20
|
|
if (!$webAfter.Ok) {
|
|
throw "Public web preview still unhealthy after restart: $($webAfter.Detail)"
|
|
}
|
|
|
|
if (!$SkipPublicHealth) {
|
|
$publicAfter = Test-JsonHealth `
|
|
-Name "public-api" `
|
|
-Uri $PublicHealthUrl `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 20
|
|
if (!$publicAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicAfter.Detail)"
|
|
}
|
|
foreach ($url in $AdditionalPublicHealthUrls) {
|
|
$publicExtraAfter = Test-JsonHealth `
|
|
-Name "public-api:$url" `
|
|
-Uri $url `
|
|
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
|
-TimeoutSec 20
|
|
if (!$publicExtraAfter.Ok) {
|
|
throw "Public API tunnel still unhealthy after restart: $($publicExtraAfter.Detail)"
|
|
}
|
|
}
|
|
}
|
|
|
|
Write-WatchdogLog "restart verified"
|