재부팅 복구와 학습자 패널을 정리

This commit is contained in:
Yun Chan 2026-08-28 18:07:28 +09:00
parent dcb76a2373
commit 99779a6ab2
10 changed files with 442 additions and 25 deletions

View file

@ -6,6 +6,15 @@
[int]$WhisperPort = 9882,
[int]$MeloTtsPort = 9883,
[int]$VoiceSidecarReadySeconds = 300,
[ValidateRange(30, 600)]
[int]$ApiReadySeconds = 180,
[ValidateRange(30, 300)]
[int]$VoiceApiReadySeconds = 90,
[ValidateRange(60, 1800)]
[int]$WebBuildTimeoutSeconds = 600,
[string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock",
[ValidateRange(0, 300)]
[int]$RecoveryLockWaitSeconds = 0,
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
@ -29,6 +38,46 @@
$ErrorActionPreference = "Stop"
function Enter-RecoveryLock {
param(
[string]$LockPath,
[int]$WaitSeconds
)
if ([string]::IsNullOrWhiteSpace($LockPath)) {
throw "Public runtime recovery lock path is empty"
}
$resolvedLockPath = [System.IO.Path]::GetFullPath($LockPath)
$lockDirectory = [System.IO.Path]::GetDirectoryName($resolvedLockPath)
if ([string]::IsNullOrWhiteSpace($lockDirectory)) {
throw "Public runtime recovery lock path has no parent directory"
}
[System.IO.Directory]::CreateDirectory($lockDirectory) | Out-Null
$deadline = (Get-Date).AddSeconds($WaitSeconds)
do {
try {
return [System.IO.File]::Open(
$resolvedLockPath,
[System.IO.FileMode]::OpenOrCreate,
[System.IO.FileAccess]::ReadWrite,
[System.IO.FileShare]::None
)
} catch [System.IO.IOException] {
if ((Get-Date) -ge $deadline) {
throw "Another public runtime recovery is already in progress"
}
Start-Sleep -Seconds 1
}
} while ($true)
}
# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다.
# lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다.
$recoveryLock = Enter-RecoveryLock `
-LockPath $RecoveryLockPath `
-WaitSeconds $RecoveryLockWaitSeconds
try {
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
@ -344,6 +393,37 @@ function Stop-ProcessesBounded {
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
}
function Stop-ProcessTreeBounded {
param(
[int]$RootProcessId,
[int]$TimeoutSec,
[string]$Role
)
$taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe"
$previousErrorActionPreference = $ErrorActionPreference
try {
$ErrorActionPreference = "Continue"
& $taskkill @("/PID", "$RootProcessId", "/T", "/F") | Out-Null
$taskkillExit = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousErrorActionPreference
}
if ($taskkillExit -ne 0) {
throw "Failed to stop $Role process tree rooted at PID $RootProcessId (taskkill exit=$taskkillExit)"
}
$deadline = (Get-Date).AddSeconds($TimeoutSec)
do {
if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) {
return @($RootProcessId)
}
Start-Sleep -Milliseconds 200
} while ((Get-Date) -lt $deadline)
throw "Timed out stopping $Role process tree rooted at PID $RootProcessId"
}
function Get-UvicornProcessesByPort {
param(
[string]$AppImport,
@ -1470,11 +1550,11 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) {
$health.db -eq $true -and
$health.engine -eq $true
} `
-TimeoutSec 30
-TimeoutSec $ApiReadySeconds
$voiceHealth = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
-TimeoutSec 30
-TimeoutSec $VoiceApiReadySeconds
}
if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) {
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
@ -1493,8 +1573,17 @@ if (!$SkipWebRestart) {
-ArgumentList @("/c", "npm run build") `
-WorkingDirectory $WebDir `
-NoNewWindow `
-Wait `
-PassThru
if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) {
$null = @(
Stop-ProcessTreeBounded `
-RootProcessId $build.Id `
-TimeoutSec $ProcessStopTimeoutSeconds `
-Role "web build"
)
throw "Web build timed out after $WebBuildTimeoutSeconds seconds"
}
$build.Refresh()
if ($build.ExitCode -ne 0) {
throw "Web build failed with exit code $($build.ExitCode)"
}
@ -1734,3 +1823,12 @@ if (!$SkipWebRestart) {
}
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
} finally {
if ($null -ne $recoveryLock) {
try {
$recoveryLock.Dispose()
} catch {
Write-Warning "Public runtime recovery lock release failed: $($_.Exception.Message)"
}
}
}