재부팅 복구와 학습자 패널을 정리
This commit is contained in:
parent
dcb76a2373
commit
99779a6ab2
10 changed files with 442 additions and 25 deletions
|
|
@ -6,6 +6,15 @@
|
|||
[int]$WhisperPort = 9882,
|
||||
[int]$MeloTtsPort = 9883,
|
||||
[int]$VoiceSidecarReadySeconds = 300,
|
||||
[ValidateRange(30, 600)]
|
||||
[int]$ApiReadySeconds = 180,
|
||||
[ValidateRange(30, 300)]
|
||||
[int]$VoiceApiReadySeconds = 90,
|
||||
[ValidateRange(60, 1800)]
|
||||
[int]$WebBuildTimeoutSeconds = 600,
|
||||
[string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock",
|
||||
[ValidateRange(0, 300)]
|
||||
[int]$RecoveryLockWaitSeconds = 0,
|
||||
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
||||
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
||||
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
||||
|
|
@ -29,6 +38,46 @@
|
|||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Enter-RecoveryLock {
|
||||
param(
|
||||
[string]$LockPath,
|
||||
[int]$WaitSeconds
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($LockPath)) {
|
||||
throw "Public runtime recovery lock path is empty"
|
||||
}
|
||||
$resolvedLockPath = [System.IO.Path]::GetFullPath($LockPath)
|
||||
$lockDirectory = [System.IO.Path]::GetDirectoryName($resolvedLockPath)
|
||||
if ([string]::IsNullOrWhiteSpace($lockDirectory)) {
|
||||
throw "Public runtime recovery lock path has no parent directory"
|
||||
}
|
||||
[System.IO.Directory]::CreateDirectory($lockDirectory) | Out-Null
|
||||
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
||||
do {
|
||||
try {
|
||||
return [System.IO.File]::Open(
|
||||
$resolvedLockPath,
|
||||
[System.IO.FileMode]::OpenOrCreate,
|
||||
[System.IO.FileAccess]::ReadWrite,
|
||||
[System.IO.FileShare]::None
|
||||
)
|
||||
} catch [System.IO.IOException] {
|
||||
if ((Get-Date) -ge $deadline) {
|
||||
throw "Another public runtime recovery is already in progress"
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
} while ($true)
|
||||
}
|
||||
|
||||
# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다.
|
||||
# lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다.
|
||||
$recoveryLock = Enter-RecoveryLock `
|
||||
-LockPath $RecoveryLockPath `
|
||||
-WaitSeconds $RecoveryLockWaitSeconds
|
||||
|
||||
try {
|
||||
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
|
||||
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
|
||||
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
|
||||
|
|
@ -344,6 +393,37 @@ function Stop-ProcessesBounded {
|
|||
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
||||
}
|
||||
|
||||
function Stop-ProcessTreeBounded {
|
||||
param(
|
||||
[int]$RootProcessId,
|
||||
[int]$TimeoutSec,
|
||||
[string]$Role
|
||||
)
|
||||
|
||||
$taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe"
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
try {
|
||||
$ErrorActionPreference = "Continue"
|
||||
& $taskkill @("/PID", "$RootProcessId", "/T", "/F") | Out-Null
|
||||
$taskkillExit = $LASTEXITCODE
|
||||
} finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($taskkillExit -ne 0) {
|
||||
throw "Failed to stop $Role process tree rooted at PID $RootProcessId (taskkill exit=$taskkillExit)"
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
||||
do {
|
||||
if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) {
|
||||
return @($RootProcessId)
|
||||
}
|
||||
Start-Sleep -Milliseconds 200
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
throw "Timed out stopping $Role process tree rooted at PID $RootProcessId"
|
||||
}
|
||||
|
||||
function Get-UvicornProcessesByPort {
|
||||
param(
|
||||
[string]$AppImport,
|
||||
|
|
@ -1470,11 +1550,11 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
|||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
-TimeoutSec $ApiReadySeconds
|
||||
$voiceHealth = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
-TimeoutSec $VoiceApiReadySeconds
|
||||
}
|
||||
if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) {
|
||||
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
|
||||
|
|
@ -1493,8 +1573,17 @@ if (!$SkipWebRestart) {
|
|||
-ArgumentList @("/c", "npm run build") `
|
||||
-WorkingDirectory $WebDir `
|
||||
-NoNewWindow `
|
||||
-Wait `
|
||||
-PassThru
|
||||
if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) {
|
||||
$null = @(
|
||||
Stop-ProcessTreeBounded `
|
||||
-RootProcessId $build.Id `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds `
|
||||
-Role "web build"
|
||||
)
|
||||
throw "Web build timed out after $WebBuildTimeoutSeconds seconds"
|
||||
}
|
||||
$build.Refresh()
|
||||
if ($build.ExitCode -ne 0) {
|
||||
throw "Web build failed with exit code $($build.ExitCode)"
|
||||
}
|
||||
|
|
@ -1734,3 +1823,12 @@ if (!$SkipWebRestart) {
|
|||
}
|
||||
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
|
||||
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
|
||||
} finally {
|
||||
if ($null -ne $recoveryLock) {
|
||||
try {
|
||||
$recoveryLock.Dispose()
|
||||
} catch {
|
||||
Write-Warning "Public runtime recovery lock release failed: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue