315 lines
12 KiB
PowerShell
315 lines
12 KiB
PowerShell
# boot-public-runtime.ps1
|
|
# 목적: PC 재부팅/재로그온 후 퍼블릭 런타임을 자동으로 복구한다.
|
|
# 순서: Docker Desktop 데몬 대기 -> postgres(vignette-dev-db) 기동 ->
|
|
# 관리자/인증 API 가 이미 healthy 이고 엔진도 healthy 면 스킵, 아니면
|
|
# start-public-runtime.ps1(-SkipWebRestart) 로 필요한 프로세스만 복구한다.
|
|
# 멱등: 어느 단계든 이미 살아있으면 건드리지 않는다. 수동으로 여러 번 실행해도 안전.
|
|
#
|
|
# 등록(로그온 시 자동 실행, 숨김 창)은 register-boot-task.ps1가 생성하는
|
|
# commit/tree/script SHA pin 인자를 사용한다. 핀 없는 직접 실행은 fail-closed한다.
|
|
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$StableSourceRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceCommit,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{40}$")]
|
|
[string]$ExpectedSourceTree,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedBootScriptSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
|
[string]$ExpectedStartScriptSha256,
|
|
[string]$DockerDesktop = "C:\Program Files\Docker\Docker\Docker Desktop.exe",
|
|
[int]$DaemonTimeoutSec = 360,
|
|
[int]$DbTimeoutSec = 90,
|
|
[int]$DbPort = 55432,
|
|
[int]$ApiPort = 8001,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$BootLog = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|
$expectedBootScript = Join-Path $resolvedSourceRoot "scripts\boot-public-runtime.ps1"
|
|
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
|
|
|
function Invoke-GitText {
|
|
param([string[]]$Arguments)
|
|
|
|
$value = & git.exe -C $resolvedSourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Assert-StableSourceProvenance {
|
|
foreach ($requiredScript in @($expectedBootScript, $startScript, $voiceSidecarProbe)) {
|
|
if (-not (Test-Path -LiteralPath $requiredScript -PathType Leaf)) {
|
|
throw "Pinned public runtime script not found at $requiredScript"
|
|
}
|
|
}
|
|
|
|
$runningBootScript = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningBootScript,
|
|
(Resolve-Path -LiteralPath $expectedBootScript).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Boot recovery is not executing from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-GitText -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Stable source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Stable source must be a detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $ExpectedSourceCommit.ToLowerInvariant()) {
|
|
throw "Stable source commit drift: expected=$ExpectedSourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $ExpectedSourceTree.ToLowerInvariant()) {
|
|
throw "Stable source tree drift: expected=$ExpectedSourceTree actual=$actualTree"
|
|
}
|
|
|
|
$dirty = Invoke-GitText -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Stable source is not clean; refusing boot recovery"
|
|
}
|
|
foreach ($relativePath in @(
|
|
"scripts/boot-public-runtime.ps1",
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/probe-public-voice-sidecars.py"
|
|
)) {
|
|
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
|
}
|
|
|
|
$actualBootScriptSha256 = (Get-FileHash -LiteralPath $expectedBootScript -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$actualStartScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualBootScriptSha256 -ne $ExpectedBootScriptSha256.ToLowerInvariant()) {
|
|
throw "Pinned boot script SHA256 drift"
|
|
}
|
|
if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) {
|
|
throw "Pinned start script SHA256 drift"
|
|
}
|
|
}
|
|
|
|
# Docker/DB/process mutation보다 먼저 stable source를 매 실행 재검증한다.
|
|
Assert-StableSourceProvenance
|
|
|
|
if (!$BootLog) {
|
|
$BootLog = Join-Path $resolvedSourceRoot "boot-public-runtime.log"
|
|
}
|
|
|
|
$ErrorActionPreference = "Continue" # provenance 이후 부트 복구는 끝까지 로깅한다.
|
|
|
|
function Write-BootLog([string]$Message) {
|
|
$line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Message
|
|
try {
|
|
Add-Content -LiteralPath $BootLog -Value $line -Encoding UTF8
|
|
} catch {
|
|
# 로그 파일이 잠겨도 부팅은 계속한다.
|
|
}
|
|
}
|
|
|
|
function Test-DockerDaemon([int]$TimeoutSec = 10) {
|
|
try {
|
|
$ver = & docker.exe info --format '{{.ServerVersion}}' 2>$null
|
|
if ($LASTEXITCODE -eq 0 -and $ver) { return $true }
|
|
} catch {}
|
|
return $false
|
|
}
|
|
|
|
function Test-Tcp([string]$Host_, [int]$Port) {
|
|
try {
|
|
$t = (Test-NetConnection -ComputerName $Host_ -Port $Port -WarningAction SilentlyContinue)
|
|
return [bool]$t.TcpTestSucceeded
|
|
} catch { return $false }
|
|
}
|
|
|
|
function Test-ApiControlPlaneHealthy {
|
|
# HttpWebRequest + Proxy=$null: WININET/시스템 프록시에 영향받지 않는 가장 직결적인 검사.
|
|
# 비대화형 스케줄러 컨텍스트에서도 127.0.0.1 로 직접 연결한다. 3회 재시도.
|
|
for ($i = 1; $i -le 3; $i++) {
|
|
try {
|
|
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/health")
|
|
$req.Timeout = 5000
|
|
$req.ReadWriteTimeout = 5000
|
|
$req.Proxy = $null
|
|
$resp = $req.GetResponse()
|
|
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
|
|
$body = $reader.ReadToEnd()
|
|
$reader.Close(); $resp.Close()
|
|
$h = $body | ConvertFrom-Json
|
|
if ($h.environment -eq "prod" -and $h.db -eq $true -and $h.engine -eq $true) { return $true }
|
|
Write-BootLog (" health probe attempt {0}: not-healthy body={1}" -f $i, $body)
|
|
return $false
|
|
} catch {
|
|
Write-BootLog (" health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
|
|
Start-Sleep -Seconds 2
|
|
}
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Test-EngineHealthy {
|
|
try {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$EnginePort/health" -TimeoutSec 10
|
|
return $response.ok -eq $true
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq "small" -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq "melotts-korean" -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-VoiceApiHealthy {
|
|
for ($i = 1; $i -le 3; $i++) {
|
|
try {
|
|
$req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$ApiPort/voice/health")
|
|
$req.Timeout = 5000
|
|
$req.ReadWriteTimeout = 5000
|
|
$req.Proxy = $null
|
|
$resp = $req.GetResponse()
|
|
$reader = New-Object System.IO.StreamReader($resp.GetResponseStream())
|
|
$body = $reader.ReadToEnd()
|
|
$reader.Close(); $resp.Close()
|
|
$health = $body | ConvertFrom-Json
|
|
if (Test-VoiceApiReady -Health $health) { return $true }
|
|
Write-BootLog (" voice health probe attempt {0}: not exact local provider/model body={1}" -f $i, $body)
|
|
return $false
|
|
} catch {
|
|
Write-BootLog (" voice health probe attempt {0} failed: {1}" -f $i, $_.Exception.Message)
|
|
Start-Sleep -Seconds 2
|
|
}
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Test-VoiceSidecarStack {
|
|
$probeArgs = @(
|
|
"-X", "utf8", "-B", $voiceSidecarProbe,
|
|
"--component", "all",
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", "small",
|
|
"--stt-language", "ko",
|
|
"--stt-device", "cpu",
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", "melotts-korean",
|
|
"--tts-language", "KR",
|
|
"--timeout-seconds", "5"
|
|
)
|
|
& $Python @probeArgs 1>$null 2>$null
|
|
return $LASTEXITCODE -eq 0
|
|
}
|
|
|
|
Write-BootLog "================ boot start ================"
|
|
|
|
# 1) Docker 데몬(내려가 있으면 Docker Desktop 기동 후 대기)
|
|
if (-not (Test-DockerDaemon)) {
|
|
Write-BootLog "docker daemon down; launching Docker Desktop"
|
|
if (Test-Path -LiteralPath $DockerDesktop) {
|
|
Start-Process -FilePath $DockerDesktop | Out-Null
|
|
} else {
|
|
Write-BootLog "ERROR: Docker Desktop.exe not found at $DockerDesktop"
|
|
exit 1
|
|
}
|
|
$deadline = (Get-Date).AddSeconds($DaemonTimeoutSec)
|
|
while ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds 5
|
|
if (Test-DockerDaemon) { break }
|
|
}
|
|
}
|
|
if (-not (Test-DockerDaemon)) {
|
|
Write-BootLog "ERROR: docker daemon did not come up within ${DaemonTimeoutSec}s"
|
|
exit 1
|
|
}
|
|
Write-BootLog "docker daemon up"
|
|
|
|
# 2) postgres 컨테이너(restart 정책 백업 + 명시 기동) 후 포트 대기
|
|
$null = & docker.exe update --restart unless-stopped vignette-dev-db 2>$null
|
|
$null = & docker.exe start vignette-dev-db 2>$null
|
|
$dbDeadline = (Get-Date).AddSeconds($DbTimeoutSec)
|
|
while ((Get-Date) -lt $dbDeadline) {
|
|
if (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort) { break }
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) {
|
|
Write-BootLog "ERROR: postgres not listening on 127.0.0.1:$DbPort"
|
|
exit 1
|
|
}
|
|
Write-BootLog "postgres 127.0.0.1:$DbPort up"
|
|
|
|
# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지)
|
|
if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
|
Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart"
|
|
} else {
|
|
Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart"
|
|
$out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
|
|
-Workspace $resolvedSourceRoot `
|
|
-Python $Python `
|
|
-EnginePort $EnginePort `
|
|
-WhisperPort $WhisperPort `
|
|
-MeloTtsPort $MeloTtsPort `
|
|
-SkipWebRestart 2>&1
|
|
$out | ForEach-Object { Write-BootLog (" pub> " + $_) }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-BootLog "ERROR: start-public-runtime.ps1 exit $LASTEXITCODE"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
# 4) 최종 확인
|
|
if (Test-ApiControlPlaneHealthy) {
|
|
if ((Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) {
|
|
Write-BootLog "boot OK: control plane, engine, exact local voice API, and sidecars healthy"
|
|
exit 0
|
|
} else {
|
|
Write-BootLog "WARN: admin/auth control plane healthy; engine, local voice API, or exact sidecars remain degraded"
|
|
exit 2
|
|
}
|
|
} else {
|
|
Write-BootLog "WARN: boot finished but admin/auth control plane failed — see apps/api/api.public.err.log"
|
|
exit 2
|
|
}
|