6369f29439
공개 런타임 승격 체인이 psutil로 이전 프로세스 환경을 통째로 이식하는 과정에서 SystemRoot 가 유실됐고, Go 계열 CLI(agy)는 시스템 인증서 풀/홈 해석에 SystemRoot 가 필요해 agy models 가 조용히 빈 목록을 반환했다(관리자 AI 운영 화면의 'Agy가 선택 가능한 모델을 반환하지 않았습니다' 두 번째 원인). - _cli_subprocess_env(): 상속 환경에서 빠진 SystemRoot/SystemDrive/ComSpec 만 기본값으로 백필해 모든 CLI 스폰(_run_process·codex app-server·agy stream·claude 세션)에 적용. os.environ 의 Windows 대문자 정규화를 고려한 대소문자 무시 조회 - Set-CompleteProcessEnvironment: 이식본에 빠진 Windows 필수 키를 Machine 스코프 표준값으로 병합해 승격 체인 자체의 유실을 원천 보강 - 검증: 신규 2단위 RED→GREEN, engine_gateway 65 passed, app 924 passed, PS 5.1 parser OK, SystemRoot 제거 환경에서 실제 agy CLI 14모델 live 조회 확인
1736 lines
59 KiB
PowerShell
1736 lines
59 KiB
PowerShell
param(
|
|
[string]$Workspace = "D:\workspace\vignette",
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[int]$VoiceSidecarReadySeconds = 300,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[switch]$SkipEngineRestart,
|
|
[switch]$ForceApiRestart,
|
|
[switch]$SkipWebRestart,
|
|
[switch]$RouteCloudflareDns,
|
|
[string]$CloudflareTunnelName = "vignette",
|
|
[switch]$SkipCloudflaredRestart,
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[switch]$RequireFreshPublicProvenance,
|
|
[string]$ExpectedSourceCommit = "",
|
|
[string]$ExpectedSourceTree = "",
|
|
[string]$ExpectedPythonSha256 = "",
|
|
[string]$ExpectedCloudflaredSha256 = "",
|
|
[string]$ExpectedCloudflaredConfigSha256 = "",
|
|
[string]$RuntimeProvenancePath = "",
|
|
[ValidateRange(1, 60)]
|
|
[int]$ProcessStopTimeoutSeconds = 15
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
|
|
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
|
|
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
|
|
if (-not $env:ENGINE_READY_TTL_SECONDS) {
|
|
$env:ENGINE_READY_TTL_SECONDS = "1800"
|
|
}
|
|
|
|
$ApiDir = Join-Path $Workspace "apps\api"
|
|
$WebDir = Join-Path $Workspace "apps\web"
|
|
$OutLog = Join-Path $ApiDir "api.public.out.log"
|
|
$ErrLog = Join-Path $ApiDir "api.public.err.log"
|
|
$EngineOutLog = Join-Path $ApiDir "engine.public.out.log"
|
|
$EngineErrLog = Join-Path $ApiDir "engine.public.err.log"
|
|
$WebOutLog = Join-Path $Workspace "web.public.out.log"
|
|
$WebErrLog = Join-Path $Workspace "web.public.err.log"
|
|
$WhisperStartScript = Join-Path $Workspace "scripts\start-local-whisper-stt.ps1"
|
|
$MeloTtsStartScript = Join-Path $Workspace "scripts\start-melotts.ps1"
|
|
$VoiceSidecarProbe = Join-Path $Workspace "scripts\probe-public-voice-sidecars.py"
|
|
$WhisperModel = "small"
|
|
$WhisperLanguage = "ko"
|
|
$WhisperDevice = "cpu"
|
|
$MeloTtsModel = "melotts-korean"
|
|
$MeloTtsLanguage = "KR"
|
|
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
|
|
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
|
|
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
|
|
$RequiredPublicApiPaths = @(
|
|
"/health",
|
|
"/voice/health",
|
|
"/voice/speech",
|
|
"/admin/voice-runtime"
|
|
)
|
|
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
|
|
$PublicWebHostnames = @("vnet.18ka.net")
|
|
|
|
function Get-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 5
|
|
)
|
|
|
|
try {
|
|
Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
} catch {
|
|
$null
|
|
}
|
|
}
|
|
|
|
function Wait-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $Uri -TimeoutSec 5
|
|
if ($null -ne $health -and (& $IsHealthy $health)) {
|
|
return $health
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for healthy response from $Uri"
|
|
}
|
|
|
|
function Test-PortListener {
|
|
param([int]$Port)
|
|
|
|
$listener = Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue `
|
|
| Select-Object -First 1
|
|
return $null -ne $listener
|
|
}
|
|
|
|
function Get-ListenerProcessIds {
|
|
param([int]$Port)
|
|
|
|
return @(
|
|
Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue |
|
|
Select-Object -ExpandProperty OwningProcess -Unique
|
|
)
|
|
}
|
|
|
|
function Test-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component
|
|
)
|
|
|
|
$probeArgs = @(
|
|
"-X", "utf8", $VoiceSidecarProbe,
|
|
"--component", $Component,
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", $WhisperModel,
|
|
"--stt-language", $WhisperLanguage,
|
|
"--stt-device", $WhisperDevice,
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", $MeloTtsModel,
|
|
"--tts-language", $MeloTtsLanguage,
|
|
"--timeout-seconds", "5"
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
& $Python @probeArgs 1>$null 2>$null
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return $probeExit -eq 0
|
|
}
|
|
|
|
function Wait-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-VoiceSidecarReady -Component $Component) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq $WhisperModel -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq $MeloTtsModel -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-PriorVoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
|
|
)
|
|
}
|
|
|
|
function ConvertTo-SafeVoiceHealthContract {
|
|
param([object]$Health)
|
|
|
|
return [ordered]@{
|
|
status = [string]$Health.status
|
|
available = [bool]$Health.available
|
|
stt_available = [bool]$Health.stt_available
|
|
tts_available = [bool]$Health.tts_available
|
|
stt_provider = [string]$Health.stt_provider
|
|
stt_model = [string]$Health.stt_model
|
|
tts_provider = [string]$Health.tts_provider
|
|
tts_model = [string]$Health.tts_model
|
|
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
|
|
}
|
|
}
|
|
|
|
function Test-VoiceHealthContract {
|
|
param(
|
|
[object]$Health,
|
|
[System.Collections.IDictionary]$Expected
|
|
)
|
|
|
|
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
|
|
return $false
|
|
}
|
|
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
|
|
foreach ($name in $Expected.Keys) {
|
|
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-RequiredOpenApiPaths {
|
|
param(
|
|
[object]$Document,
|
|
[string[]]$RequiredPaths
|
|
)
|
|
|
|
if ($null -eq $Document -or $null -eq $Document.paths) {
|
|
return $false
|
|
}
|
|
$actualPaths = @($Document.paths.PSObject.Properties.Name)
|
|
foreach ($requiredPath in $RequiredPaths) {
|
|
if ($actualPaths -notcontains $requiredPath) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 45
|
|
)
|
|
|
|
$headers = $null
|
|
if ($env:ENGINE_GATEWAY_SHARED_SECRET) {
|
|
$headers = @{ "X-Vignette-Engine-Token" = $env:ENGINE_GATEWAY_SHARED_SECRET }
|
|
}
|
|
try {
|
|
if ($headers) {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec -Headers $headers
|
|
} else {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec
|
|
}
|
|
return [bool]($response.ok -eq $true)
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Wait-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 90
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-EngineReady -Port $Port) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Wait-HttpStatus {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Uri -TimeoutSec 5
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 500) {
|
|
return $response
|
|
}
|
|
} catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for HTTP response from $Uri"
|
|
}
|
|
|
|
function Stop-ProcessesBounded {
|
|
param(
|
|
[object[]]$Processes,
|
|
[int]$TimeoutSec,
|
|
[string]$Role
|
|
)
|
|
|
|
$processIds = @(
|
|
$Processes |
|
|
ForEach-Object { [int]$_.ProcessId } |
|
|
Sort-Object -Unique
|
|
)
|
|
foreach ($processId in $processIds) {
|
|
Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$remaining = @(
|
|
$processIds |
|
|
Where-Object { $null -ne (Get-Process -Id $_ -ErrorAction SilentlyContinue) }
|
|
)
|
|
if ($remaining.Count -eq 0) {
|
|
return $processIds
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
|
}
|
|
|
|
function Get-UvicornProcessesByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port
|
|
)
|
|
|
|
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
|
|
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -like "python*" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*uvicorn $AppImport*" -and
|
|
$_.CommandLine -like "*--port $Port*"
|
|
}
|
|
)
|
|
}
|
|
|
|
function Stop-UvicornByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
|
|
return @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $processes `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "uvicorn $AppImport on port $Port"
|
|
)
|
|
}
|
|
|
|
function Get-CloudflaredProcessesForConfig {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[switch]$ExactPath
|
|
)
|
|
|
|
$configLeaf = Split-Path -Leaf $ConfigPath
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*--config*" -and
|
|
(
|
|
$_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -or
|
|
(-not $ExactPath -and $_.CommandLine -like "*$configLeaf*")
|
|
)
|
|
}
|
|
)
|
|
}
|
|
|
|
function Wait-ProcessIdentity {
|
|
param(
|
|
[int]$ProcessId,
|
|
[string]$Role,
|
|
[string]$ExpectedCwd = "",
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $ProcessId" `
|
|
-ErrorAction SilentlyContinue
|
|
if (
|
|
$null -ne $process -and
|
|
$process.ExecutablePath -and
|
|
$process.CommandLine
|
|
) {
|
|
$identityProbeArgs = @(
|
|
"-X", "utf8", "-c",
|
|
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
|
|
"$ProcessId"
|
|
)
|
|
$identityProbe = @(& $Python @identityProbeArgs)
|
|
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
|
|
throw "Could not prove $Role psutil identity for PID $ProcessId"
|
|
}
|
|
$actualCwd = $identityProbe[0].Trim()
|
|
$startedAtUtc = $identityProbe[1].Trim()
|
|
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
|
|
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
|
|
$environmentObject = $identityProbe[4] | ConvertFrom-Json
|
|
$processEnvironment = [ordered]@{}
|
|
foreach ($property in $environmentObject.PSObject.Properties) {
|
|
$processEnvironment[$property.Name] = [string]$property.Value
|
|
}
|
|
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "$Role psutil identity is incomplete for PID $ProcessId"
|
|
}
|
|
if ($ExpectedCwd -and -not [string]::Equals(
|
|
[System.IO.Path]::GetFullPath($actualCwd),
|
|
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "$Role working directory drift: expected=$ExpectedCwd actual=$actualCwd"
|
|
}
|
|
return [ordered]@{
|
|
role = $Role
|
|
pid = [int]$process.ProcessId
|
|
started_at_utc = $startedAtUtc
|
|
executable_path = $process.ExecutablePath
|
|
executable_name = Split-Path -Leaf $process.ExecutablePath
|
|
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
command_line = $process.CommandLine
|
|
command_line_sha256 = $commandLineSha256
|
|
argument_list = $argumentList
|
|
environment = $processEnvironment
|
|
cwd = $actualCwd
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out reading $Role process identity for PID $ProcessId"
|
|
}
|
|
|
|
function ConvertTo-SafeProcessIdentity {
|
|
param([System.Collections.IDictionary]$Identity)
|
|
|
|
# Raw command line이나 executable full path는 config/token을 우발적으로
|
|
# 영구 보존할 수 있다. topology 결속에 필요한 비밀 비포함 투영만 기록한다.
|
|
return [ordered]@{
|
|
pid = [int]$Identity.pid
|
|
started_at_utc = $Identity.started_at_utc
|
|
executable_name = $Identity.executable_name
|
|
executable_sha256 = $Identity.executable_sha256
|
|
command_line_sha256 = $Identity.command_line_sha256
|
|
cwd = $Identity.cwd
|
|
}
|
|
}
|
|
|
|
function Save-ManagedEnvironment {
|
|
param([string[]]$Names)
|
|
|
|
$snapshot = [ordered]@{}
|
|
foreach ($name in $Names) {
|
|
$value = [System.Environment]::GetEnvironmentVariable(
|
|
$name,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
$snapshot[$name] = [ordered]@{
|
|
present = $null -ne $value
|
|
value = $value
|
|
}
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Restore-ManagedEnvironment {
|
|
param([System.Collections.IDictionary]$Snapshot)
|
|
|
|
foreach ($name in $Snapshot.Keys) {
|
|
$entry = $Snapshot[$name]
|
|
if ($entry.present) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
[string]$entry.value,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
} else {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
function Save-CompleteProcessEnvironment {
|
|
$snapshot = [ordered]@{}
|
|
$environment = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in $environment.Keys) {
|
|
$snapshot[[string]$name] = [string]$environment[$name]
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Set-CompleteProcessEnvironment {
|
|
param([System.Collections.IDictionary]$Environment)
|
|
|
|
# psutil 환경 캡색에서 SystemRoot 같은 Windows 필수 변수가 유실되면 Go 계열 CLI(agy)가
|
|
# 시스템 인증서 풀/홈 해석에 실패해 조용히 빈 결과를 낸다(2026-08-18 실측). 이식본에
|
|
# 빠진 필수 키는 Machine 스코프 표준값으로 되살린다.
|
|
$windowsEssentials = @('SystemRoot', 'windir', 'SystemDrive', 'ComSpec')
|
|
foreach ($name in $windowsEssentials) {
|
|
$missing = -not $Environment.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$Environment[$name])
|
|
if ($missing) {
|
|
$machineValue = [System.Environment]::GetEnvironmentVariable($name, 'Machine')
|
|
if ($machineValue) {
|
|
$Environment[$name] = $machineValue
|
|
}
|
|
}
|
|
}
|
|
|
|
$current = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in @($current.Keys)) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
foreach ($name in $Environment.Keys) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
[string]$Environment[$name],
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
|
|
function ConvertTo-WindowsCommandLineArgument {
|
|
param([AllowEmptyString()][string]$Argument)
|
|
|
|
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
|
return $Argument
|
|
}
|
|
|
|
$builder = New-Object System.Text.StringBuilder
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
foreach ($character in $Argument.ToCharArray()) {
|
|
if ($character -eq '\') {
|
|
$backslashes++
|
|
continue
|
|
}
|
|
if ($character -eq '"') {
|
|
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
continue
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * $backslashes))
|
|
$backslashes = 0
|
|
}
|
|
$null = $builder.Append($character)
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * ($backslashes * 2)))
|
|
}
|
|
$null = $builder.Append('"')
|
|
return $builder.ToString()
|
|
}
|
|
|
|
function Join-WindowsArgumentList {
|
|
param([object[]]$ArgumentList)
|
|
|
|
return (@(
|
|
foreach ($argument in $ArgumentList) {
|
|
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
|
}
|
|
) -join ' ')
|
|
}
|
|
|
|
function Start-PinnedPriorProcess {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role,
|
|
[string]$StdoutLog,
|
|
[string]$StderrLog
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
|
|
throw "Prior $Role executable is unavailable"
|
|
}
|
|
$actualExecutableSha256 = (
|
|
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
|
|
throw "Prior $Role executable SHA256 drift"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
|
throw "Prior $Role working directory is unavailable"
|
|
}
|
|
if (@($Identity.argument_list).Count -eq 0) {
|
|
throw "Prior $Role argument list is unavailable"
|
|
}
|
|
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
|
|
throw "Prior $Role environment is unavailable"
|
|
}
|
|
|
|
$callerEnvironment = Save-CompleteProcessEnvironment
|
|
try {
|
|
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
|
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
|
return Start-Process -WindowStyle Hidden `
|
|
-FilePath $Identity.executable_path `
|
|
-ArgumentList $argumentString `
|
|
-WorkingDirectory $Identity.cwd `
|
|
-RedirectStandardOutput $StdoutLog `
|
|
-RedirectStandardError $StderrLog `
|
|
-PassThru
|
|
} finally {
|
|
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
|
}
|
|
}
|
|
|
|
function Restore-PriorPublicRuntime {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApi,
|
|
[System.Collections.IDictionary]$PriorCloudflared,
|
|
[System.Collections.IDictionary]$PriorLocalVoiceContract,
|
|
[System.Collections.IDictionary]$PriorPublicVoiceContract,
|
|
[System.Collections.IDictionary]$EnvironmentSnapshot,
|
|
[string]$ConfigPath,
|
|
[int]$ApiPortValue,
|
|
[string]$HealthUrl,
|
|
[string]$VoiceHealthUrl,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$null = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "app.main:app" `
|
|
-Port $ApiPortValue `
|
|
-TimeoutSec $TimeoutSec
|
|
)
|
|
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
|
|
$priorApiProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorApi `
|
|
-Role "api" `
|
|
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
|
|
$priorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiProcess.Id `
|
|
-Role "restored prior api" `
|
|
-ExpectedCwd $PriorApi.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
|
|
throw "Restored prior API identity drift: $field"
|
|
}
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Restored prior runtime does not have the exact local voice sidecars"
|
|
}
|
|
|
|
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
|
|
$currentCloudflared = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedConfigPath `
|
|
-ExactPath
|
|
)
|
|
$null = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $currentCloudflared `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "failed fresh cloudflared"
|
|
)
|
|
$priorCloudflaredProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorCloudflared `
|
|
-Role "cloudflared" `
|
|
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
|
|
$priorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcess.Id `
|
|
-Role "restored prior cloudflared" `
|
|
-ExpectedCwd $PriorCloudflared.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
|
|
throw "Restored prior cloudflared identity drift: $field"
|
|
}
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $HealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri $VoiceHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
|
|
return [ordered]@{
|
|
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
|
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
|
|
local_health = $true
|
|
local_voice_health = $true
|
|
public_health = $true
|
|
public_voice_health = $true
|
|
}
|
|
}
|
|
|
|
function Stop-NodeByPortHint {
|
|
param([int]$Port)
|
|
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "node.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*vite*preview*" -and
|
|
$_.CommandLine -like "*$Port*"
|
|
} |
|
|
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
|
}
|
|
|
|
function ConvertTo-CompactJson {
|
|
param([object]$Value)
|
|
ConvertTo-Json -InputObject $Value -Compress
|
|
}
|
|
|
|
function Invoke-StableGitText {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string[]]$Arguments
|
|
)
|
|
|
|
$value = & git.exe -C $SourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Initialize-RuntimeProvenanceOutput {
|
|
param([string]$OutputPath)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($OutputPath)) {
|
|
throw "Fresh public provenance output path must be absolute"
|
|
}
|
|
|
|
try {
|
|
$resolvedOutputPath = [System.IO.Path]::GetFullPath($OutputPath)
|
|
$outputDirectory = Split-Path -Parent $resolvedOutputPath
|
|
if (-not $outputDirectory) {
|
|
throw "Fresh public provenance output path has no parent directory"
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Container) {
|
|
throw "Fresh public provenance output path is a directory: $resolvedOutputPath"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
throw "Fresh public provenance output directory is unavailable: $outputDirectory"
|
|
}
|
|
|
|
# 기존 receipt가 잠겨 있거나 read-only라면 프로세스 교체 전에 실패해야 한다.
|
|
# sibling probe 두 개를 atomic replace해 디렉터리의 create/flush/replace/delete
|
|
# 권한도 미리 검증한다. 실제 receipt 내용은 이 단계에서 건드리지 않는다.
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Leaf) {
|
|
$attributes = [System.IO.File]::GetAttributes($resolvedOutputPath)
|
|
if (($attributes -band [System.IO.FileAttributes]::ReadOnly) -ne 0) {
|
|
throw "Fresh public provenance output is read-only: $resolvedOutputPath"
|
|
}
|
|
$existingStream = [System.IO.File]::Open(
|
|
$resolvedOutputPath,
|
|
[System.IO.FileMode]::Open,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::Read
|
|
)
|
|
$existingStream.Dispose()
|
|
}
|
|
|
|
$probeId = [Guid]::NewGuid().ToString("N")
|
|
$probeSource = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.source.tmp"
|
|
$probeTarget = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.target.tmp"
|
|
$probeBackup = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.backup.tmp"
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
[System.IO.File]::WriteAllText($probeSource, "probe-source", $encoding)
|
|
[System.IO.File]::WriteAllText($probeTarget, "probe-target", $encoding)
|
|
[System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup)
|
|
[System.IO.File]::Delete($probeTarget)
|
|
[System.IO.File]::Delete($probeBackup)
|
|
} finally {
|
|
foreach ($probePath in @($probeSource, $probeTarget, $probeBackup)) {
|
|
if ($probePath -and [System.IO.File]::Exists($probePath)) {
|
|
[System.IO.File]::Delete($probePath)
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
throw "Fresh public provenance output preflight failed before runtime mutation: $($_.Exception.Message)"
|
|
}
|
|
|
|
return $resolvedOutputPath
|
|
}
|
|
|
|
function Write-Utf8TextAtomically {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$Value
|
|
)
|
|
|
|
$outputDirectory = Split-Path -Parent $OutputPath
|
|
$temporaryPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).tmp"
|
|
$backupPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).backup.tmp"
|
|
$published = $false
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
$bytes = $encoding.GetBytes($Value)
|
|
$stream = [System.IO.FileStream]::new(
|
|
$temporaryPath,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
try {
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
} finally {
|
|
$stream.Dispose()
|
|
}
|
|
|
|
if ([System.IO.File]::Exists($OutputPath)) {
|
|
[System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath)
|
|
} elseif (Test-Path -LiteralPath $OutputPath) {
|
|
throw "Fresh public provenance output became a non-file before commit: $OutputPath"
|
|
} else {
|
|
[System.IO.File]::Move($temporaryPath, $OutputPath)
|
|
}
|
|
$published = $true
|
|
} finally {
|
|
if ([System.IO.File]::Exists($temporaryPath)) {
|
|
[System.IO.File]::Delete($temporaryPath)
|
|
}
|
|
if ([System.IO.File]::Exists($backupPath)) {
|
|
try {
|
|
[System.IO.File]::Delete($backupPath)
|
|
} catch {
|
|
if ($published) {
|
|
Write-Warning "Atomic provenance receipt was published, but its temporary backup could not be removed: $backupPath"
|
|
} else {
|
|
throw
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Write-FailedFreshPromotionEvidence {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$FailureStage,
|
|
[bool]$RollbackSucceeded,
|
|
[object]$RollbackResult,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree
|
|
)
|
|
|
|
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
|
|
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
|
|
$failedPath = "$OutputPath.failed.log"
|
|
$payload = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-failure.v1"
|
|
status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" }
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
failure_stage = $FailureStage
|
|
source = [ordered]@{
|
|
git_commit = $SourceCommit.ToLowerInvariant()
|
|
git_tree = $SourceTree.ToLowerInvariant()
|
|
}
|
|
rollback = [ordered]@{
|
|
attempted = $true
|
|
succeeded = $RollbackSucceeded
|
|
result = $RollbackResult
|
|
}
|
|
}
|
|
$json = ConvertTo-Json -InputObject $payload -Depth 8
|
|
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
|
|
return $failedPath
|
|
}
|
|
|
|
function Assert-FreshPublicProvenanceContract {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree,
|
|
[string]$PythonPath,
|
|
[string]$PythonSha256,
|
|
[string]$CloudflaredPath,
|
|
[string]$CloudflaredSha256,
|
|
[string]$ConfigPath,
|
|
[string]$ConfigSha256
|
|
)
|
|
|
|
if (-not $ForceApiRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -ForceApiRestart"
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
|
|
}
|
|
if (-not $SkipEngineRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
|
|
}
|
|
if (-not $SkipWebRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
|
|
}
|
|
if ($RouteCloudflareDns) {
|
|
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
|
|
}
|
|
if (-not [string]::Equals(
|
|
$PublicHealthUrl,
|
|
$CanonicalPublicHealthUrl,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion requires the canonical HTTPS public health URL"
|
|
}
|
|
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
|
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
|
|
throw "Fresh public provenance requires exact source commit and tree pins"
|
|
}
|
|
}
|
|
foreach ($shaPin in @($PythonSha256, $CloudflaredSha256, $ConfigSha256)) {
|
|
if ($shaPin -notmatch "^[0-9a-fA-F]{64}$") {
|
|
throw "Fresh public provenance requires exact Python, cloudflared, and config SHA256 pins"
|
|
}
|
|
}
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path
|
|
$expectedStartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$runningStartScript = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningStartScript,
|
|
(Resolve-Path -LiteralPath $expectedStartScript).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion must execute the launcher from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Fresh public promotion requires detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $SourceCommit.ToLowerInvariant()) {
|
|
throw "Fresh public source commit drift: expected=$SourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $SourceTree.ToLowerInvariant()) {
|
|
throw "Fresh public source tree drift: expected=$SourceTree actual=$actualTree"
|
|
}
|
|
$dirty = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Fresh public promotion requires a clean stable source"
|
|
}
|
|
|
|
foreach ($pin in @(
|
|
[pscustomobject]@{ Path = $PythonPath; Sha256 = $PythonSha256; Label = "Python" },
|
|
[pscustomobject]@{ Path = $CloudflaredPath; Sha256 = $CloudflaredSha256; Label = "cloudflared" },
|
|
[pscustomobject]@{ Path = $ConfigPath; Sha256 = $ConfigSha256; Label = "cloudflared config" }
|
|
)) {
|
|
if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) {
|
|
throw "Pinned $($pin.Label) file not found at $($pin.Path)"
|
|
}
|
|
$actualSha256 = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualSha256 -ne $pin.Sha256.ToLowerInvariant()) {
|
|
throw "Pinned $($pin.Label) SHA256 drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Set-CloudflaredIngress {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[string[]]$ApiHostnames,
|
|
[string[]]$WebHostnames,
|
|
[int]$ApiPortValue,
|
|
[int]$WebPortValue,
|
|
[switch]$RequireUnchanged
|
|
)
|
|
|
|
$lines = Get-Content -Encoding UTF8 -Path $ConfigPath
|
|
$ingressIndex = -1
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -match "^\s*ingress:\s*$") {
|
|
$ingressIndex = $i
|
|
break
|
|
}
|
|
}
|
|
if ($ingressIndex -lt 0) {
|
|
throw "Could not find ingress: in $ConfigPath"
|
|
}
|
|
|
|
$nextLines = @()
|
|
if ($ingressIndex -gt 0) {
|
|
$nextLines += $lines[0..($ingressIndex - 1)]
|
|
}
|
|
$nextLines += "ingress:"
|
|
foreach ($hostname in $WebHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$WebPortValue"
|
|
}
|
|
foreach ($hostname in $ApiHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$ApiPortValue"
|
|
}
|
|
$nextLines += " - service: http_status:404"
|
|
|
|
$matches = $lines.Count -eq $nextLines.Count
|
|
if ($matches) {
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -cne $nextLines[$i]) {
|
|
$matches = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if ($RequireUnchanged -and -not $matches) {
|
|
throw "Pinned cloudflared config ingress does not match the requested public topology"
|
|
}
|
|
if (-not $matches) {
|
|
Set-Content -Encoding UTF8 -Path $ConfigPath -Value $nextLines
|
|
}
|
|
}
|
|
|
|
$freshMutationStarted = $false
|
|
$freshPromotionCommitted = $false
|
|
$freshFailureStage = "preflight"
|
|
$freshPriorApiIdentity = $null
|
|
$freshPriorCloudflaredIdentity = $null
|
|
$freshPriorLocalVoiceContract = $null
|
|
$freshPriorPublicVoiceContract = $null
|
|
$freshEnvironmentSnapshot = $null
|
|
$resolvedRuntimeProvenancePath = $null
|
|
$freshManagedEnvironmentNames = @(
|
|
"ENVIRONMENT",
|
|
"ENGINE_URL",
|
|
"ENGINE_MODE",
|
|
"VIGNETTE_LIVE_CLIENT_PROVIDER",
|
|
"AUTH_DEV_LOGIN_ENABLED",
|
|
"AUTO_SEED_PERSONAS",
|
|
"ALLOW_SEED_PERSONA_FALLBACK",
|
|
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
|
|
"VIGNETTE_VOICE_STT_PROVIDER",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_URL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
|
|
"VIGNETTE_VOICE_TTS_PROVIDER",
|
|
"VIGNETTE_MELOTTS_TTS_URL",
|
|
"FRONTEND_BASE_URL",
|
|
"CORS_ORIGINS",
|
|
"FRONTEND_ORIGIN_MAP"
|
|
)
|
|
|
|
trap {
|
|
$caught = $_
|
|
if (
|
|
$RequireFreshPublicProvenance -and
|
|
$freshMutationStarted -and
|
|
-not $freshPromotionCommitted
|
|
) {
|
|
$rollbackResult = $null
|
|
$rollbackSucceeded = $false
|
|
$rollbackFailureType = "none"
|
|
try {
|
|
$rollbackResult = Restore-PriorPublicRuntime `
|
|
-PriorApi $freshPriorApiIdentity `
|
|
-PriorCloudflared $freshPriorCloudflaredIdentity `
|
|
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
|
|
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
|
|
-EnvironmentSnapshot $freshEnvironmentSnapshot `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiPortValue $ApiPort `
|
|
-HealthUrl $PublicHealthUrl `
|
|
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$rollbackSucceeded = $true
|
|
} catch {
|
|
$rollbackFailureType = $_.Exception.GetType().Name
|
|
}
|
|
|
|
$failedEvidencePath = ""
|
|
if ($resolvedRuntimeProvenancePath) {
|
|
try {
|
|
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-FailureStage $freshFailureStage `
|
|
-RollbackSucceeded $rollbackSucceeded `
|
|
-RollbackResult $rollbackResult `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree
|
|
} catch {
|
|
$failedEvidencePath = "unavailable"
|
|
}
|
|
}
|
|
|
|
if ($rollbackSucceeded) {
|
|
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw $caught
|
|
}
|
|
|
|
if (!(Test-Path $Python)) {
|
|
throw "Python 3.11 not found at $Python"
|
|
}
|
|
if (!(Test-Path $ApiDir)) {
|
|
throw "API directory not found at $ApiDir"
|
|
}
|
|
if (!(Test-Path $WebDir)) {
|
|
throw "Web directory not found at $WebDir"
|
|
}
|
|
foreach ($voiceScript in @($WhisperStartScript, $MeloTtsStartScript, $VoiceSidecarProbe)) {
|
|
if (!(Test-Path -LiteralPath $voiceScript)) {
|
|
throw "Voice sidecar prerequisite not found at $voiceScript"
|
|
}
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
if (!(Test-Path -LiteralPath $Cloudflared -PathType Leaf)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path -LiteralPath $CloudflaredConfig -PathType Leaf)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
if (-not $RuntimeProvenancePath) {
|
|
$RuntimeProvenancePath = Join-Path $Workspace "public-runtime-launch-provenance.log"
|
|
}
|
|
|
|
Assert-FreshPublicProvenanceContract `
|
|
-SourceRoot $Workspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256 `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredSha256 $ExpectedCloudflaredSha256 `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ConfigSha256 $ExpectedCloudflaredConfigSha256
|
|
|
|
$resolvedRuntimeProvenancePath = Initialize-RuntimeProvenanceOutput `
|
|
-OutputPath $RuntimeProvenancePath
|
|
|
|
# 승격 모드에서 config를 재작성하면 사전 pin과 실제 tunnel 입력이 달라진다.
|
|
# exact ingress가 이미 들어 있는 경우에만 이후 프로세스 mutation으로 진행한다.
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort `
|
|
-RequireUnchanged
|
|
|
|
$priorApiProcesses = @(
|
|
Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort
|
|
)
|
|
if ($priorApiProcesses.Count -ne 1) {
|
|
throw "Fresh public promotion requires exactly one prior API process for transactional rollback"
|
|
}
|
|
$priorCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
|
|
-ExactPath
|
|
)
|
|
if ($priorCloudflaredProcesses.Count -ne 1) {
|
|
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
|
|
}
|
|
$freshPriorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiProcesses[0].ProcessId `
|
|
-Role "prior api" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
|
|
-Role "prior cloudflared" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorLocalVoiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorLocalVoiceHealth
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorPublicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorPublicVoiceHealth
|
|
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
|
|
-Names $freshManagedEnvironmentNames
|
|
}
|
|
|
|
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
|
# 프로세스는 살아 있는데 그 프로세스의 claude 세션만 죽은 상태는 /health를 통과하므로,
|
|
# /health 기준으로는 복구가 필요한 순간에 오히려 재기동을 건너뛴다(2026-08-07 사고).
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
$engineReady = $false
|
|
if ($null -ne $engineHealth -and $engineHealth.ok) {
|
|
$engineReady = Test-EngineReady -Port $EnginePort
|
|
}
|
|
if ($SkipEngineRestart) {
|
|
if (-not $engineReady) {
|
|
throw "Engine gateway is not ready on http://127.0.0.1:$EnginePort/ready"
|
|
}
|
|
} elseif (-not $engineReady) {
|
|
$null = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "engine_gateway.gateway:app" `
|
|
-Port $EnginePort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
|
|
# Start-Process는 리다이렉트 대상 로그를 덮어쓴다. 직전 사고 로그를 보존해야
|
|
# 재기동 후에도 원인을 추적할 수 있다.
|
|
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
|
|
if (Test-Path $logFile) {
|
|
# stable detached source의 provenance gate는 untracked 파일도 차단한다.
|
|
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
|
|
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @("-m", "uvicorn", "engine_gateway.gateway:app", "--host", "127.0.0.1", "--port", "$EnginePort") `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $EngineOutLog `
|
|
-RedirectStandardError $EngineErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
$engineReady = Wait-EngineReady -Port $EnginePort -TimeoutSec 90
|
|
if (-not $engineReady) {
|
|
Write-Warning "Engine gateway is still degraded; continuing admin/auth recovery"
|
|
}
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
}
|
|
|
|
$env:ENVIRONMENT = "prod"
|
|
$env:ENGINE_URL = "http://127.0.0.1:$EnginePort"
|
|
$env:ENGINE_MODE = "claude_cli"
|
|
$env:VIGNETTE_LIVE_CLIENT_PROVIDER = "claude_cli"
|
|
$env:AUTH_DEV_LOGIN_ENABLED = "false"
|
|
$env:AUTO_SEED_PERSONAS = "false"
|
|
$env:ALLOW_SEED_PERSONA_FALLBACK = "false"
|
|
$env:VIGNETTE_VOICE_POC_SAMPLE_TTS = "false"
|
|
$env:VIGNETTE_VOICE_STT_PROVIDER = "local_whisper"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_URL = "ws://127.0.0.1:$WhisperPort/v1/listen"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_MODEL = $WhisperModel
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE = $WhisperLanguage
|
|
$env:VIGNETTE_VOICE_TTS_PROVIDER = "melotts"
|
|
$env:VIGNETTE_MELOTTS_TTS_URL = "http://127.0.0.1:$MeloTtsPort"
|
|
$env:FRONTEND_BASE_URL = "https://vignette.chanpaca.net"
|
|
$frontendOrigins = @("https://vignette.chanpaca.net", "https://vnet.18ka.net", "https://vignette-b1q.pages.dev")
|
|
$localViteOrigins = @()
|
|
foreach ($port in 5170..5180) {
|
|
$localViteOrigins += "http://localhost:$port"
|
|
$localViteOrigins += "http://127.0.0.1:$port"
|
|
}
|
|
$env:CORS_ORIGINS = ConvertTo-CompactJson -Value ($frontendOrigins + $localViteOrigins)
|
|
$env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
|
|
"api-vignette.chanpaca.net" = "https://vignette.chanpaca.net"
|
|
"api-vnet.18ka.net" = "https://vnet.18ka.net"
|
|
})
|
|
|
|
# 포트 리스너만으로는 올바른 provider/model을 증명하지 못한다. 첫 WS ready
|
|
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
|
|
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $WhisperPort) {
|
|
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
|
|
}
|
|
& $WhisperStartScript `
|
|
-Port $WhisperPort `
|
|
-Model $WhisperModel `
|
|
-Device $WhisperDevice `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "stt" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "local_whisper/$WhisperModel/$WhisperDevice did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
if (-not (Test-VoiceSidecarReady -Component "tts")) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $MeloTtsPort) {
|
|
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
|
|
}
|
|
& $MeloTtsStartScript `
|
|
-Port $MeloTtsPort `
|
|
-Language $MeloTtsLanguage `
|
|
-Device "cpu" `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "tts" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "melotts/$MeloTtsModel did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
# 두 sidecar를 한 번 더 함께 검사해 개별 probe 사이의 TOCTOU를 닫는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Voice sidecar readiness changed before the API restart gate"
|
|
}
|
|
|
|
$health = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/health"
|
|
$voiceHealth = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/voice/health"
|
|
$apiControlPlaneReady = (
|
|
$null -ne $health -and
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true -and
|
|
(Test-VoiceApiReady -Health $voiceHealth)
|
|
)
|
|
$proc = $null
|
|
$apiStoppedProcessIds = @()
|
|
$apiLaunchIdentity = $null
|
|
if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
|
Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
|
|
} else {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "api_cutover"
|
|
$freshMutationStarted = $true
|
|
}
|
|
$apiStoppedProcessIds = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "app.main:app" `
|
|
-Port $ApiPort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
|
|
$proc = Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @(
|
|
"-m", "uvicorn", "app.main:app",
|
|
"--host", "127.0.0.1",
|
|
"--port", "$ApiPort",
|
|
"--ws", "websockets",
|
|
"--ws-max-queue", "4"
|
|
) `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $OutLog `
|
|
-RedirectStandardError $ErrLog `
|
|
-PassThru
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
if ($apiStoppedProcessIds -contains $proc.Id) {
|
|
throw "Fresh public API did not receive a replacement PID"
|
|
}
|
|
$apiLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $proc.Id `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($apiLaunchIdentity.executable_sha256 -ne $ExpectedPythonSha256.ToLowerInvariant()) {
|
|
throw "Fresh public API executable SHA256 does not match the pinned Python"
|
|
}
|
|
foreach ($requiredArgument in @("uvicorn", "app.main:app", "--port", "$ApiPort", "--ws-max-queue", "4")) {
|
|
if ($apiLaunchIdentity.command_line.IndexOf($requiredArgument, [System.StringComparison]::Ordinal) -lt 0) {
|
|
throw "Fresh public API command line is missing required argument: $requiredArgument"
|
|
}
|
|
}
|
|
}
|
|
|
|
Start-Sleep -Seconds 3
|
|
$health = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$voiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
}
|
|
if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) {
|
|
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
|
|
}
|
|
if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
|
|
throw "Public voice API does not match the exact local provider/model contract: $($voiceHealth | ConvertTo-Json -Compress)"
|
|
}
|
|
|
|
if (!$SkipWebRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "web_preview"
|
|
}
|
|
Stop-NodeByPortHint -Port $WebPort
|
|
|
|
$build = Start-Process -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run build") `
|
|
-WorkingDirectory $WebDir `
|
|
-NoNewWindow `
|
|
-Wait `
|
|
-PassThru
|
|
if ($build.ExitCode -ne 0) {
|
|
throw "Web build failed with exit code $($build.ExitCode)"
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run preview -- --host 127.0.0.1 --port $WebPort") `
|
|
-WorkingDirectory $WebDir `
|
|
-RedirectStandardOutput $WebOutLog `
|
|
-RedirectStandardError $WebErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
Wait-HttpStatus -Uri "http://127.0.0.1:$WebPort/" -TimeoutSec 30 | Out-Null
|
|
}
|
|
|
|
$cloudflaredProcess = $null
|
|
$cloudflaredLaunchIdentity = $null
|
|
$cloudflaredStoppedProcessIds = @()
|
|
if (!$SkipCloudflaredRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "cloudflared_cutover"
|
|
}
|
|
if (!(Test-Path $Cloudflared)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path $CloudflaredConfig)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
|
|
if ($RouteCloudflareDns) {
|
|
foreach ($hostname in ($PublicWebHostnames + $PublicApiHostnames)) {
|
|
& $Cloudflared tunnel route dns $CloudflareTunnelName $hostname
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Warning "cloudflared DNS route failed for $hostname"
|
|
}
|
|
}
|
|
}
|
|
|
|
if (-not $RequireFreshPublicProvenance) {
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort
|
|
}
|
|
|
|
$resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
if ($RequireFreshPublicProvenance) {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
} else {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig -ConfigPath $resolvedCloudflaredConfig
|
|
)
|
|
}
|
|
$cloudflaredStoppedProcessIds = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $existingCloudflaredProcesses `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "cloudflared for $resolvedCloudflaredConfig"
|
|
)
|
|
|
|
$cloudflaredProcess = Start-Process -WindowStyle Hidden -FilePath $Cloudflared `
|
|
-ArgumentList @("tunnel", "--config", $resolvedCloudflaredConfig, "run") `
|
|
-WorkingDirectory $Workspace `
|
|
-RedirectStandardOutput (Join-Path $Workspace "cloudflared.public.out.log") `
|
|
-RedirectStandardError (Join-Path $Workspace "cloudflared.public.err.log") `
|
|
-PassThru
|
|
|
|
if ($cloudflaredStoppedProcessIds -contains $cloudflaredProcess.Id) {
|
|
throw "Cloudflared did not receive a replacement PID"
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
$cloudflaredLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredProcess.Id `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($cloudflaredLaunchIdentity.executable_sha256 -ne $ExpectedCloudflaredSha256.ToLowerInvariant()) {
|
|
throw "Fresh cloudflared executable SHA256 does not match its pin"
|
|
}
|
|
if ($cloudflaredLaunchIdentity.command_line.IndexOf($resolvedCloudflaredConfig, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) {
|
|
throw "Fresh cloudflared command line is not pinned to the expected config"
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "identity_revalidation"
|
|
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
|
|
throw "Fresh public promotion did not produce both API and cloudflared identities"
|
|
}
|
|
|
|
$apiFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $apiLaunchIdentity.pid `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$cloudflaredFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredLaunchIdentity.pid `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
foreach ($identityPair in @(
|
|
[pscustomobject]@{ Role = "api"; Launch = $apiLaunchIdentity; Final = $apiFinalIdentity },
|
|
[pscustomobject]@{ Role = "cloudflared"; Launch = $cloudflaredLaunchIdentity; Final = $cloudflaredFinalIdentity }
|
|
)) {
|
|
foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($identityPair.Launch[$field].ToString() -cne $identityPair.Final[$field].ToString()) {
|
|
throw "Fresh $($identityPair.Role) provenance drifted before receipt: $field"
|
|
}
|
|
}
|
|
}
|
|
|
|
$finalConfigSha256 = (Get-FileHash -LiteralPath $CloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
|
|
throw "Pinned cloudflared config drifted before provenance receipt"
|
|
}
|
|
$freshFailureStage = "public_health_validation"
|
|
$publicHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$publicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$publicOpenApi = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicOpenApiUrl `
|
|
-IsHealthy {
|
|
param($document)
|
|
Test-RequiredOpenApiPaths `
|
|
-Document $document `
|
|
-RequiredPaths $RequiredPublicApiPaths
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Exact local voice sidecars changed before provenance receipt"
|
|
}
|
|
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
|
|
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
|
|
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
|
|
throw "Could not record the psutil version used for process provenance"
|
|
}
|
|
$safeApiIdentity = ConvertTo-SafeProcessIdentity -Identity $apiFinalIdentity
|
|
$safeCloudflaredIdentity = ConvertTo-SafeProcessIdentity -Identity $cloudflaredFinalIdentity
|
|
|
|
$provenance = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-provenance.v1"
|
|
status = "passed"
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
source = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_commit = $ExpectedSourceCommit.ToLowerInvariant()
|
|
git_tree = $ExpectedSourceTree.ToLowerInvariant()
|
|
launcher_sha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
clean_detached_head = $true
|
|
}
|
|
config = [ordered]@{
|
|
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
sha256 = $finalConfigSha256
|
|
}
|
|
public_validation = [ordered]@{
|
|
health_url = $CanonicalPublicHealthUrl
|
|
health = $true
|
|
voice_health_url = $CanonicalPublicVoiceHealthUrl
|
|
voice_health = $true
|
|
openapi_url = $CanonicalPublicOpenApiUrl
|
|
required_openapi_paths = @($RequiredPublicApiPaths)
|
|
openapi = $true
|
|
local_voice_sidecars = $true
|
|
}
|
|
replacement = [ordered]@{
|
|
api_stopped_pids = @($apiStoppedProcessIds)
|
|
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
|
|
api_new_pid = [int]$apiFinalIdentity.pid
|
|
cloudflared_new_pid = [int]$cloudflaredFinalIdentity.pid
|
|
}
|
|
processes = [ordered]@{
|
|
api = $safeApiIdentity
|
|
cloudflared = $safeCloudflaredIdentity
|
|
}
|
|
topology_inputs = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_sha = $ExpectedSourceCommit.ToLowerInvariant()
|
|
api_pid = [int]$apiFinalIdentity.pid
|
|
api_started_at_utc = $apiFinalIdentity.started_at_utc
|
|
api_executable_name = $apiFinalIdentity.executable_name
|
|
api_executable_sha256 = $apiFinalIdentity.executable_sha256
|
|
api_command_line_sha256 = $apiFinalIdentity.command_line_sha256
|
|
api_cwd = $apiFinalIdentity.cwd
|
|
api_listen_port = $ApiPort
|
|
cloudflared_pid = [int]$cloudflaredFinalIdentity.pid
|
|
cloudflared_started_at_utc = $cloudflaredFinalIdentity.started_at_utc
|
|
cloudflared_executable_name = $cloudflaredFinalIdentity.executable_name
|
|
cloudflared_executable_sha256 = $cloudflaredFinalIdentity.executable_sha256
|
|
cloudflared_command_line_sha256 = $cloudflaredFinalIdentity.command_line_sha256
|
|
cloudflared_cwd = $cloudflaredFinalIdentity.cwd
|
|
psutil_version = $psutilVersion
|
|
}
|
|
}
|
|
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
|
|
$freshFailureStage = "receipt_publish"
|
|
try {
|
|
Write-Utf8TextAtomically `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-Value ($provenanceJson + [Environment]::NewLine)
|
|
} catch {
|
|
# 새 PID들은 이미 health/identity gate를 통과했지만, atomic receipt가 없으면
|
|
# 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다.
|
|
throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)"
|
|
}
|
|
$freshPromotionCommitted = $true
|
|
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
|
|
}
|
|
|
|
if ($engineReady) {
|
|
Write-Output "Engine gateway ready (real generation proven) on http://127.0.0.1:$EnginePort"
|
|
} else {
|
|
Write-Warning "Engine gateway degraded; admin/auth control plane remains available"
|
|
}
|
|
if ($null -ne $proc) {
|
|
Write-Output "Public API running on http://127.0.0.1:$ApiPort with PID $($proc.Id)"
|
|
} else {
|
|
Write-Output "Public API kept running on http://127.0.0.1:$ApiPort"
|
|
}
|
|
if (!$SkipWebRestart) {
|
|
Write-Output "Public vnet web preview running on http://127.0.0.1:$WebPort"
|
|
}
|
|
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
|
|
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
|