1722 lines
59 KiB
PowerShell
1722 lines
59 KiB
PowerShell
param(
|
|
[string]$Workspace = "D:\workspace\vignette",
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[int]$VoiceSidecarReadySeconds = 300,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[switch]$SkipEngineRestart,
|
|
[switch]$ForceApiRestart,
|
|
[switch]$SkipWebRestart,
|
|
[switch]$RouteCloudflareDns,
|
|
[string]$CloudflareTunnelName = "vignette",
|
|
[switch]$SkipCloudflaredRestart,
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[switch]$RequireFreshPublicProvenance,
|
|
[string]$ExpectedSourceCommit = "",
|
|
[string]$ExpectedSourceTree = "",
|
|
[string]$ExpectedPythonSha256 = "",
|
|
[string]$ExpectedCloudflaredSha256 = "",
|
|
[string]$ExpectedCloudflaredConfigSha256 = "",
|
|
[string]$RuntimeProvenancePath = "",
|
|
[ValidateRange(1, 60)]
|
|
[int]$ProcessStopTimeoutSeconds = 15
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
|
|
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
|
|
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
|
|
if (-not $env:ENGINE_READY_TTL_SECONDS) {
|
|
$env:ENGINE_READY_TTL_SECONDS = "1800"
|
|
}
|
|
|
|
$ApiDir = Join-Path $Workspace "apps\api"
|
|
$WebDir = Join-Path $Workspace "apps\web"
|
|
$OutLog = Join-Path $ApiDir "api.public.out.log"
|
|
$ErrLog = Join-Path $ApiDir "api.public.err.log"
|
|
$EngineOutLog = Join-Path $ApiDir "engine.public.out.log"
|
|
$EngineErrLog = Join-Path $ApiDir "engine.public.err.log"
|
|
$WebOutLog = Join-Path $Workspace "web.public.out.log"
|
|
$WebErrLog = Join-Path $Workspace "web.public.err.log"
|
|
$WhisperStartScript = Join-Path $Workspace "scripts\start-local-whisper-stt.ps1"
|
|
$MeloTtsStartScript = Join-Path $Workspace "scripts\start-melotts.ps1"
|
|
$VoiceSidecarProbe = Join-Path $Workspace "scripts\probe-public-voice-sidecars.py"
|
|
$WhisperModel = "small"
|
|
$WhisperLanguage = "ko"
|
|
$WhisperDevice = "cpu"
|
|
$MeloTtsModel = "melotts-korean"
|
|
$MeloTtsLanguage = "KR"
|
|
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
|
|
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
|
|
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
|
|
$RequiredPublicApiPaths = @(
|
|
"/health",
|
|
"/voice/health",
|
|
"/voice/speech",
|
|
"/admin/voice-runtime"
|
|
)
|
|
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
|
|
$PublicWebHostnames = @("vnet.18ka.net")
|
|
|
|
function Get-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 5
|
|
)
|
|
|
|
try {
|
|
Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
} catch {
|
|
$null
|
|
}
|
|
}
|
|
|
|
function Wait-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $Uri -TimeoutSec 5
|
|
if ($null -ne $health -and (& $IsHealthy $health)) {
|
|
return $health
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for healthy response from $Uri"
|
|
}
|
|
|
|
function Test-PortListener {
|
|
param([int]$Port)
|
|
|
|
$listener = Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue `
|
|
| Select-Object -First 1
|
|
return $null -ne $listener
|
|
}
|
|
|
|
function Get-ListenerProcessIds {
|
|
param([int]$Port)
|
|
|
|
return @(
|
|
Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue |
|
|
Select-Object -ExpandProperty OwningProcess -Unique
|
|
)
|
|
}
|
|
|
|
function Test-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component
|
|
)
|
|
|
|
$probeArgs = @(
|
|
"-X", "utf8", $VoiceSidecarProbe,
|
|
"--component", $Component,
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", $WhisperModel,
|
|
"--stt-language", $WhisperLanguage,
|
|
"--stt-device", $WhisperDevice,
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", $MeloTtsModel,
|
|
"--tts-language", $MeloTtsLanguage,
|
|
"--timeout-seconds", "5"
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
& $Python @probeArgs 1>$null 2>$null
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return $probeExit -eq 0
|
|
}
|
|
|
|
function Wait-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-VoiceSidecarReady -Component $Component) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq $WhisperModel -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq $MeloTtsModel -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-PriorVoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
|
|
)
|
|
}
|
|
|
|
function ConvertTo-SafeVoiceHealthContract {
|
|
param([object]$Health)
|
|
|
|
return [ordered]@{
|
|
status = [string]$Health.status
|
|
available = [bool]$Health.available
|
|
stt_available = [bool]$Health.stt_available
|
|
tts_available = [bool]$Health.tts_available
|
|
stt_provider = [string]$Health.stt_provider
|
|
stt_model = [string]$Health.stt_model
|
|
tts_provider = [string]$Health.tts_provider
|
|
tts_model = [string]$Health.tts_model
|
|
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
|
|
}
|
|
}
|
|
|
|
function Test-VoiceHealthContract {
|
|
param(
|
|
[object]$Health,
|
|
[System.Collections.IDictionary]$Expected
|
|
)
|
|
|
|
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
|
|
return $false
|
|
}
|
|
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
|
|
foreach ($name in $Expected.Keys) {
|
|
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-RequiredOpenApiPaths {
|
|
param(
|
|
[object]$Document,
|
|
[string[]]$RequiredPaths
|
|
)
|
|
|
|
if ($null -eq $Document -or $null -eq $Document.paths) {
|
|
return $false
|
|
}
|
|
$actualPaths = @($Document.paths.PSObject.Properties.Name)
|
|
foreach ($requiredPath in $RequiredPaths) {
|
|
if ($actualPaths -notcontains $requiredPath) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 45
|
|
)
|
|
|
|
$headers = $null
|
|
if ($env:ENGINE_GATEWAY_SHARED_SECRET) {
|
|
$headers = @{ "X-Vignette-Engine-Token" = $env:ENGINE_GATEWAY_SHARED_SECRET }
|
|
}
|
|
try {
|
|
if ($headers) {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec -Headers $headers
|
|
} else {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec
|
|
}
|
|
return [bool]($response.ok -eq $true)
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Wait-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 90
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-EngineReady -Port $Port) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Wait-HttpStatus {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Uri -TimeoutSec 5
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 500) {
|
|
return $response
|
|
}
|
|
} catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for HTTP response from $Uri"
|
|
}
|
|
|
|
function Stop-ProcessesBounded {
|
|
param(
|
|
[object[]]$Processes,
|
|
[int]$TimeoutSec,
|
|
[string]$Role
|
|
)
|
|
|
|
$processIds = @(
|
|
$Processes |
|
|
ForEach-Object { [int]$_.ProcessId } |
|
|
Sort-Object -Unique
|
|
)
|
|
foreach ($processId in $processIds) {
|
|
Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$remaining = @(
|
|
$processIds |
|
|
Where-Object { $null -ne (Get-Process -Id $_ -ErrorAction SilentlyContinue) }
|
|
)
|
|
if ($remaining.Count -eq 0) {
|
|
return $processIds
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
|
}
|
|
|
|
function Get-UvicornProcessesByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port
|
|
)
|
|
|
|
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
|
|
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -like "python*" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*uvicorn $AppImport*" -and
|
|
$_.CommandLine -like "*--port $Port*"
|
|
}
|
|
)
|
|
}
|
|
|
|
function Stop-UvicornByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
|
|
return @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $processes `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "uvicorn $AppImport on port $Port"
|
|
)
|
|
}
|
|
|
|
function Get-CloudflaredProcessesForConfig {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[switch]$ExactPath
|
|
)
|
|
|
|
$configLeaf = Split-Path -Leaf $ConfigPath
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*--config*" -and
|
|
(
|
|
$_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -or
|
|
(-not $ExactPath -and $_.CommandLine -like "*$configLeaf*")
|
|
)
|
|
}
|
|
)
|
|
}
|
|
|
|
function Wait-ProcessIdentity {
|
|
param(
|
|
[int]$ProcessId,
|
|
[string]$Role,
|
|
[string]$ExpectedCwd = "",
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $ProcessId" `
|
|
-ErrorAction SilentlyContinue
|
|
if (
|
|
$null -ne $process -and
|
|
$process.ExecutablePath -and
|
|
$process.CommandLine
|
|
) {
|
|
$identityProbeArgs = @(
|
|
"-X", "utf8", "-c",
|
|
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
|
|
"$ProcessId"
|
|
)
|
|
$identityProbe = @(& $Python @identityProbeArgs)
|
|
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
|
|
throw "Could not prove $Role psutil identity for PID $ProcessId"
|
|
}
|
|
$actualCwd = $identityProbe[0].Trim()
|
|
$startedAtUtc = $identityProbe[1].Trim()
|
|
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
|
|
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
|
|
$environmentObject = $identityProbe[4] | ConvertFrom-Json
|
|
$processEnvironment = [ordered]@{}
|
|
foreach ($property in $environmentObject.PSObject.Properties) {
|
|
$processEnvironment[$property.Name] = [string]$property.Value
|
|
}
|
|
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "$Role psutil identity is incomplete for PID $ProcessId"
|
|
}
|
|
if ($ExpectedCwd -and -not [string]::Equals(
|
|
[System.IO.Path]::GetFullPath($actualCwd),
|
|
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "$Role working directory drift: expected=$ExpectedCwd actual=$actualCwd"
|
|
}
|
|
return [ordered]@{
|
|
role = $Role
|
|
pid = [int]$process.ProcessId
|
|
started_at_utc = $startedAtUtc
|
|
executable_path = $process.ExecutablePath
|
|
executable_name = Split-Path -Leaf $process.ExecutablePath
|
|
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
command_line = $process.CommandLine
|
|
command_line_sha256 = $commandLineSha256
|
|
argument_list = $argumentList
|
|
environment = $processEnvironment
|
|
cwd = $actualCwd
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out reading $Role process identity for PID $ProcessId"
|
|
}
|
|
|
|
function ConvertTo-SafeProcessIdentity {
|
|
param([System.Collections.IDictionary]$Identity)
|
|
|
|
# Raw command line이나 executable full path는 config/token을 우발적으로
|
|
# 영구 보존할 수 있다. topology 결속에 필요한 비밀 비포함 투영만 기록한다.
|
|
return [ordered]@{
|
|
pid = [int]$Identity.pid
|
|
started_at_utc = $Identity.started_at_utc
|
|
executable_name = $Identity.executable_name
|
|
executable_sha256 = $Identity.executable_sha256
|
|
command_line_sha256 = $Identity.command_line_sha256
|
|
cwd = $Identity.cwd
|
|
}
|
|
}
|
|
|
|
function Save-ManagedEnvironment {
|
|
param([string[]]$Names)
|
|
|
|
$snapshot = [ordered]@{}
|
|
foreach ($name in $Names) {
|
|
$value = [System.Environment]::GetEnvironmentVariable(
|
|
$name,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
$snapshot[$name] = [ordered]@{
|
|
present = $null -ne $value
|
|
value = $value
|
|
}
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Restore-ManagedEnvironment {
|
|
param([System.Collections.IDictionary]$Snapshot)
|
|
|
|
foreach ($name in $Snapshot.Keys) {
|
|
$entry = $Snapshot[$name]
|
|
if ($entry.present) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
[string]$entry.value,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
} else {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
function Save-CompleteProcessEnvironment {
|
|
$snapshot = [ordered]@{}
|
|
$environment = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in $environment.Keys) {
|
|
$snapshot[[string]$name] = [string]$environment[$name]
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Set-CompleteProcessEnvironment {
|
|
param([System.Collections.IDictionary]$Environment)
|
|
|
|
$current = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in @($current.Keys)) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
foreach ($name in $Environment.Keys) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
[string]$Environment[$name],
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
|
|
function ConvertTo-WindowsCommandLineArgument {
|
|
param([AllowEmptyString()][string]$Argument)
|
|
|
|
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
|
return $Argument
|
|
}
|
|
|
|
$builder = New-Object System.Text.StringBuilder
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
foreach ($character in $Argument.ToCharArray()) {
|
|
if ($character -eq '\') {
|
|
$backslashes++
|
|
continue
|
|
}
|
|
if ($character -eq '"') {
|
|
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
continue
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * $backslashes))
|
|
$backslashes = 0
|
|
}
|
|
$null = $builder.Append($character)
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * ($backslashes * 2)))
|
|
}
|
|
$null = $builder.Append('"')
|
|
return $builder.ToString()
|
|
}
|
|
|
|
function Join-WindowsArgumentList {
|
|
param([object[]]$ArgumentList)
|
|
|
|
return (@(
|
|
foreach ($argument in $ArgumentList) {
|
|
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
|
}
|
|
) -join ' ')
|
|
}
|
|
|
|
function Start-PinnedPriorProcess {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role,
|
|
[string]$StdoutLog,
|
|
[string]$StderrLog
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
|
|
throw "Prior $Role executable is unavailable"
|
|
}
|
|
$actualExecutableSha256 = (
|
|
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
|
|
throw "Prior $Role executable SHA256 drift"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
|
throw "Prior $Role working directory is unavailable"
|
|
}
|
|
if (@($Identity.argument_list).Count -eq 0) {
|
|
throw "Prior $Role argument list is unavailable"
|
|
}
|
|
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
|
|
throw "Prior $Role environment is unavailable"
|
|
}
|
|
|
|
$callerEnvironment = Save-CompleteProcessEnvironment
|
|
try {
|
|
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
|
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
|
return Start-Process -WindowStyle Hidden `
|
|
-FilePath $Identity.executable_path `
|
|
-ArgumentList $argumentString `
|
|
-WorkingDirectory $Identity.cwd `
|
|
-RedirectStandardOutput $StdoutLog `
|
|
-RedirectStandardError $StderrLog `
|
|
-PassThru
|
|
} finally {
|
|
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
|
}
|
|
}
|
|
|
|
function Restore-PriorPublicRuntime {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApi,
|
|
[System.Collections.IDictionary]$PriorCloudflared,
|
|
[System.Collections.IDictionary]$PriorLocalVoiceContract,
|
|
[System.Collections.IDictionary]$PriorPublicVoiceContract,
|
|
[System.Collections.IDictionary]$EnvironmentSnapshot,
|
|
[string]$ConfigPath,
|
|
[int]$ApiPortValue,
|
|
[string]$HealthUrl,
|
|
[string]$VoiceHealthUrl,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$null = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "app.main:app" `
|
|
-Port $ApiPortValue `
|
|
-TimeoutSec $TimeoutSec
|
|
)
|
|
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
|
|
$priorApiProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorApi `
|
|
-Role "api" `
|
|
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
|
|
$priorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiProcess.Id `
|
|
-Role "restored prior api" `
|
|
-ExpectedCwd $PriorApi.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
|
|
throw "Restored prior API identity drift: $field"
|
|
}
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Restored prior runtime does not have the exact local voice sidecars"
|
|
}
|
|
|
|
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
|
|
$currentCloudflared = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedConfigPath `
|
|
-ExactPath
|
|
)
|
|
$null = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $currentCloudflared `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "failed fresh cloudflared"
|
|
)
|
|
$priorCloudflaredProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorCloudflared `
|
|
-Role "cloudflared" `
|
|
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
|
|
$priorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcess.Id `
|
|
-Role "restored prior cloudflared" `
|
|
-ExpectedCwd $PriorCloudflared.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
|
|
throw "Restored prior cloudflared identity drift: $field"
|
|
}
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $HealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri $VoiceHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
|
|
return [ordered]@{
|
|
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
|
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
|
|
local_health = $true
|
|
local_voice_health = $true
|
|
public_health = $true
|
|
public_voice_health = $true
|
|
}
|
|
}
|
|
|
|
function Stop-NodeByPortHint {
|
|
param([int]$Port)
|
|
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "node.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*vite*preview*" -and
|
|
$_.CommandLine -like "*$Port*"
|
|
} |
|
|
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
|
}
|
|
|
|
function ConvertTo-CompactJson {
|
|
param([object]$Value)
|
|
ConvertTo-Json -InputObject $Value -Compress
|
|
}
|
|
|
|
function Invoke-StableGitText {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string[]]$Arguments
|
|
)
|
|
|
|
$value = & git.exe -C $SourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Initialize-RuntimeProvenanceOutput {
|
|
param([string]$OutputPath)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($OutputPath)) {
|
|
throw "Fresh public provenance output path must be absolute"
|
|
}
|
|
|
|
try {
|
|
$resolvedOutputPath = [System.IO.Path]::GetFullPath($OutputPath)
|
|
$outputDirectory = Split-Path -Parent $resolvedOutputPath
|
|
if (-not $outputDirectory) {
|
|
throw "Fresh public provenance output path has no parent directory"
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Container) {
|
|
throw "Fresh public provenance output path is a directory: $resolvedOutputPath"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
throw "Fresh public provenance output directory is unavailable: $outputDirectory"
|
|
}
|
|
|
|
# 기존 receipt가 잠겨 있거나 read-only라면 프로세스 교체 전에 실패해야 한다.
|
|
# sibling probe 두 개를 atomic replace해 디렉터리의 create/flush/replace/delete
|
|
# 권한도 미리 검증한다. 실제 receipt 내용은 이 단계에서 건드리지 않는다.
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Leaf) {
|
|
$attributes = [System.IO.File]::GetAttributes($resolvedOutputPath)
|
|
if (($attributes -band [System.IO.FileAttributes]::ReadOnly) -ne 0) {
|
|
throw "Fresh public provenance output is read-only: $resolvedOutputPath"
|
|
}
|
|
$existingStream = [System.IO.File]::Open(
|
|
$resolvedOutputPath,
|
|
[System.IO.FileMode]::Open,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::Read
|
|
)
|
|
$existingStream.Dispose()
|
|
}
|
|
|
|
$probeId = [Guid]::NewGuid().ToString("N")
|
|
$probeSource = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.source.tmp"
|
|
$probeTarget = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.target.tmp"
|
|
$probeBackup = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.backup.tmp"
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
[System.IO.File]::WriteAllText($probeSource, "probe-source", $encoding)
|
|
[System.IO.File]::WriteAllText($probeTarget, "probe-target", $encoding)
|
|
[System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup)
|
|
[System.IO.File]::Delete($probeTarget)
|
|
[System.IO.File]::Delete($probeBackup)
|
|
} finally {
|
|
foreach ($probePath in @($probeSource, $probeTarget, $probeBackup)) {
|
|
if ($probePath -and [System.IO.File]::Exists($probePath)) {
|
|
[System.IO.File]::Delete($probePath)
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
throw "Fresh public provenance output preflight failed before runtime mutation: $($_.Exception.Message)"
|
|
}
|
|
|
|
return $resolvedOutputPath
|
|
}
|
|
|
|
function Write-Utf8TextAtomically {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$Value
|
|
)
|
|
|
|
$outputDirectory = Split-Path -Parent $OutputPath
|
|
$temporaryPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).tmp"
|
|
$backupPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).backup.tmp"
|
|
$published = $false
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
$bytes = $encoding.GetBytes($Value)
|
|
$stream = [System.IO.FileStream]::new(
|
|
$temporaryPath,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
try {
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
} finally {
|
|
$stream.Dispose()
|
|
}
|
|
|
|
if ([System.IO.File]::Exists($OutputPath)) {
|
|
[System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath)
|
|
} elseif (Test-Path -LiteralPath $OutputPath) {
|
|
throw "Fresh public provenance output became a non-file before commit: $OutputPath"
|
|
} else {
|
|
[System.IO.File]::Move($temporaryPath, $OutputPath)
|
|
}
|
|
$published = $true
|
|
} finally {
|
|
if ([System.IO.File]::Exists($temporaryPath)) {
|
|
[System.IO.File]::Delete($temporaryPath)
|
|
}
|
|
if ([System.IO.File]::Exists($backupPath)) {
|
|
try {
|
|
[System.IO.File]::Delete($backupPath)
|
|
} catch {
|
|
if ($published) {
|
|
Write-Warning "Atomic provenance receipt was published, but its temporary backup could not be removed: $backupPath"
|
|
} else {
|
|
throw
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Write-FailedFreshPromotionEvidence {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$FailureStage,
|
|
[bool]$RollbackSucceeded,
|
|
[object]$RollbackResult,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree
|
|
)
|
|
|
|
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
|
|
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
|
|
$failedPath = "$OutputPath.failed.log"
|
|
$payload = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-failure.v1"
|
|
status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" }
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
failure_stage = $FailureStage
|
|
source = [ordered]@{
|
|
git_commit = $SourceCommit.ToLowerInvariant()
|
|
git_tree = $SourceTree.ToLowerInvariant()
|
|
}
|
|
rollback = [ordered]@{
|
|
attempted = $true
|
|
succeeded = $RollbackSucceeded
|
|
result = $RollbackResult
|
|
}
|
|
}
|
|
$json = ConvertTo-Json -InputObject $payload -Depth 8
|
|
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
|
|
return $failedPath
|
|
}
|
|
|
|
function Assert-FreshPublicProvenanceContract {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree,
|
|
[string]$PythonPath,
|
|
[string]$PythonSha256,
|
|
[string]$CloudflaredPath,
|
|
[string]$CloudflaredSha256,
|
|
[string]$ConfigPath,
|
|
[string]$ConfigSha256
|
|
)
|
|
|
|
if (-not $ForceApiRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -ForceApiRestart"
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
|
|
}
|
|
if (-not $SkipEngineRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
|
|
}
|
|
if (-not $SkipWebRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
|
|
}
|
|
if ($RouteCloudflareDns) {
|
|
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
|
|
}
|
|
if (-not [string]::Equals(
|
|
$PublicHealthUrl,
|
|
$CanonicalPublicHealthUrl,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion requires the canonical HTTPS public health URL"
|
|
}
|
|
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
|
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
|
|
throw "Fresh public provenance requires exact source commit and tree pins"
|
|
}
|
|
}
|
|
foreach ($shaPin in @($PythonSha256, $CloudflaredSha256, $ConfigSha256)) {
|
|
if ($shaPin -notmatch "^[0-9a-fA-F]{64}$") {
|
|
throw "Fresh public provenance requires exact Python, cloudflared, and config SHA256 pins"
|
|
}
|
|
}
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path
|
|
$expectedStartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$runningStartScript = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningStartScript,
|
|
(Resolve-Path -LiteralPath $expectedStartScript).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion must execute the launcher from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Fresh public promotion requires detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $SourceCommit.ToLowerInvariant()) {
|
|
throw "Fresh public source commit drift: expected=$SourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $SourceTree.ToLowerInvariant()) {
|
|
throw "Fresh public source tree drift: expected=$SourceTree actual=$actualTree"
|
|
}
|
|
$dirty = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Fresh public promotion requires a clean stable source"
|
|
}
|
|
|
|
foreach ($pin in @(
|
|
[pscustomobject]@{ Path = $PythonPath; Sha256 = $PythonSha256; Label = "Python" },
|
|
[pscustomobject]@{ Path = $CloudflaredPath; Sha256 = $CloudflaredSha256; Label = "cloudflared" },
|
|
[pscustomobject]@{ Path = $ConfigPath; Sha256 = $ConfigSha256; Label = "cloudflared config" }
|
|
)) {
|
|
if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) {
|
|
throw "Pinned $($pin.Label) file not found at $($pin.Path)"
|
|
}
|
|
$actualSha256 = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualSha256 -ne $pin.Sha256.ToLowerInvariant()) {
|
|
throw "Pinned $($pin.Label) SHA256 drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Set-CloudflaredIngress {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[string[]]$ApiHostnames,
|
|
[string[]]$WebHostnames,
|
|
[int]$ApiPortValue,
|
|
[int]$WebPortValue,
|
|
[switch]$RequireUnchanged
|
|
)
|
|
|
|
$lines = Get-Content -Encoding UTF8 -Path $ConfigPath
|
|
$ingressIndex = -1
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -match "^\s*ingress:\s*$") {
|
|
$ingressIndex = $i
|
|
break
|
|
}
|
|
}
|
|
if ($ingressIndex -lt 0) {
|
|
throw "Could not find ingress: in $ConfigPath"
|
|
}
|
|
|
|
$nextLines = @()
|
|
if ($ingressIndex -gt 0) {
|
|
$nextLines += $lines[0..($ingressIndex - 1)]
|
|
}
|
|
$nextLines += "ingress:"
|
|
foreach ($hostname in $WebHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$WebPortValue"
|
|
}
|
|
foreach ($hostname in $ApiHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$ApiPortValue"
|
|
}
|
|
$nextLines += " - service: http_status:404"
|
|
|
|
$matches = $lines.Count -eq $nextLines.Count
|
|
if ($matches) {
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -cne $nextLines[$i]) {
|
|
$matches = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if ($RequireUnchanged -and -not $matches) {
|
|
throw "Pinned cloudflared config ingress does not match the requested public topology"
|
|
}
|
|
if (-not $matches) {
|
|
Set-Content -Encoding UTF8 -Path $ConfigPath -Value $nextLines
|
|
}
|
|
}
|
|
|
|
$freshMutationStarted = $false
|
|
$freshPromotionCommitted = $false
|
|
$freshFailureStage = "preflight"
|
|
$freshPriorApiIdentity = $null
|
|
$freshPriorCloudflaredIdentity = $null
|
|
$freshPriorLocalVoiceContract = $null
|
|
$freshPriorPublicVoiceContract = $null
|
|
$freshEnvironmentSnapshot = $null
|
|
$resolvedRuntimeProvenancePath = $null
|
|
$freshManagedEnvironmentNames = @(
|
|
"ENVIRONMENT",
|
|
"ENGINE_URL",
|
|
"ENGINE_MODE",
|
|
"VIGNETTE_LIVE_CLIENT_PROVIDER",
|
|
"AUTH_DEV_LOGIN_ENABLED",
|
|
"AUTO_SEED_PERSONAS",
|
|
"ALLOW_SEED_PERSONA_FALLBACK",
|
|
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
|
|
"VIGNETTE_VOICE_STT_PROVIDER",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_URL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
|
|
"VIGNETTE_VOICE_TTS_PROVIDER",
|
|
"VIGNETTE_MELOTTS_TTS_URL",
|
|
"FRONTEND_BASE_URL",
|
|
"CORS_ORIGINS",
|
|
"FRONTEND_ORIGIN_MAP"
|
|
)
|
|
|
|
trap {
|
|
$caught = $_
|
|
if (
|
|
$RequireFreshPublicProvenance -and
|
|
$freshMutationStarted -and
|
|
-not $freshPromotionCommitted
|
|
) {
|
|
$rollbackResult = $null
|
|
$rollbackSucceeded = $false
|
|
$rollbackFailureType = "none"
|
|
try {
|
|
$rollbackResult = Restore-PriorPublicRuntime `
|
|
-PriorApi $freshPriorApiIdentity `
|
|
-PriorCloudflared $freshPriorCloudflaredIdentity `
|
|
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
|
|
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
|
|
-EnvironmentSnapshot $freshEnvironmentSnapshot `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiPortValue $ApiPort `
|
|
-HealthUrl $PublicHealthUrl `
|
|
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$rollbackSucceeded = $true
|
|
} catch {
|
|
$rollbackFailureType = $_.Exception.GetType().Name
|
|
}
|
|
|
|
$failedEvidencePath = ""
|
|
if ($resolvedRuntimeProvenancePath) {
|
|
try {
|
|
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-FailureStage $freshFailureStage `
|
|
-RollbackSucceeded $rollbackSucceeded `
|
|
-RollbackResult $rollbackResult `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree
|
|
} catch {
|
|
$failedEvidencePath = "unavailable"
|
|
}
|
|
}
|
|
|
|
if ($rollbackSucceeded) {
|
|
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw $caught
|
|
}
|
|
|
|
if (!(Test-Path $Python)) {
|
|
throw "Python 3.11 not found at $Python"
|
|
}
|
|
if (!(Test-Path $ApiDir)) {
|
|
throw "API directory not found at $ApiDir"
|
|
}
|
|
if (!(Test-Path $WebDir)) {
|
|
throw "Web directory not found at $WebDir"
|
|
}
|
|
foreach ($voiceScript in @($WhisperStartScript, $MeloTtsStartScript, $VoiceSidecarProbe)) {
|
|
if (!(Test-Path -LiteralPath $voiceScript)) {
|
|
throw "Voice sidecar prerequisite not found at $voiceScript"
|
|
}
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
if (!(Test-Path -LiteralPath $Cloudflared -PathType Leaf)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path -LiteralPath $CloudflaredConfig -PathType Leaf)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
if (-not $RuntimeProvenancePath) {
|
|
$RuntimeProvenancePath = Join-Path $Workspace "public-runtime-launch-provenance.log"
|
|
}
|
|
|
|
Assert-FreshPublicProvenanceContract `
|
|
-SourceRoot $Workspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256 `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredSha256 $ExpectedCloudflaredSha256 `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ConfigSha256 $ExpectedCloudflaredConfigSha256
|
|
|
|
$resolvedRuntimeProvenancePath = Initialize-RuntimeProvenanceOutput `
|
|
-OutputPath $RuntimeProvenancePath
|
|
|
|
# 승격 모드에서 config를 재작성하면 사전 pin과 실제 tunnel 입력이 달라진다.
|
|
# exact ingress가 이미 들어 있는 경우에만 이후 프로세스 mutation으로 진행한다.
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort `
|
|
-RequireUnchanged
|
|
|
|
$priorApiProcesses = @(
|
|
Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort
|
|
)
|
|
if ($priorApiProcesses.Count -ne 1) {
|
|
throw "Fresh public promotion requires exactly one prior API process for transactional rollback"
|
|
}
|
|
$priorCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
|
|
-ExactPath
|
|
)
|
|
if ($priorCloudflaredProcesses.Count -ne 1) {
|
|
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
|
|
}
|
|
$freshPriorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiProcesses[0].ProcessId `
|
|
-Role "prior api" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
|
|
-Role "prior cloudflared" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorLocalVoiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorLocalVoiceHealth
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorPublicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorPublicVoiceHealth
|
|
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
|
|
-Names $freshManagedEnvironmentNames
|
|
}
|
|
|
|
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
|
# 프로세스는 살아 있는데 그 프로세스의 claude 세션만 죽은 상태는 /health를 통과하므로,
|
|
# /health 기준으로는 복구가 필요한 순간에 오히려 재기동을 건너뛴다(2026-08-07 사고).
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
$engineReady = $false
|
|
if ($null -ne $engineHealth -and $engineHealth.ok) {
|
|
$engineReady = Test-EngineReady -Port $EnginePort
|
|
}
|
|
if ($SkipEngineRestart) {
|
|
if (-not $engineReady) {
|
|
throw "Engine gateway is not ready on http://127.0.0.1:$EnginePort/ready"
|
|
}
|
|
} elseif (-not $engineReady) {
|
|
$null = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "engine_gateway.gateway:app" `
|
|
-Port $EnginePort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
|
|
# Start-Process는 리다이렉트 대상 로그를 덮어쓴다. 직전 사고 로그를 보존해야
|
|
# 재기동 후에도 원인을 추적할 수 있다.
|
|
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
|
|
if (Test-Path $logFile) {
|
|
# stable detached source의 provenance gate는 untracked 파일도 차단한다.
|
|
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
|
|
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @("-m", "uvicorn", "engine_gateway.gateway:app", "--host", "127.0.0.1", "--port", "$EnginePort") `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $EngineOutLog `
|
|
-RedirectStandardError $EngineErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
$engineReady = Wait-EngineReady -Port $EnginePort -TimeoutSec 90
|
|
if (-not $engineReady) {
|
|
Write-Warning "Engine gateway is still degraded; continuing admin/auth recovery"
|
|
}
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
}
|
|
|
|
$env:ENVIRONMENT = "prod"
|
|
$env:ENGINE_URL = "http://127.0.0.1:$EnginePort"
|
|
$env:ENGINE_MODE = "claude_cli"
|
|
$env:VIGNETTE_LIVE_CLIENT_PROVIDER = "claude_cli"
|
|
$env:AUTH_DEV_LOGIN_ENABLED = "false"
|
|
$env:AUTO_SEED_PERSONAS = "false"
|
|
$env:ALLOW_SEED_PERSONA_FALLBACK = "false"
|
|
$env:VIGNETTE_VOICE_POC_SAMPLE_TTS = "false"
|
|
$env:VIGNETTE_VOICE_STT_PROVIDER = "local_whisper"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_URL = "ws://127.0.0.1:$WhisperPort/v1/listen"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_MODEL = $WhisperModel
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE = $WhisperLanguage
|
|
$env:VIGNETTE_VOICE_TTS_PROVIDER = "melotts"
|
|
$env:VIGNETTE_MELOTTS_TTS_URL = "http://127.0.0.1:$MeloTtsPort"
|
|
$env:FRONTEND_BASE_URL = "https://vignette.chanpaca.net"
|
|
$frontendOrigins = @("https://vignette.chanpaca.net", "https://vnet.18ka.net", "https://vignette-b1q.pages.dev")
|
|
$localViteOrigins = @()
|
|
foreach ($port in 5170..5180) {
|
|
$localViteOrigins += "http://localhost:$port"
|
|
$localViteOrigins += "http://127.0.0.1:$port"
|
|
}
|
|
$env:CORS_ORIGINS = ConvertTo-CompactJson -Value ($frontendOrigins + $localViteOrigins)
|
|
$env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
|
|
"api-vignette.chanpaca.net" = "https://vignette.chanpaca.net"
|
|
"api-vnet.18ka.net" = "https://vnet.18ka.net"
|
|
})
|
|
|
|
# 포트 리스너만으로는 올바른 provider/model을 증명하지 못한다. 첫 WS ready
|
|
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
|
|
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $WhisperPort) {
|
|
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
|
|
}
|
|
& $WhisperStartScript `
|
|
-Port $WhisperPort `
|
|
-Model $WhisperModel `
|
|
-Device $WhisperDevice `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "stt" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "local_whisper/$WhisperModel/$WhisperDevice did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
if (-not (Test-VoiceSidecarReady -Component "tts")) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $MeloTtsPort) {
|
|
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
|
|
}
|
|
& $MeloTtsStartScript `
|
|
-Port $MeloTtsPort `
|
|
-Language $MeloTtsLanguage `
|
|
-Device "cpu" `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "tts" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "melotts/$MeloTtsModel did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
# 두 sidecar를 한 번 더 함께 검사해 개별 probe 사이의 TOCTOU를 닫는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Voice sidecar readiness changed before the API restart gate"
|
|
}
|
|
|
|
$health = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/health"
|
|
$voiceHealth = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/voice/health"
|
|
$apiControlPlaneReady = (
|
|
$null -ne $health -and
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true -and
|
|
(Test-VoiceApiReady -Health $voiceHealth)
|
|
)
|
|
$proc = $null
|
|
$apiStoppedProcessIds = @()
|
|
$apiLaunchIdentity = $null
|
|
if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
|
Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
|
|
} else {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "api_cutover"
|
|
$freshMutationStarted = $true
|
|
}
|
|
$apiStoppedProcessIds = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "app.main:app" `
|
|
-Port $ApiPort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
|
|
$proc = Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @(
|
|
"-m", "uvicorn", "app.main:app",
|
|
"--host", "127.0.0.1",
|
|
"--port", "$ApiPort",
|
|
"--ws", "websockets",
|
|
"--ws-max-queue", "4"
|
|
) `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $OutLog `
|
|
-RedirectStandardError $ErrLog `
|
|
-PassThru
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
if ($apiStoppedProcessIds -contains $proc.Id) {
|
|
throw "Fresh public API did not receive a replacement PID"
|
|
}
|
|
$apiLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $proc.Id `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($apiLaunchIdentity.executable_sha256 -ne $ExpectedPythonSha256.ToLowerInvariant()) {
|
|
throw "Fresh public API executable SHA256 does not match the pinned Python"
|
|
}
|
|
foreach ($requiredArgument in @("uvicorn", "app.main:app", "--port", "$ApiPort", "--ws-max-queue", "4")) {
|
|
if ($apiLaunchIdentity.command_line.IndexOf($requiredArgument, [System.StringComparison]::Ordinal) -lt 0) {
|
|
throw "Fresh public API command line is missing required argument: $requiredArgument"
|
|
}
|
|
}
|
|
}
|
|
|
|
Start-Sleep -Seconds 3
|
|
$health = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$voiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
}
|
|
if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) {
|
|
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
|
|
}
|
|
if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
|
|
throw "Public voice API does not match the exact local provider/model contract: $($voiceHealth | ConvertTo-Json -Compress)"
|
|
}
|
|
|
|
if (!$SkipWebRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "web_preview"
|
|
}
|
|
Stop-NodeByPortHint -Port $WebPort
|
|
|
|
$build = Start-Process -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run build") `
|
|
-WorkingDirectory $WebDir `
|
|
-NoNewWindow `
|
|
-Wait `
|
|
-PassThru
|
|
if ($build.ExitCode -ne 0) {
|
|
throw "Web build failed with exit code $($build.ExitCode)"
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run preview -- --host 127.0.0.1 --port $WebPort") `
|
|
-WorkingDirectory $WebDir `
|
|
-RedirectStandardOutput $WebOutLog `
|
|
-RedirectStandardError $WebErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
Wait-HttpStatus -Uri "http://127.0.0.1:$WebPort/" -TimeoutSec 30 | Out-Null
|
|
}
|
|
|
|
$cloudflaredProcess = $null
|
|
$cloudflaredLaunchIdentity = $null
|
|
$cloudflaredStoppedProcessIds = @()
|
|
if (!$SkipCloudflaredRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "cloudflared_cutover"
|
|
}
|
|
if (!(Test-Path $Cloudflared)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path $CloudflaredConfig)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
|
|
if ($RouteCloudflareDns) {
|
|
foreach ($hostname in ($PublicWebHostnames + $PublicApiHostnames)) {
|
|
& $Cloudflared tunnel route dns $CloudflareTunnelName $hostname
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Warning "cloudflared DNS route failed for $hostname"
|
|
}
|
|
}
|
|
}
|
|
|
|
if (-not $RequireFreshPublicProvenance) {
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort
|
|
}
|
|
|
|
$resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
if ($RequireFreshPublicProvenance) {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
} else {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig -ConfigPath $resolvedCloudflaredConfig
|
|
)
|
|
}
|
|
$cloudflaredStoppedProcessIds = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $existingCloudflaredProcesses `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "cloudflared for $resolvedCloudflaredConfig"
|
|
)
|
|
|
|
$cloudflaredProcess = Start-Process -WindowStyle Hidden -FilePath $Cloudflared `
|
|
-ArgumentList @("tunnel", "--config", $resolvedCloudflaredConfig, "run") `
|
|
-WorkingDirectory $Workspace `
|
|
-RedirectStandardOutput (Join-Path $Workspace "cloudflared.public.out.log") `
|
|
-RedirectStandardError (Join-Path $Workspace "cloudflared.public.err.log") `
|
|
-PassThru
|
|
|
|
if ($cloudflaredStoppedProcessIds -contains $cloudflaredProcess.Id) {
|
|
throw "Cloudflared did not receive a replacement PID"
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
$cloudflaredLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredProcess.Id `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($cloudflaredLaunchIdentity.executable_sha256 -ne $ExpectedCloudflaredSha256.ToLowerInvariant()) {
|
|
throw "Fresh cloudflared executable SHA256 does not match its pin"
|
|
}
|
|
if ($cloudflaredLaunchIdentity.command_line.IndexOf($resolvedCloudflaredConfig, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) {
|
|
throw "Fresh cloudflared command line is not pinned to the expected config"
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "identity_revalidation"
|
|
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
|
|
throw "Fresh public promotion did not produce both API and cloudflared identities"
|
|
}
|
|
|
|
$apiFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $apiLaunchIdentity.pid `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$cloudflaredFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredLaunchIdentity.pid `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
foreach ($identityPair in @(
|
|
[pscustomobject]@{ Role = "api"; Launch = $apiLaunchIdentity; Final = $apiFinalIdentity },
|
|
[pscustomobject]@{ Role = "cloudflared"; Launch = $cloudflaredLaunchIdentity; Final = $cloudflaredFinalIdentity }
|
|
)) {
|
|
foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($identityPair.Launch[$field].ToString() -cne $identityPair.Final[$field].ToString()) {
|
|
throw "Fresh $($identityPair.Role) provenance drifted before receipt: $field"
|
|
}
|
|
}
|
|
}
|
|
|
|
$finalConfigSha256 = (Get-FileHash -LiteralPath $CloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
|
|
throw "Pinned cloudflared config drifted before provenance receipt"
|
|
}
|
|
$freshFailureStage = "public_health_validation"
|
|
$publicHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$publicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$publicOpenApi = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicOpenApiUrl `
|
|
-IsHealthy {
|
|
param($document)
|
|
Test-RequiredOpenApiPaths `
|
|
-Document $document `
|
|
-RequiredPaths $RequiredPublicApiPaths
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Exact local voice sidecars changed before provenance receipt"
|
|
}
|
|
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
|
|
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
|
|
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
|
|
throw "Could not record the psutil version used for process provenance"
|
|
}
|
|
$safeApiIdentity = ConvertTo-SafeProcessIdentity -Identity $apiFinalIdentity
|
|
$safeCloudflaredIdentity = ConvertTo-SafeProcessIdentity -Identity $cloudflaredFinalIdentity
|
|
|
|
$provenance = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-provenance.v1"
|
|
status = "passed"
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
source = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_commit = $ExpectedSourceCommit.ToLowerInvariant()
|
|
git_tree = $ExpectedSourceTree.ToLowerInvariant()
|
|
launcher_sha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
clean_detached_head = $true
|
|
}
|
|
config = [ordered]@{
|
|
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
sha256 = $finalConfigSha256
|
|
}
|
|
public_validation = [ordered]@{
|
|
health_url = $CanonicalPublicHealthUrl
|
|
health = $true
|
|
voice_health_url = $CanonicalPublicVoiceHealthUrl
|
|
voice_health = $true
|
|
openapi_url = $CanonicalPublicOpenApiUrl
|
|
required_openapi_paths = @($RequiredPublicApiPaths)
|
|
openapi = $true
|
|
local_voice_sidecars = $true
|
|
}
|
|
replacement = [ordered]@{
|
|
api_stopped_pids = @($apiStoppedProcessIds)
|
|
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
|
|
api_new_pid = [int]$apiFinalIdentity.pid
|
|
cloudflared_new_pid = [int]$cloudflaredFinalIdentity.pid
|
|
}
|
|
processes = [ordered]@{
|
|
api = $safeApiIdentity
|
|
cloudflared = $safeCloudflaredIdentity
|
|
}
|
|
topology_inputs = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_sha = $ExpectedSourceCommit.ToLowerInvariant()
|
|
api_pid = [int]$apiFinalIdentity.pid
|
|
api_started_at_utc = $apiFinalIdentity.started_at_utc
|
|
api_executable_name = $apiFinalIdentity.executable_name
|
|
api_executable_sha256 = $apiFinalIdentity.executable_sha256
|
|
api_command_line_sha256 = $apiFinalIdentity.command_line_sha256
|
|
api_cwd = $apiFinalIdentity.cwd
|
|
api_listen_port = $ApiPort
|
|
cloudflared_pid = [int]$cloudflaredFinalIdentity.pid
|
|
cloudflared_started_at_utc = $cloudflaredFinalIdentity.started_at_utc
|
|
cloudflared_executable_name = $cloudflaredFinalIdentity.executable_name
|
|
cloudflared_executable_sha256 = $cloudflaredFinalIdentity.executable_sha256
|
|
cloudflared_command_line_sha256 = $cloudflaredFinalIdentity.command_line_sha256
|
|
cloudflared_cwd = $cloudflaredFinalIdentity.cwd
|
|
psutil_version = $psutilVersion
|
|
}
|
|
}
|
|
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
|
|
$freshFailureStage = "receipt_publish"
|
|
try {
|
|
Write-Utf8TextAtomically `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-Value ($provenanceJson + [Environment]::NewLine)
|
|
} catch {
|
|
# 새 PID들은 이미 health/identity gate를 통과했지만, atomic receipt가 없으면
|
|
# 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다.
|
|
throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)"
|
|
}
|
|
$freshPromotionCommitted = $true
|
|
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
|
|
}
|
|
|
|
if ($engineReady) {
|
|
Write-Output "Engine gateway ready (real generation proven) on http://127.0.0.1:$EnginePort"
|
|
} else {
|
|
Write-Warning "Engine gateway degraded; admin/auth control plane remains available"
|
|
}
|
|
if ($null -ne $proc) {
|
|
Write-Output "Public API running on http://127.0.0.1:$ApiPort with PID $($proc.Id)"
|
|
} else {
|
|
Write-Output "Public API kept running on http://127.0.0.1:$ApiPort"
|
|
}
|
|
if (!$SkipWebRestart) {
|
|
Write-Output "Public vnet web preview running on http://127.0.0.1:$WebPort"
|
|
}
|
|
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
|
|
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
|