3315 lines
120 KiB
PowerShell
3315 lines
120 KiB
PowerShell
param(
|
|
[string]$Workspace = "D:\workspace\vignette",
|
|
[int]$ApiPort = 8001,
|
|
[int]$WebPort = 5174,
|
|
[int]$EnginePort = 9099,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[int]$VoiceSidecarReadySeconds = 300,
|
|
[ValidateRange(30, 600)]
|
|
[int]$ApiReadySeconds = 180,
|
|
[ValidateRange(30, 300)]
|
|
[int]$VoiceApiReadySeconds = 90,
|
|
[ValidateRange(60, 1800)]
|
|
[int]$WebBuildTimeoutSeconds = 600,
|
|
[string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock",
|
|
[ValidateRange(0, 300)]
|
|
[int]$RecoveryLockWaitSeconds = 0,
|
|
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
|
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
|
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$UserUploadDir,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$UserUploadManifestPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{64}$")]
|
|
[string]$ExpectedUserUploadManifestSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$UserUploadWriteFreezePath,
|
|
[switch]$SkipEngineRestart,
|
|
[switch]$ForceApiRestart,
|
|
[switch]$SkipWebRestart,
|
|
[switch]$RouteCloudflareDns,
|
|
[string]$CloudflareTunnelName = "vignette",
|
|
[switch]$SkipCloudflaredRestart,
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[switch]$RequireFreshPublicProvenance,
|
|
[string]$ExpectedSourceCommit = "",
|
|
[string]$ExpectedSourceTree = "",
|
|
[string]$ExpectedPythonSha256 = "",
|
|
[string]$ExpectedCloudflaredSha256 = "",
|
|
[string]$ExpectedCloudflaredConfigSha256 = "",
|
|
[string]$RuntimeProvenancePath = "",
|
|
[ValidateRange(1, 60)]
|
|
[int]$ProcessStopTimeoutSeconds = 15,
|
|
[string[]]$CoordinatedTaskNames = @(
|
|
"VignettePublicRuntimeWatchdog",
|
|
"VignettePublicRuntime"
|
|
),
|
|
[string]$OfflineBootstrapQuiescenceReceiptPath = "",
|
|
[ValidatePattern("^$|^[0-9a-f]{64}$")]
|
|
[string]$ExpectedOfflineBootstrapQuiescenceReceiptSha256 = "",
|
|
[ValidatePattern("^$|^[0-9a-f]{40}$")]
|
|
[string]$ExpectedOfflineBootstrapLegacySourceCommit = "",
|
|
[ValidatePattern("^$|^[0-9a-f]{40}$")]
|
|
[string]$ExpectedOfflineBootstrapLegacySourceTree = "",
|
|
[string]$InheritedRecoveryLockReceiptPath = "",
|
|
[ValidatePattern("^$|^[0-9a-f]{64}$")]
|
|
[string]$ExpectedInheritedRecoveryLockReceiptSha256 = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Repair-CaseInsensitiveProcessEnvironment {
|
|
# Windows 환경 변수 이름은 대소문자를 구분하지 않지만, 비-Windows 부모가 만든
|
|
# 프로세스 블록에는 Path/PATH 같은 충돌 키가 함께 들어올 수 있다. Windows
|
|
# PowerShell 5.1의 Start-Process는 이 블록을 case-insensitive 사전으로 옮기다가
|
|
# 충돌하므로, 실행 전 .NET의 Windows 환경 갱신 경로를 한 번 거쳐 정규화한다.
|
|
# .NET Framework의 GetEnvironmentVariables()는 충돌 키를 이미 하나로 접어서
|
|
# 보여주므로 개수를 세어서는 원본 환경 블록의 중복을 발견할 수 없다. 현재
|
|
# 프로세스가 실제로 해석하는 값을 먼저 보존하고 두 대표 casing을 각각 지운 뒤
|
|
# canonical `Path` 하나만 다시 만든다.
|
|
$effectivePath = [System.Environment]::GetEnvironmentVariable(
|
|
"Path",
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
if ([string]::IsNullOrWhiteSpace($effectivePath)) {
|
|
throw "Process Path environment is empty"
|
|
}
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
"Path",
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
"PATH",
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
"Path",
|
|
$effectivePath,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
|
|
Repair-CaseInsensitiveProcessEnvironment
|
|
|
|
$resolvedWorkspace = (Resolve-Path -LiteralPath $Workspace).Path
|
|
$uploadRootContract = Join-Path $resolvedWorkspace "scripts\public-runtime-upload-root.ps1"
|
|
$uploadRootProbe = Join-Path $resolvedWorkspace "scripts\probe-public-runtime-upload-root.py"
|
|
$uploadManifestProbe = Join-Path $resolvedWorkspace "scripts\validate-public-runtime-upload-manifest.py"
|
|
$databaseIdentityHelper = Join-Path $resolvedWorkspace "scripts\public_runtime_database_identity.py"
|
|
$offlineQuiescenceProbe = Join-Path $resolvedWorkspace "scripts\validate-public-runtime-offline-quiescence.py"
|
|
$taskMaintenanceContract = Join-Path $resolvedWorkspace "scripts\public-runtime-task-maintenance.ps1"
|
|
$taskDefinitionCutoverContract = Join-Path $resolvedWorkspace "scripts\public-runtime-task-definition-cutover.ps1"
|
|
$bootTaskInstaller = Join-Path $resolvedWorkspace "scripts\register-boot-task.ps1"
|
|
$watchdogTaskInstaller = Join-Path $resolvedWorkspace "scripts\install-public-runtime-task.ps1"
|
|
foreach ($uploadContractFile in @(
|
|
$uploadRootContract,
|
|
$uploadRootProbe,
|
|
$uploadManifestProbe,
|
|
$databaseIdentityHelper,
|
|
$offlineQuiescenceProbe,
|
|
$taskMaintenanceContract,
|
|
$taskDefinitionCutoverContract,
|
|
$bootTaskInstaller,
|
|
$watchdogTaskInstaller
|
|
)) {
|
|
if (-not (Test-Path -LiteralPath $uploadContractFile -PathType Leaf)) {
|
|
throw "Public runtime upload-root contract file not found: $uploadContractFile"
|
|
}
|
|
}
|
|
$Workspace = $resolvedWorkspace
|
|
. $taskMaintenanceContract
|
|
. $taskDefinitionCutoverContract
|
|
$offlineReceiptPathPresent = -not [string]::IsNullOrWhiteSpace(
|
|
$OfflineBootstrapQuiescenceReceiptPath
|
|
)
|
|
$offlineReceiptHashPresent = -not [string]::IsNullOrWhiteSpace(
|
|
$ExpectedOfflineBootstrapQuiescenceReceiptSha256
|
|
)
|
|
if ($offlineReceiptPathPresent -ne $offlineReceiptHashPresent) {
|
|
throw "Offline bootstrap quiescence receipt path and lowercase SHA256 are required together"
|
|
}
|
|
$offlineBootstrapMode = $offlineReceiptPathPresent -and $offlineReceiptHashPresent
|
|
$offlineLegacyPinsPresent = (
|
|
-not [string]::IsNullOrWhiteSpace($ExpectedOfflineBootstrapLegacySourceCommit) -and
|
|
-not [string]::IsNullOrWhiteSpace($ExpectedOfflineBootstrapLegacySourceTree)
|
|
)
|
|
$inheritedLockPathPresent = -not [string]::IsNullOrWhiteSpace(
|
|
$InheritedRecoveryLockReceiptPath
|
|
)
|
|
$inheritedLockHashPresent = -not [string]::IsNullOrWhiteSpace(
|
|
$ExpectedInheritedRecoveryLockReceiptSha256
|
|
)
|
|
if ($inheritedLockPathPresent -ne $inheritedLockHashPresent) {
|
|
throw "Inherited recovery-lock receipt path and lowercase SHA256 are required together"
|
|
}
|
|
$inheritedRecoveryLockMode = $inheritedLockPathPresent -and $inheritedLockHashPresent
|
|
if ($offlineBootstrapMode) {
|
|
if (-not $offlineLegacyPinsPresent -or -not $inheritedRecoveryLockMode) {
|
|
throw "Offline bootstrap requires legacy source pins and an inherited recovery-lock receipt"
|
|
}
|
|
} elseif ($offlineLegacyPinsPresent -or $inheritedRecoveryLockMode) {
|
|
throw "Legacy source pins and inherited recovery lock are valid only in offline bootstrap mode"
|
|
}
|
|
|
|
function Get-RequiredPrivacySafeCount {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][string]$Name,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$property = $Payload.PSObject.Properties[$Name]
|
|
if ($null -eq $property -or $null -eq $property.Value) {
|
|
throw "$Role did not return privacy-safe count $Name"
|
|
}
|
|
$typeCode = [System.Type]::GetTypeCode($property.Value.GetType())
|
|
if (@(
|
|
[System.TypeCode]::Byte,
|
|
[System.TypeCode]::SByte,
|
|
[System.TypeCode]::Int16,
|
|
[System.TypeCode]::UInt16,
|
|
[System.TypeCode]::Int32,
|
|
[System.TypeCode]::UInt32,
|
|
[System.TypeCode]::Int64,
|
|
[System.TypeCode]::UInt64
|
|
) -notcontains $typeCode) {
|
|
throw "$Role returned non-integer privacy-safe count $Name"
|
|
}
|
|
$value = [decimal]$property.Value
|
|
if ($value -lt 0 -or $value -gt [int]::MaxValue) {
|
|
throw "$Role returned out-of-range privacy-safe count $Name"
|
|
}
|
|
return [int]$value
|
|
}
|
|
|
|
function Get-PreservedDecodeCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$PreservedObjectCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$validCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "preserved_decode_valid_count" `
|
|
-Role $Role
|
|
$invalidCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "preserved_decode_invalid_count" `
|
|
-Role $Role
|
|
if (($validCount + $invalidCount) -ne $PreservedObjectCount) {
|
|
throw "$Role preserved decode counts do not cover the exact object inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ValidCount = $validCount
|
|
InvalidCount = $invalidCount
|
|
}
|
|
}
|
|
|
|
function Get-RequiredDecodeInvalidCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$RequiredObjectCount,
|
|
[Parameter(Mandatory = $true)][int]$RequiredReferenceCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$objectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "required_decode_invalid_object_count" `
|
|
-Role $Role
|
|
$referenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "required_decode_invalid_reference_count" `
|
|
-Role $Role
|
|
if (
|
|
$objectCount -gt $RequiredObjectCount -or
|
|
$referenceCount -gt $RequiredReferenceCount
|
|
) {
|
|
throw "$Role required decode-invalid counts exceed the bound DB inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ObjectCount = $objectCount
|
|
ReferenceCount = $referenceCount
|
|
}
|
|
}
|
|
|
|
function Get-CurrentDecodeInvalidCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$CurrentObjectCount,
|
|
[Parameter(Mandatory = $true)][int]$CurrentReferenceCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$objectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "current_decode_invalid_object_count" `
|
|
-Role $Role
|
|
$referenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "current_decode_invalid_reference_count" `
|
|
-Role $Role
|
|
if (
|
|
$objectCount -gt $CurrentObjectCount -or
|
|
$referenceCount -gt $CurrentReferenceCount
|
|
) {
|
|
throw "$Role current decode-invalid counts exceed the bound DB inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ObjectCount = $objectCount
|
|
ReferenceCount = $referenceCount
|
|
}
|
|
}
|
|
|
|
function Test-StartPathIsSameOrChild {
|
|
param(
|
|
[string]$Candidate,
|
|
[string]$Parent
|
|
)
|
|
|
|
$candidateFull = [System.IO.Path]::GetFullPath($Candidate).TrimEnd('\', '/')
|
|
$parentFull = [System.IO.Path]::GetFullPath($Parent).TrimEnd('\', '/')
|
|
if ([string]::Equals(
|
|
$candidateFull,
|
|
$parentFull,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
return $true
|
|
}
|
|
return $candidateFull.StartsWith(
|
|
$parentFull + [System.IO.Path]::DirectorySeparatorChar,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)
|
|
}
|
|
|
|
function Assert-StartPathHasNoReparsePoint {
|
|
param([string]$Path)
|
|
|
|
$cursor = [System.IO.Path]::GetFullPath($Path)
|
|
while (-not [string]::IsNullOrWhiteSpace($cursor)) {
|
|
if (Test-Path -LiteralPath $cursor) {
|
|
$item = Get-Item -LiteralPath $cursor -Force
|
|
if (($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) {
|
|
throw "Inherited recovery-lock receipt cannot traverse a reparse point"
|
|
}
|
|
}
|
|
$parent = [System.IO.Directory]::GetParent($cursor)
|
|
if ($null -eq $parent) {
|
|
break
|
|
}
|
|
$cursor = $parent.FullName
|
|
}
|
|
}
|
|
|
|
function Assert-InheritedRecoveryLockReceipt {
|
|
param(
|
|
[string]$ReceiptPath,
|
|
[string]$ExpectedReceiptSha256,
|
|
[string]$LockPath,
|
|
[string]$SourceRoot,
|
|
[string]$UploadRoot,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree
|
|
)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($ReceiptPath)) {
|
|
throw "Inherited recovery-lock receipt must be absolute"
|
|
}
|
|
$resolvedReceipt = (Resolve-Path -LiteralPath $ReceiptPath).Path
|
|
$resolvedLock = [System.IO.Path]::GetFullPath($LockPath)
|
|
if (-not [string]::Equals(
|
|
$resolvedReceipt,
|
|
$resolvedLock,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Inherited recovery-lock receipt must be the active recovery lock"
|
|
}
|
|
foreach ($boundary in @($SourceRoot, $UploadRoot)) {
|
|
if (
|
|
(Test-StartPathIsSameOrChild -Candidate $resolvedReceipt -Parent $boundary) -or
|
|
(Test-StartPathIsSameOrChild -Candidate $boundary -Parent $resolvedReceipt)
|
|
) {
|
|
throw "Inherited recovery-lock receipt must be private and disjoint"
|
|
}
|
|
}
|
|
Assert-StartPathHasNoReparsePoint -Path $resolvedReceipt
|
|
$item = Get-Item -LiteralPath $resolvedReceipt -Force
|
|
if ($item.PSIsContainer) {
|
|
throw "Inherited recovery-lock receipt must be a regular file"
|
|
}
|
|
$actualSha256 = (
|
|
Get-FileHash -LiteralPath $resolvedReceipt -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($actualSha256 -cne $ExpectedReceiptSha256) {
|
|
throw "Inherited recovery-lock receipt SHA256 drift"
|
|
}
|
|
$payload = Get-Content -LiteralPath $resolvedReceipt -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
$actualKeys = @($payload.PSObject.Properties.Name | Sort-Object)
|
|
$expectedKeys = @(
|
|
"nonce_sha256",
|
|
"owner_pid",
|
|
"owner_started_at_utc",
|
|
"schema_version",
|
|
"source_commit",
|
|
"source_tree",
|
|
"status"
|
|
) | Sort-Object
|
|
if ((@($actualKeys) -join "`n") -cne (@($expectedKeys) -join "`n")) {
|
|
throw "Inherited recovery-lock receipt schema drift"
|
|
}
|
|
$currentStartedAtUtc = (
|
|
Get-Process -Id $PID -ErrorAction Stop
|
|
).StartTime.ToUniversalTime().ToString("o")
|
|
if (
|
|
$payload.schema_version -ne "vignette.public-runtime-inherited-lock.v1" -or
|
|
$payload.status -ne "held" -or
|
|
[int]$payload.owner_pid -ne $PID -or
|
|
[string]$payload.owner_started_at_utc -cne $currentStartedAtUtc -or
|
|
[string]$payload.source_commit -cne $SourceCommit -or
|
|
[string]$payload.source_tree -cne $SourceTree -or
|
|
[string]$payload.nonce_sha256 -notmatch "^[0-9a-f]{64}$"
|
|
) {
|
|
throw "Inherited recovery-lock receipt identity drift"
|
|
}
|
|
|
|
$exclusiveProbe = $null
|
|
try {
|
|
$exclusiveProbe = [System.IO.File]::Open(
|
|
$resolvedReceipt,
|
|
[System.IO.FileMode]::Open,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
throw "Inherited recovery lock is not held"
|
|
} catch [System.IO.IOException] {
|
|
return
|
|
} finally {
|
|
if ($null -ne $exclusiveProbe) {
|
|
$exclusiveProbe.Dispose()
|
|
}
|
|
}
|
|
}
|
|
|
|
function Enter-RecoveryLock {
|
|
param(
|
|
[string]$LockPath,
|
|
[int]$WaitSeconds
|
|
)
|
|
|
|
if ([string]::IsNullOrWhiteSpace($LockPath)) {
|
|
throw "Public runtime recovery lock path is empty"
|
|
}
|
|
$resolvedLockPath = [System.IO.Path]::GetFullPath($LockPath)
|
|
$lockDirectory = [System.IO.Path]::GetDirectoryName($resolvedLockPath)
|
|
if ([string]::IsNullOrWhiteSpace($lockDirectory)) {
|
|
throw "Public runtime recovery lock path has no parent directory"
|
|
}
|
|
[System.IO.Directory]::CreateDirectory($lockDirectory) | Out-Null
|
|
$deadline = (Get-Date).AddSeconds($WaitSeconds)
|
|
do {
|
|
try {
|
|
return [System.IO.File]::Open(
|
|
$resolvedLockPath,
|
|
[System.IO.FileMode]::OpenOrCreate,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
} catch [System.IO.IOException] {
|
|
if ((Get-Date) -ge $deadline) {
|
|
throw "Another public runtime recovery is already in progress"
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
} while ($true)
|
|
}
|
|
|
|
# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다.
|
|
# lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다.
|
|
$recoveryLock = $null
|
|
if ($inheritedRecoveryLockMode) {
|
|
Assert-InheritedRecoveryLockReceipt `
|
|
-ReceiptPath $InheritedRecoveryLockReceiptPath `
|
|
-ExpectedReceiptSha256 $ExpectedInheritedRecoveryLockReceiptSha256 `
|
|
-LockPath $RecoveryLockPath `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-UploadRoot $UserUploadDir `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree
|
|
} else {
|
|
$recoveryLock = Enter-RecoveryLock `
|
|
-LockPath $RecoveryLockPath `
|
|
-WaitSeconds $RecoveryLockWaitSeconds
|
|
}
|
|
|
|
try {
|
|
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
|
|
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
|
|
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
|
|
if (-not $env:ENGINE_READY_TTL_SECONDS) {
|
|
$env:ENGINE_READY_TTL_SECONDS = "1800"
|
|
}
|
|
|
|
$ApiDir = Join-Path $Workspace "apps\api"
|
|
$WebDir = Join-Path $Workspace "apps\web"
|
|
$OutLog = Join-Path $ApiDir "api.public.out.log"
|
|
$ErrLog = Join-Path $ApiDir "api.public.err.log"
|
|
$EngineOutLog = Join-Path $ApiDir "engine.public.out.log"
|
|
$EngineErrLog = Join-Path $ApiDir "engine.public.err.log"
|
|
$WebOutLog = Join-Path $Workspace "web.public.out.log"
|
|
$WebErrLog = Join-Path $Workspace "web.public.err.log"
|
|
$WhisperStartScript = Join-Path $Workspace "scripts\start-local-whisper-stt.ps1"
|
|
$MeloTtsStartScript = Join-Path $Workspace "scripts\start-melotts.ps1"
|
|
$VoiceSidecarProbe = Join-Path $Workspace "scripts\probe-public-voice-sidecars.py"
|
|
$WhisperModel = "small"
|
|
$WhisperLanguage = "ko"
|
|
$WhisperDevice = "cpu"
|
|
$MeloTtsModel = "melotts-korean"
|
|
$MeloTtsLanguage = "KR"
|
|
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
|
|
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
|
|
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
|
|
$RequiredPublicApiPaths = @(
|
|
"/health",
|
|
"/voice/health",
|
|
"/voice/speech",
|
|
"/admin/voice-runtime"
|
|
)
|
|
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
|
|
$PublicWebHostnames = @("vnet.18ka.net")
|
|
|
|
function Get-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 5
|
|
)
|
|
|
|
try {
|
|
Invoke-RestMethod -Uri $Uri -TimeoutSec $TimeoutSec
|
|
} catch {
|
|
$null
|
|
}
|
|
}
|
|
|
|
function Wait-JsonHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[scriptblock]$IsHealthy,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $Uri -TimeoutSec 5
|
|
if ($null -ne $health -and (& $IsHealthy $health)) {
|
|
return $health
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for healthy response from $Uri"
|
|
}
|
|
|
|
function Get-Utf8Sha256 {
|
|
param([Parameter(Mandatory = $true)][string]$Value)
|
|
|
|
$algorithm = [System.Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
$bytes = [System.Text.UTF8Encoding]::new($false).GetBytes($Value)
|
|
$digest = $algorithm.ComputeHash($bytes)
|
|
return ([System.BitConverter]::ToString($digest)).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$algorithm.Dispose()
|
|
}
|
|
}
|
|
|
|
function Assert-PublicUploadWriteFreezeReady {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$HealthUri,
|
|
[Parameter(Mandatory = $true)][string]$ExpectedTokenSha256,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5
|
|
$freeze = $null
|
|
if ($null -ne $health) {
|
|
$freeze = $health.upload_write_freeze
|
|
}
|
|
if (
|
|
$null -ne $freeze -and
|
|
$freeze.capable -eq $true -and
|
|
$freeze.active -eq $true -and
|
|
$freeze.valid -eq $true -and
|
|
[int]$freeze.in_flight -eq 0 -and
|
|
[string]$freeze.token_sha256 -ceq $ExpectedTokenSha256
|
|
) {
|
|
return $health
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Fresh public promotion requires a drained upload-write freeze"
|
|
}
|
|
|
|
function Exit-PublicUploadWriteFreeze {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$FreezePath,
|
|
[Parameter(Mandatory = $true)][string]$ExpectedTokenSha256,
|
|
[Parameter(Mandatory = $true)][string]$HealthUri,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $FreezePath -PathType Leaf)) {
|
|
throw "Upload-write freeze sentinel is missing before release"
|
|
}
|
|
$payload = Get-Content -LiteralPath $FreezePath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
if (
|
|
$null -eq $payload -or
|
|
[string]$payload.schema_version -cne "vignette.public-upload-write-freeze.v1" -or
|
|
[string]::IsNullOrWhiteSpace([string]$payload.token) -or
|
|
(Get-Utf8Sha256 -Value ([string]$payload.token)) -cne $ExpectedTokenSha256
|
|
) {
|
|
throw "Upload-write freeze sentinel ownership proof failed"
|
|
}
|
|
[System.IO.File]::Delete($FreezePath)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5
|
|
$freeze = $null
|
|
if ($null -ne $health) {
|
|
$freeze = $health.upload_write_freeze
|
|
}
|
|
if (
|
|
$null -ne $freeze -and
|
|
$freeze.capable -eq $true -and
|
|
$freeze.active -eq $false -and
|
|
$freeze.valid -eq $true -and
|
|
[int]$freeze.in_flight -eq 0
|
|
) {
|
|
return
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Fresh public rollback did not restore upload-write availability"
|
|
}
|
|
|
|
function Assert-PublicUploadWritesAvailable {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$HealthUri,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $HealthUri -TimeoutSec 5
|
|
$freeze = $null
|
|
if ($null -ne $health) {
|
|
$freeze = $health.upload_write_freeze
|
|
}
|
|
if (
|
|
$null -ne $freeze -and
|
|
$freeze.capable -eq $true -and
|
|
$freeze.active -eq $false -and
|
|
$freeze.valid -eq $true -and
|
|
[int]$freeze.in_flight -eq 0
|
|
) {
|
|
return
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Fresh public rollback did not restore upload-write availability"
|
|
}
|
|
|
|
function Test-PublicUploadManifestHealth {
|
|
param(
|
|
[object]$Health,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ExpectedManifestSha256
|
|
)
|
|
|
|
if ($null -eq $Health -or $null -eq $Health.upload_manifest) {
|
|
return $false
|
|
}
|
|
return (
|
|
$Health.upload_manifest.required -eq $true -and
|
|
$Health.upload_manifest.validated -eq $true -and
|
|
[string]$Health.upload_manifest.manifest_sha256 -ceq $ExpectedManifestSha256
|
|
)
|
|
}
|
|
|
|
function Test-PortListener {
|
|
param([int]$Port)
|
|
|
|
$listener = Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue `
|
|
| Select-Object -First 1
|
|
return $null -ne $listener
|
|
}
|
|
|
|
function Get-ListenerProcessIds {
|
|
param([int]$Port)
|
|
|
|
return @(
|
|
Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue |
|
|
Where-Object { $_.LocalAddress -eq "127.0.0.1" } |
|
|
Select-Object -ExpandProperty OwningProcess -Unique
|
|
)
|
|
}
|
|
|
|
function Get-ExactLoopbackListenerProcess {
|
|
param(
|
|
[int]$Port,
|
|
[string]$Role
|
|
)
|
|
|
|
$listenerIds = @(Get-ListenerProcessIds -Port $Port)
|
|
if ($listenerIds.Count -gt 1) {
|
|
throw "$Role has more than one 127.0.0.1:$Port listener PID"
|
|
}
|
|
if ($listenerIds.Count -eq 0) {
|
|
return $null
|
|
}
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $([int]$listenerIds[0])" `
|
|
-ErrorAction SilentlyContinue
|
|
if ($null -eq $process) {
|
|
throw "$Role listener PID disappeared before identity capture"
|
|
}
|
|
return $process
|
|
}
|
|
|
|
function Test-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component
|
|
)
|
|
|
|
$probeArgs = @(
|
|
"-X", "utf8", $VoiceSidecarProbe,
|
|
"--component", $Component,
|
|
"--stt-url", "ws://127.0.0.1:$WhisperPort/v1/listen",
|
|
"--stt-provider", "local_whisper",
|
|
"--stt-model", $WhisperModel,
|
|
"--stt-language", $WhisperLanguage,
|
|
"--stt-device", $WhisperDevice,
|
|
"--tts-url", "http://127.0.0.1:$MeloTtsPort",
|
|
"--tts-provider", "melotts",
|
|
"--tts-model", $MeloTtsModel,
|
|
"--tts-language", $MeloTtsLanguage,
|
|
"--timeout-seconds", "5"
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
& $Python @probeArgs 1>$null 2>$null
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return $probeExit -eq 0
|
|
}
|
|
|
|
function Wait-VoiceSidecarReady {
|
|
param(
|
|
[ValidateSet("stt", "tts")]
|
|
[string]$Component,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-VoiceSidecarReady -Component $Component) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Test-VoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
$Health.stt_provider -eq "local_whisper" -and
|
|
$Health.stt_model -eq $WhisperModel -and
|
|
$Health.tts_provider -eq "melotts" -and
|
|
$Health.tts_model -eq $MeloTtsModel -and
|
|
$Health.limits.uvicorn_ws_max_queue -eq 4
|
|
)
|
|
}
|
|
|
|
function Test-PriorVoiceApiReady {
|
|
param([object]$Health)
|
|
|
|
return (
|
|
$null -ne $Health -and
|
|
$Health.status -eq "ok" -and
|
|
$Health.available -eq $true -and
|
|
$Health.stt_available -eq $true -and
|
|
$Health.tts_available -eq $true -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
|
|
)
|
|
}
|
|
|
|
function ConvertTo-SafeVoiceHealthContract {
|
|
param([object]$Health)
|
|
|
|
return [ordered]@{
|
|
status = [string]$Health.status
|
|
available = [bool]$Health.available
|
|
stt_available = [bool]$Health.stt_available
|
|
tts_available = [bool]$Health.tts_available
|
|
stt_provider = [string]$Health.stt_provider
|
|
stt_model = [string]$Health.stt_model
|
|
tts_provider = [string]$Health.tts_provider
|
|
tts_model = [string]$Health.tts_model
|
|
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
|
|
}
|
|
}
|
|
|
|
function Test-VoiceHealthContract {
|
|
param(
|
|
[object]$Health,
|
|
[System.Collections.IDictionary]$Expected
|
|
)
|
|
|
|
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
|
|
return $false
|
|
}
|
|
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
|
|
foreach ($name in $Expected.Keys) {
|
|
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-RequiredOpenApiPaths {
|
|
param(
|
|
[object]$Document,
|
|
[string[]]$RequiredPaths
|
|
)
|
|
|
|
if ($null -eq $Document -or $null -eq $Document.paths) {
|
|
return $false
|
|
}
|
|
$actualPaths = @($Document.paths.PSObject.Properties.Name)
|
|
foreach ($requiredPath in $RequiredPaths) {
|
|
if ($actualPaths -notcontains $requiredPath) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 45
|
|
)
|
|
|
|
$headers = $null
|
|
if ($env:ENGINE_GATEWAY_SHARED_SECRET) {
|
|
$headers = @{ "X-Vignette-Engine-Token" = $env:ENGINE_GATEWAY_SHARED_SECRET }
|
|
}
|
|
try {
|
|
if ($headers) {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec -Headers $headers
|
|
} else {
|
|
$response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/ready" -TimeoutSec $TimeoutSec
|
|
}
|
|
return [bool]($response.ok -eq $true)
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Wait-EngineReady {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 90
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if (Test-EngineReady -Port $Port) {
|
|
return $true
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
return $false
|
|
}
|
|
|
|
function Wait-HttpStatus {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec = 30
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Uri -TimeoutSec 5
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 500) {
|
|
return $response
|
|
}
|
|
} catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out waiting for HTTP response from $Uri"
|
|
}
|
|
|
|
function Stop-ProcessesBounded {
|
|
param(
|
|
[object[]]$Processes,
|
|
[int]$TimeoutSec,
|
|
[string]$Role
|
|
)
|
|
|
|
$processIds = @(
|
|
$Processes |
|
|
ForEach-Object { [int]$_.ProcessId } |
|
|
Sort-Object -Unique
|
|
)
|
|
foreach ($processId in $processIds) {
|
|
Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$remaining = @(
|
|
$processIds |
|
|
Where-Object { $null -ne (Get-Process -Id $_ -ErrorAction SilentlyContinue) }
|
|
)
|
|
if ($remaining.Count -eq 0) {
|
|
return $processIds
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
|
}
|
|
|
|
function Stop-ProcessTreeBounded {
|
|
param(
|
|
[System.Diagnostics.Process]$RootProcess,
|
|
[int]$TimeoutSec,
|
|
[string]$Role
|
|
)
|
|
|
|
if ($null -eq $RootProcess) {
|
|
throw "Cannot stop $Role without its owned process handle"
|
|
}
|
|
$RootProcessId = $RootProcess.Id
|
|
try {
|
|
$null = $RootProcess.Handle
|
|
$expectedStartTimeUtc = $RootProcess.StartTime.ToUniversalTime()
|
|
} catch {
|
|
if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) {
|
|
return @($RootProcessId)
|
|
}
|
|
throw "Cannot verify the owned $Role process handle for PID $RootProcessId"
|
|
}
|
|
|
|
$currentRoot = Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue
|
|
if ($null -eq $currentRoot) {
|
|
return @($RootProcessId)
|
|
}
|
|
if ($currentRoot.StartTime.ToUniversalTime() -ne $expectedStartTimeUtc) {
|
|
throw "Refusing to stop reused PID $RootProcessId for $Role"
|
|
}
|
|
if ($currentRoot.SessionId -ne [System.Diagnostics.Process]::GetCurrentProcess().SessionId) {
|
|
throw "Refusing to stop $Role outside the current process session: PID $RootProcessId"
|
|
}
|
|
|
|
$taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe"
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
& $taskkill @("/PID", "$RootProcessId", "/T", "/F") | Out-Null
|
|
$taskkillExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($taskkillExit -ne 0) {
|
|
throw "Failed to stop $Role process tree rooted at PID $RootProcessId (taskkill exit=$taskkillExit)"
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) {
|
|
return @($RootProcessId)
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out stopping $Role process tree rooted at PID $RootProcessId"
|
|
}
|
|
|
|
function Get-UvicornProcessesByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port
|
|
)
|
|
|
|
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
|
|
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -like "python*" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*uvicorn $AppImport*" -and
|
|
$_.CommandLine -like "*--port $Port*"
|
|
}
|
|
)
|
|
}
|
|
|
|
function Stop-UvicornByPort {
|
|
param(
|
|
[string]$AppImport,
|
|
[int]$Port,
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
|
|
return @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $processes `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "uvicorn $AppImport on port $Port"
|
|
)
|
|
}
|
|
|
|
function Get-CloudflaredProcessesForConfig {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[switch]$ExactPath
|
|
)
|
|
|
|
$configLeaf = Split-Path -Leaf $ConfigPath
|
|
return @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*--config*" -and
|
|
(
|
|
$_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -or
|
|
(-not $ExactPath -and $_.CommandLine -like "*$configLeaf*")
|
|
)
|
|
}
|
|
)
|
|
}
|
|
|
|
function Wait-ProcessIdentity {
|
|
param(
|
|
[int]$ProcessId,
|
|
[string]$Role,
|
|
[string]$ExpectedCwd = "",
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $ProcessId" `
|
|
-ErrorAction SilentlyContinue
|
|
if (
|
|
$null -ne $process -and
|
|
$process.ExecutablePath -and
|
|
$process.CommandLine
|
|
) {
|
|
$identityProbeArgs = @(
|
|
"-X", "utf8", "-c",
|
|
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
|
|
"$ProcessId"
|
|
)
|
|
$identityProbe = @(& $Python @identityProbeArgs)
|
|
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
|
|
throw "Could not prove $Role psutil identity for PID $ProcessId"
|
|
}
|
|
$actualCwd = $identityProbe[0].Trim()
|
|
$startedAtUtc = $identityProbe[1].Trim()
|
|
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
|
|
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
|
|
$environmentObject = $identityProbe[4] | ConvertFrom-Json
|
|
$processEnvironment = [ordered]@{}
|
|
foreach ($property in $environmentObject.PSObject.Properties) {
|
|
$processEnvironment[$property.Name] = [string]$property.Value
|
|
}
|
|
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "$Role psutil identity is incomplete for PID $ProcessId"
|
|
}
|
|
if ($ExpectedCwd -and -not [string]::Equals(
|
|
[System.IO.Path]::GetFullPath($actualCwd),
|
|
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "$Role working directory drift: expected=$ExpectedCwd actual=$actualCwd"
|
|
}
|
|
return [ordered]@{
|
|
role = $Role
|
|
pid = [int]$process.ProcessId
|
|
started_at_utc = $startedAtUtc
|
|
executable_path = $process.ExecutablePath
|
|
executable_name = Split-Path -Leaf $process.ExecutablePath
|
|
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
command_line = $process.CommandLine
|
|
command_line_sha256 = $commandLineSha256
|
|
argument_list = $argumentList
|
|
environment = $processEnvironment
|
|
cwd = $actualCwd
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
throw "Timed out reading $Role process identity for PID $ProcessId"
|
|
}
|
|
|
|
function Get-VerifiedProcessFromIdentity {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role,
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
if ($null -eq $Identity) {
|
|
return $null
|
|
}
|
|
$processId = [int]$Identity.pid
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $processId" `
|
|
-ErrorAction SilentlyContinue
|
|
if ($null -eq $process) {
|
|
return $null
|
|
}
|
|
$current = Wait-ProcessIdentity `
|
|
-ProcessId $processId `
|
|
-Role $Role `
|
|
-ExpectedCwd ([string]$Identity.cwd) `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @(
|
|
"pid",
|
|
"started_at_utc",
|
|
"executable_sha256",
|
|
"command_line_sha256",
|
|
"cwd"
|
|
)) {
|
|
if ($current[$field].ToString() -cne $Identity[$field].ToString()) {
|
|
throw "Refusing to stop drifted or reused $Role process: $field"
|
|
}
|
|
}
|
|
return $process
|
|
}
|
|
|
|
function ConvertTo-SafeProcessIdentity {
|
|
param([System.Collections.IDictionary]$Identity)
|
|
|
|
# Raw command line이나 executable full path는 config/token을 우발적으로
|
|
# 영구 보존할 수 있다. topology 결속에 필요한 비밀 비포함 투영만 기록한다.
|
|
return [ordered]@{
|
|
pid = [int]$Identity.pid
|
|
started_at_utc = $Identity.started_at_utc
|
|
executable_name = $Identity.executable_name
|
|
executable_sha256 = $Identity.executable_sha256
|
|
command_line_sha256 = $Identity.command_line_sha256
|
|
cwd = $Identity.cwd
|
|
}
|
|
}
|
|
|
|
function Save-ManagedEnvironment {
|
|
param([string[]]$Names)
|
|
|
|
$snapshot = [ordered]@{}
|
|
foreach ($name in $Names) {
|
|
$value = [System.Environment]::GetEnvironmentVariable(
|
|
$name,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
$snapshot[$name] = [ordered]@{
|
|
present = $null -ne $value
|
|
value = $value
|
|
}
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Restore-ManagedEnvironment {
|
|
param([System.Collections.IDictionary]$Snapshot)
|
|
|
|
foreach ($name in $Snapshot.Keys) {
|
|
$entry = $Snapshot[$name]
|
|
if ($entry.present) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
[string]$entry.value,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
} else {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
function Save-CompleteProcessEnvironment {
|
|
$snapshot = [ordered]@{}
|
|
$environment = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in $environment.Keys) {
|
|
$snapshot[[string]$name] = [string]$environment[$name]
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Set-CompleteProcessEnvironment {
|
|
param([System.Collections.IDictionary]$Environment)
|
|
|
|
# psutil 환경 캡색에서 SystemRoot 같은 Windows 필수 변수가 유실되면 Go 계열 CLI(agy)가
|
|
# 시스템 인증서 풀/홈 해석에 실패해 조용히 빈 결과를 낸다(2026-08-18 실측). 이식본에
|
|
# 빠진 필수 키는 Machine 스코프 표준값으로 되살린다.
|
|
$windowsEssentials = @('SystemRoot', 'windir', 'SystemDrive', 'ComSpec')
|
|
foreach ($name in $windowsEssentials) {
|
|
$missing = -not $Environment.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$Environment[$name])
|
|
if ($missing) {
|
|
$machineValue = [System.Environment]::GetEnvironmentVariable($name, 'Machine')
|
|
if ($machineValue) {
|
|
$Environment[$name] = $machineValue
|
|
}
|
|
}
|
|
}
|
|
|
|
$current = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in @($current.Keys)) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
foreach ($name in $Environment.Keys) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
[string]$Environment[$name],
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
|
|
function ConvertTo-WindowsCommandLineArgument {
|
|
param([AllowEmptyString()][string]$Argument)
|
|
|
|
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
|
return $Argument
|
|
}
|
|
|
|
$builder = New-Object System.Text.StringBuilder
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
foreach ($character in $Argument.ToCharArray()) {
|
|
if ($character -eq '\') {
|
|
$backslashes++
|
|
continue
|
|
}
|
|
if ($character -eq '"') {
|
|
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
continue
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * $backslashes))
|
|
$backslashes = 0
|
|
}
|
|
$null = $builder.Append($character)
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * ($backslashes * 2)))
|
|
}
|
|
$null = $builder.Append('"')
|
|
return $builder.ToString()
|
|
}
|
|
|
|
function Join-WindowsArgumentList {
|
|
param([object[]]$ArgumentList)
|
|
|
|
return (@(
|
|
foreach ($argument in $ArgumentList) {
|
|
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
|
}
|
|
) -join ' ')
|
|
}
|
|
|
|
function Start-PinnedPriorProcess {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role,
|
|
[string]$StdoutLog,
|
|
[string]$StderrLog
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
|
|
throw "Prior $Role executable is unavailable"
|
|
}
|
|
$actualExecutableSha256 = (
|
|
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
|
|
throw "Prior $Role executable SHA256 drift"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
|
throw "Prior $Role working directory is unavailable"
|
|
}
|
|
if (@($Identity.argument_list).Count -eq 0) {
|
|
throw "Prior $Role argument list is unavailable"
|
|
}
|
|
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
|
|
throw "Prior $Role environment is unavailable"
|
|
}
|
|
|
|
$callerEnvironment = Save-CompleteProcessEnvironment
|
|
try {
|
|
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
|
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
|
return Start-Process -WindowStyle Hidden `
|
|
-FilePath $Identity.executable_path `
|
|
-ArgumentList $argumentString `
|
|
-WorkingDirectory $Identity.cwd `
|
|
-RedirectStandardOutput $StdoutLog `
|
|
-RedirectStandardError $StderrLog `
|
|
-PassThru
|
|
} finally {
|
|
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
|
}
|
|
}
|
|
|
|
function Restore-PriorPublicRuntime {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApi,
|
|
[System.Collections.IDictionary]$PriorCloudflared,
|
|
[System.Collections.IDictionary]$ReplacementApi,
|
|
[System.Collections.IDictionary]$ReplacementCloudflared,
|
|
[System.Collections.IDictionary]$PriorLocalVoiceContract,
|
|
[System.Collections.IDictionary]$PriorPublicVoiceContract,
|
|
[System.Collections.IDictionary]$EnvironmentSnapshot,
|
|
[string]$ConfigPath,
|
|
[int]$ApiPortValue,
|
|
[string]$HealthUrl,
|
|
[string]$VoiceHealthUrl,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
# Rollback도 ingress를 먼저 닫은 뒤 listener를 바꾼다. 공개 ingress가 살아
|
|
# 있는 상태에서 API/root를 되돌리면 rollback 중 새 write가 prior/new root에
|
|
# 갈라질 수 있으므로 replacement tunnel의 exact identity와 absence를 먼저 증명한다.
|
|
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
|
|
$verifiedReplacementCloudflared = Get-VerifiedProcessFromIdentity `
|
|
-Identity $ReplacementCloudflared `
|
|
-Role "failed fresh cloudflared" `
|
|
-TimeoutSec $TimeoutSec
|
|
if ($null -ne $verifiedReplacementCloudflared) {
|
|
$null = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes @($verifiedReplacementCloudflared) `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "failed fresh cloudflared"
|
|
)
|
|
}
|
|
$unexpectedCloudflared = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedConfigPath `
|
|
-ExactPath
|
|
)
|
|
if ($unexpectedCloudflared.Count -ne 0) {
|
|
throw "Refusing rollback because the tunnel config is owned by an unpinned process"
|
|
}
|
|
|
|
$replacementApiListener = Get-ExactLoopbackListenerProcess `
|
|
-Port $ApiPortValue `
|
|
-Role "failed fresh API"
|
|
if ($null -ne $replacementApiListener) {
|
|
if (
|
|
$null -eq $ReplacementApi -or
|
|
[int]$replacementApiListener.ProcessId -ne [int]$ReplacementApi.pid
|
|
) {
|
|
throw "Refusing rollback because 127.0.0.1:$ApiPortValue is owned by an unpinned listener"
|
|
}
|
|
$verifiedReplacementApi = Get-VerifiedProcessFromIdentity `
|
|
-Identity $ReplacementApi `
|
|
-Role "failed fresh API" `
|
|
-TimeoutSec $TimeoutSec
|
|
$null = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes @($verifiedReplacementApi) `
|
|
-TimeoutSec $TimeoutSec `
|
|
-Role "failed fresh API listener"
|
|
)
|
|
}
|
|
if (@(Get-ListenerProcessIds -Port $ApiPortValue).Count -ne 0) {
|
|
throw "Failed fresh API listener remains on 127.0.0.1:$ApiPortValue"
|
|
}
|
|
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
|
|
$priorApiProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorApi `
|
|
-Role "api" `
|
|
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
|
|
$priorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiProcess.Id `
|
|
-Role "restored prior api" `
|
|
-ExpectedCwd $PriorApi.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
|
|
throw "Restored prior API identity drift: $field"
|
|
}
|
|
}
|
|
$restoredListenerIds = @(Get-ListenerProcessIds -Port $ApiPortValue)
|
|
if (
|
|
$restoredListenerIds.Count -ne 1 -or
|
|
[int]$restoredListenerIds[0] -ne [int]$priorApiProcess.Id
|
|
) {
|
|
throw "Restored prior API does not own the exact 127.0.0.1:$ApiPortValue listener"
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Restored prior runtime does not have the exact local voice sidecars"
|
|
}
|
|
|
|
$priorCloudflaredProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorCloudflared `
|
|
-Role "cloudflared" `
|
|
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
|
|
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
|
|
$priorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcess.Id `
|
|
-Role "restored prior cloudflared" `
|
|
-ExpectedCwd $PriorCloudflared.cwd `
|
|
-TimeoutSec $TimeoutSec
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
|
|
throw "Restored prior cloudflared identity drift: $field"
|
|
}
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $HealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$null = Wait-JsonHealth `
|
|
-Uri $VoiceHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
|
|
} `
|
|
-TimeoutSec 30
|
|
|
|
return [ordered]@{
|
|
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
|
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
|
|
local_health = $true
|
|
local_voice_health = $true
|
|
public_health = $true
|
|
public_voice_health = $true
|
|
}
|
|
}
|
|
|
|
function Stop-NodeByPortHint {
|
|
param([int]$Port)
|
|
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "node.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine -like "*vite*preview*" -and
|
|
$_.CommandLine -like "*$Port*"
|
|
} |
|
|
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
|
}
|
|
|
|
function ConvertTo-CompactJson {
|
|
param([object]$Value)
|
|
ConvertTo-Json -InputObject $Value -Compress
|
|
}
|
|
|
|
function Invoke-StableGitText {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string[]]$Arguments
|
|
)
|
|
|
|
$value = & git.exe -C $SourceRoot @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git command failed (exit=$LASTEXITCODE): git $($Arguments -join ' ')"
|
|
}
|
|
return (@($value) -join [Environment]::NewLine).Trim()
|
|
}
|
|
|
|
function Initialize-RuntimeProvenanceOutput {
|
|
param([string]$OutputPath)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($OutputPath)) {
|
|
throw "Fresh public provenance output path must be absolute"
|
|
}
|
|
|
|
try {
|
|
$resolvedOutputPath = [System.IO.Path]::GetFullPath($OutputPath)
|
|
$outputDirectory = Split-Path -Parent $resolvedOutputPath
|
|
if (-not $outputDirectory) {
|
|
throw "Fresh public provenance output path has no parent directory"
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Container) {
|
|
throw "Fresh public provenance output path is a directory: $resolvedOutputPath"
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null
|
|
}
|
|
if (-not (Test-Path -LiteralPath $outputDirectory -PathType Container)) {
|
|
throw "Fresh public provenance output directory is unavailable: $outputDirectory"
|
|
}
|
|
|
|
# 기존 receipt가 잠겨 있거나 read-only라면 프로세스 교체 전에 실패해야 한다.
|
|
# sibling probe 두 개를 atomic replace해 디렉터리의 create/flush/replace/delete
|
|
# 권한도 미리 검증한다. 실제 receipt 내용은 이 단계에서 건드리지 않는다.
|
|
if (Test-Path -LiteralPath $resolvedOutputPath -PathType Leaf) {
|
|
$attributes = [System.IO.File]::GetAttributes($resolvedOutputPath)
|
|
if (($attributes -band [System.IO.FileAttributes]::ReadOnly) -ne 0) {
|
|
throw "Fresh public provenance output is read-only: $resolvedOutputPath"
|
|
}
|
|
$existingStream = [System.IO.File]::Open(
|
|
$resolvedOutputPath,
|
|
[System.IO.FileMode]::Open,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::Read
|
|
)
|
|
$existingStream.Dispose()
|
|
}
|
|
|
|
$probeId = [Guid]::NewGuid().ToString("N")
|
|
$probeSource = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.source.tmp"
|
|
$probeTarget = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.target.tmp"
|
|
$probeBackup = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($resolvedOutputPath)).$probeId.backup.tmp"
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
[System.IO.File]::WriteAllText($probeSource, "probe-source", $encoding)
|
|
[System.IO.File]::WriteAllText($probeTarget, "probe-target", $encoding)
|
|
[System.IO.File]::Replace($probeSource, $probeTarget, $probeBackup, $true)
|
|
[System.IO.File]::Delete($probeTarget)
|
|
[System.IO.File]::Delete($probeBackup)
|
|
} finally {
|
|
foreach ($probePath in @($probeSource, $probeTarget, $probeBackup)) {
|
|
if ($probePath -and [System.IO.File]::Exists($probePath)) {
|
|
[System.IO.File]::Delete($probePath)
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
throw "Fresh public provenance output preflight failed before runtime mutation: $($_.Exception.Message)"
|
|
}
|
|
|
|
return $resolvedOutputPath
|
|
}
|
|
|
|
function Write-Utf8TextAtomically {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$Value
|
|
)
|
|
|
|
$outputDirectory = Split-Path -Parent $OutputPath
|
|
$temporaryPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).tmp"
|
|
$backupPath = Join-Path $outputDirectory ".$([System.IO.Path]::GetFileName($OutputPath)).$([Guid]::NewGuid().ToString('N')).backup.tmp"
|
|
$published = $false
|
|
try {
|
|
$encoding = [System.Text.UTF8Encoding]::new($false)
|
|
$bytes = $encoding.GetBytes($Value)
|
|
$stream = [System.IO.FileStream]::new(
|
|
$temporaryPath,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
try {
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
} finally {
|
|
$stream.Dispose()
|
|
}
|
|
|
|
if ([System.IO.File]::Exists($OutputPath)) {
|
|
[System.IO.File]::Replace($temporaryPath, $OutputPath, $backupPath, $true)
|
|
} elseif (Test-Path -LiteralPath $OutputPath) {
|
|
throw "Fresh public provenance output became a non-file before commit: $OutputPath"
|
|
} else {
|
|
[System.IO.File]::Move($temporaryPath, $OutputPath)
|
|
}
|
|
$published = $true
|
|
} finally {
|
|
if ([System.IO.File]::Exists($temporaryPath)) {
|
|
[System.IO.File]::Delete($temporaryPath)
|
|
}
|
|
if ([System.IO.File]::Exists($backupPath)) {
|
|
try {
|
|
[System.IO.File]::Delete($backupPath)
|
|
} catch {
|
|
if ($published) {
|
|
Write-Warning "Atomic provenance receipt was published, but its temporary backup could not be removed: $backupPath"
|
|
} else {
|
|
throw
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Write-FailedFreshPromotionEvidence {
|
|
param(
|
|
[string]$OutputPath,
|
|
[string]$FailureStage,
|
|
[bool]$RollbackAttempted,
|
|
[bool]$RollbackSucceeded,
|
|
[bool]$CurrentRuntimeRetained,
|
|
[bool]$TasksRemainDisabled,
|
|
[bool]$TasksRestored,
|
|
[bool]$TaskStateVerified,
|
|
[object]$RollbackResult,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree,
|
|
[string]$UserUploadRoot
|
|
)
|
|
|
|
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
|
|
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
|
|
$failedPath = "$OutputPath.failed.log"
|
|
$payload = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-failure.v1"
|
|
status = if ($CurrentRuntimeRetained) {
|
|
"failed_current_runtime_retained"
|
|
} elseif ($RollbackSucceeded) {
|
|
"failed_rolled_back"
|
|
} else {
|
|
"failed_rollback"
|
|
}
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
failure_stage = $FailureStage
|
|
source = [ordered]@{
|
|
git_commit = $SourceCommit.ToLowerInvariant()
|
|
git_tree = $SourceTree.ToLowerInvariant()
|
|
}
|
|
storage = [ordered]@{
|
|
user_upload_root = $UserUploadRoot
|
|
}
|
|
rollback = [ordered]@{
|
|
attempted = $RollbackAttempted
|
|
succeeded = $RollbackSucceeded
|
|
result = $RollbackResult
|
|
}
|
|
current_runtime_retained = $CurrentRuntimeRetained
|
|
tasks_remain_disabled = $TasksRemainDisabled
|
|
tasks_restored_to_snapshot = $TasksRestored
|
|
task_state_verified = $TaskStateVerified
|
|
}
|
|
$json = ConvertTo-Json -InputObject $payload -Depth 8
|
|
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
|
|
return $failedPath
|
|
}
|
|
|
|
function Assert-OfflineBootstrapContract {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree,
|
|
[string]$PythonPath,
|
|
[string]$PythonSha256
|
|
)
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
throw "Offline bootstrap quiescence receipt is mutually exclusive with fresh promotion"
|
|
}
|
|
if (-not $ForceApiRestart) {
|
|
throw "Offline bootstrap requires -ForceApiRestart"
|
|
}
|
|
if (-not $SkipEngineRestart -or -not $SkipWebRestart -or -not $SkipCloudflaredRestart) {
|
|
throw "Offline bootstrap is API-only and requires all engine, web, and cloudflared restart skips"
|
|
}
|
|
if ($RouteCloudflareDns) {
|
|
throw "Offline bootstrap forbids DNS route mutation"
|
|
}
|
|
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
|
if ($sourcePin -notmatch "^[0-9a-f]{40}$") {
|
|
throw "Offline bootstrap requires lowercase source commit and tree pins"
|
|
}
|
|
}
|
|
if ($PythonSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Offline bootstrap requires a lowercase Python SHA256 pin"
|
|
}
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path
|
|
$expectedStartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
if (-not [string]::Equals(
|
|
(Resolve-Path -LiteralPath $PSCommandPath).Path,
|
|
(Resolve-Path -LiteralPath $expectedStartScript).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Offline bootstrap launcher must execute from the pinned stable source root"
|
|
}
|
|
$gitRoot = Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("rev-parse", "--show-toplevel")
|
|
if (-not [string]::Equals(
|
|
(Resolve-Path -LiteralPath $gitRoot).Path,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Offline bootstrap source root does not match its Git toplevel"
|
|
}
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Offline bootstrap requires detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Offline bootstrap could not prove detached HEAD"
|
|
}
|
|
$actualCommit = Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -cne $SourceCommit -or $actualTree -cne $SourceTree) {
|
|
throw "Offline bootstrap source commit or tree drift"
|
|
}
|
|
$dirty = Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Offline bootstrap requires a clean stable source"
|
|
}
|
|
foreach ($relativePath in @(
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/public-runtime-task-maintenance.ps1",
|
|
"scripts/public-runtime-task-definition-cutover.ps1",
|
|
"scripts/validate-public-runtime-offline-quiescence.py",
|
|
"scripts/initialize-public-runtime-upload-root.py",
|
|
"scripts/public_runtime_database_identity.py",
|
|
"apps/api/app/upload_storage.py",
|
|
"apps/api/app/upload_runtime.py"
|
|
)) {
|
|
Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("ls-files", "--error-unmatch", "--", $relativePath) `
|
|
| Out-Null
|
|
}
|
|
$actualPythonSha256 = (
|
|
Get-FileHash -LiteralPath $PythonPath -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($actualPythonSha256 -cne $PythonSha256) {
|
|
throw "Offline bootstrap Python SHA256 drift"
|
|
}
|
|
}
|
|
|
|
function Test-OfflineBootstrapQuiescenceReceipt {
|
|
param(
|
|
[string]$PythonPath,
|
|
[string]$ProbePath,
|
|
[string]$ReceiptPath,
|
|
[string]$ExpectedReceiptSha256,
|
|
[string]$ManifestPath,
|
|
[string]$ExpectedManifestSha256,
|
|
[string]$StableSourceRoot,
|
|
[string]$UploadRoot,
|
|
[string]$ExpectedLegacySourceCommit,
|
|
[string]$ExpectedLegacySourceTree
|
|
)
|
|
|
|
$probeArgs = @(
|
|
"-X", "utf8", "-B", $ProbePath,
|
|
"--receipt-path", $ReceiptPath,
|
|
"--expected-receipt-sha256", $ExpectedReceiptSha256,
|
|
"--manifest-path", $ManifestPath,
|
|
"--expected-manifest-sha256", $ExpectedManifestSha256,
|
|
"--stable-source-root", $StableSourceRoot,
|
|
"--upload-root", $UploadRoot,
|
|
"--expected-legacy-source-commit", $ExpectedLegacySourceCommit,
|
|
"--expected-legacy-source-tree", $ExpectedLegacySourceTree
|
|
)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
try {
|
|
$ErrorActionPreference = "Continue"
|
|
$output = @(& $PythonPath @probeArgs 2>$null)
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
$detail = (@($output) -join "").Trim()
|
|
$payload = $null
|
|
if (-not [string]::IsNullOrWhiteSpace($detail)) {
|
|
try {
|
|
$payload = $detail | ConvertFrom-Json
|
|
} catch {
|
|
$payload = $null
|
|
}
|
|
}
|
|
$safeDetail = "invalid_json"
|
|
if ($null -ne $payload) {
|
|
$safeDetail = [string]$payload.status
|
|
}
|
|
return [pscustomobject]@{
|
|
Ok = $probeExit -eq 0 -and $null -ne $payload -and $payload.status -eq "passed"
|
|
Detail = $safeDetail
|
|
Payload = $payload
|
|
}
|
|
}
|
|
|
|
function Assert-FreshPublicProvenanceContract {
|
|
param(
|
|
[string]$SourceRoot,
|
|
[string]$SourceCommit,
|
|
[string]$SourceTree,
|
|
[string]$PythonPath,
|
|
[string]$PythonSha256,
|
|
[string]$CloudflaredPath,
|
|
[string]$CloudflaredSha256,
|
|
[string]$ConfigPath,
|
|
[string]$ConfigSha256
|
|
)
|
|
|
|
if (-not $ForceApiRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -ForceApiRestart"
|
|
}
|
|
if ($SkipCloudflaredRestart) {
|
|
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
|
|
}
|
|
if (-not $SkipEngineRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
|
|
}
|
|
if (-not $SkipWebRestart) {
|
|
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
|
|
}
|
|
if ($RouteCloudflareDns) {
|
|
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
|
|
}
|
|
if (-not [string]::Equals(
|
|
$PublicHealthUrl,
|
|
$CanonicalPublicHealthUrl,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion requires the canonical HTTPS public health URL"
|
|
}
|
|
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
|
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
|
|
throw "Fresh public provenance requires exact source commit and tree pins"
|
|
}
|
|
}
|
|
foreach ($shaPin in @($PythonSha256, $CloudflaredSha256, $ConfigSha256)) {
|
|
if ($shaPin -notmatch "^[0-9a-fA-F]{64}$") {
|
|
throw "Fresh public provenance requires exact Python, cloudflared, and config SHA256 pins"
|
|
}
|
|
}
|
|
|
|
$resolvedSourceRoot = (Resolve-Path -LiteralPath $SourceRoot).Path
|
|
$expectedStartScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
|
$runningStartScript = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
|
if (-not [string]::Equals(
|
|
$runningStartScript,
|
|
(Resolve-Path -LiteralPath $expectedStartScript).Path,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion must execute the launcher from the pinned stable source root"
|
|
}
|
|
|
|
$gitRoot = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedGitRoot = (Resolve-Path -LiteralPath $gitRoot).Path
|
|
if (-not [string]::Equals(
|
|
$resolvedGitRoot,
|
|
$resolvedSourceRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Fresh public promotion source root does not match its Git toplevel"
|
|
}
|
|
|
|
$symbolicHead = & git.exe -C $resolvedSourceRoot symbolic-ref --quiet HEAD
|
|
$symbolicHeadExit = $LASTEXITCODE
|
|
if ($symbolicHeadExit -eq 0) {
|
|
throw "Fresh public promotion requires detached HEAD, not branch $symbolicHead"
|
|
}
|
|
if ($symbolicHeadExit -ne 1) {
|
|
throw "Could not prove detached HEAD (git exit=$symbolicHeadExit)"
|
|
}
|
|
|
|
$actualCommit = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -ne $SourceCommit.ToLowerInvariant()) {
|
|
throw "Fresh public source commit drift: expected=$SourceCommit actual=$actualCommit"
|
|
}
|
|
if ($actualTree -ne $SourceTree.ToLowerInvariant()) {
|
|
throw "Fresh public source tree drift: expected=$SourceTree actual=$actualTree"
|
|
}
|
|
$dirty = Invoke-StableGitText -SourceRoot $resolvedSourceRoot -Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Fresh public promotion requires a clean stable source"
|
|
}
|
|
foreach ($relativePath in @(
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/public-runtime-upload-root.ps1",
|
|
"scripts/public-runtime-task-maintenance.ps1",
|
|
"scripts/public-runtime-task-definition-cutover.ps1",
|
|
"scripts/register-boot-task.ps1",
|
|
"scripts/install-public-runtime-task.ps1",
|
|
"scripts/probe-public-runtime-upload-root.py",
|
|
"scripts/validate-public-runtime-upload-manifest.py",
|
|
"scripts/public_runtime_database_identity.py",
|
|
"apps/api/app/upload_storage.py",
|
|
"apps/api/app/upload_runtime.py"
|
|
)) {
|
|
Invoke-StableGitText `
|
|
-SourceRoot $resolvedSourceRoot `
|
|
-Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
|
}
|
|
|
|
foreach ($pin in @(
|
|
[pscustomobject]@{ Path = $PythonPath; Sha256 = $PythonSha256; Label = "Python" },
|
|
[pscustomobject]@{ Path = $CloudflaredPath; Sha256 = $CloudflaredSha256; Label = "cloudflared" },
|
|
[pscustomobject]@{ Path = $ConfigPath; Sha256 = $ConfigSha256; Label = "cloudflared config" }
|
|
)) {
|
|
if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) {
|
|
throw "Pinned $($pin.Label) file not found at $($pin.Path)"
|
|
}
|
|
$actualSha256 = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualSha256 -ne $pin.Sha256.ToLowerInvariant()) {
|
|
throw "Pinned $($pin.Label) SHA256 drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Set-CloudflaredIngress {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[string[]]$ApiHostnames,
|
|
[string[]]$WebHostnames,
|
|
[int]$ApiPortValue,
|
|
[int]$WebPortValue,
|
|
[switch]$RequireUnchanged
|
|
)
|
|
|
|
$lines = Get-Content -Encoding UTF8 -Path $ConfigPath
|
|
$ingressIndex = -1
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -match "^\s*ingress:\s*$") {
|
|
$ingressIndex = $i
|
|
break
|
|
}
|
|
}
|
|
if ($ingressIndex -lt 0) {
|
|
throw "Could not find ingress: in $ConfigPath"
|
|
}
|
|
|
|
$nextLines = @()
|
|
if ($ingressIndex -gt 0) {
|
|
$nextLines += $lines[0..($ingressIndex - 1)]
|
|
}
|
|
$nextLines += "ingress:"
|
|
foreach ($hostname in $WebHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$WebPortValue"
|
|
}
|
|
foreach ($hostname in $ApiHostnames) {
|
|
$nextLines += " - hostname: $hostname"
|
|
$nextLines += " service: http://127.0.0.1:$ApiPortValue"
|
|
}
|
|
$nextLines += " - service: http_status:404"
|
|
|
|
$matches = $lines.Count -eq $nextLines.Count
|
|
if ($matches) {
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -cne $nextLines[$i]) {
|
|
$matches = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if ($RequireUnchanged -and -not $matches) {
|
|
throw "Pinned cloudflared config ingress does not match the requested public topology"
|
|
}
|
|
if (-not $matches) {
|
|
Set-Content -Encoding UTF8 -Path $ConfigPath -Value $nextLines
|
|
}
|
|
}
|
|
|
|
$freshMutationStarted = $false
|
|
$freshPromotionCommitted = $false
|
|
$freshNoRollback = $false
|
|
$freshFailureStage = "preflight"
|
|
$freshPriorApiIdentity = $null
|
|
$freshPriorCloudflaredIdentity = $null
|
|
$freshPriorLocalVoiceContract = $null
|
|
$freshPriorPublicVoiceContract = $null
|
|
$freshEnvironmentSnapshot = $null
|
|
$resolvedRuntimeProvenancePath = $null
|
|
$resolvedTaskRecoveryReceiptPath = $null
|
|
$freshStoppedTunnelIds = @()
|
|
$freshTaskMaintenanceSnapshot = @()
|
|
$freshTaskMaintenanceEntered = $false
|
|
$freshTaskMaintenanceWasEntered = $false
|
|
$freshTasksRestored = $false
|
|
$freshOriginalTaskDefinitionSnapshot = $null
|
|
$freshDisabledOriginalTaskDefinitionSnapshot = $null
|
|
$freshNewDisabledTaskDefinitionSnapshot = $null
|
|
$freshOperationalTaskDefinitionSnapshot = $null
|
|
$freshManagedEnvironmentNames = @(
|
|
"ENVIRONMENT",
|
|
"ENGINE_URL",
|
|
"ENGINE_MODE",
|
|
"VIGNETTE_LIVE_CLIENT_PROVIDER",
|
|
"AUTH_DEV_LOGIN_ENABLED",
|
|
"AUTO_SEED_PERSONAS",
|
|
"ALLOW_SEED_PERSONA_FALLBACK",
|
|
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
|
|
"VIGNETTE_VOICE_STT_PROVIDER",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_URL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
|
|
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
|
|
"VIGNETTE_VOICE_TTS_PROVIDER",
|
|
"VIGNETTE_MELOTTS_TTS_URL",
|
|
"FRONTEND_BASE_URL",
|
|
"CORS_ORIGINS",
|
|
"FRONTEND_ORIGIN_MAP",
|
|
"USER_UPLOAD_DIR",
|
|
"USER_UPLOAD_MANIFEST_REQUIRED",
|
|
"USER_UPLOAD_MANIFEST_PATH",
|
|
"USER_UPLOAD_MANIFEST_SHA256",
|
|
"USER_UPLOAD_WRITE_FREEZE_PATH",
|
|
"PUBLIC_RUNTIME_DB_TARGET_SHA256"
|
|
)
|
|
|
|
trap {
|
|
$caught = $_
|
|
if (
|
|
$RequireFreshPublicProvenance -and
|
|
$freshTaskMaintenanceWasEntered
|
|
) {
|
|
$rollbackResult = $null
|
|
$rollbackAttempted = $false
|
|
$rollbackSucceeded = $false
|
|
$tasksRemainDisabled = $false
|
|
$tasksRestored = $false
|
|
$taskStateVerified = $false
|
|
$rollbackFailureType = "none"
|
|
if (-not $freshNoRollback) {
|
|
try {
|
|
if ($freshMutationStarted) {
|
|
$rollbackAttempted = $true
|
|
$rollbackResult = Restore-PriorPublicRuntime `
|
|
-PriorApi $freshPriorApiIdentity `
|
|
-PriorCloudflared $freshPriorCloudflaredIdentity `
|
|
-ReplacementApi $apiLaunchIdentity `
|
|
-ReplacementCloudflared $cloudflaredLaunchIdentity `
|
|
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
|
|
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
|
|
-EnvironmentSnapshot $freshEnvironmentSnapshot `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiPortValue $ApiPort `
|
|
-HealthUrl $PublicHealthUrl `
|
|
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
} else {
|
|
$rollbackResult = [ordered]@{ runtime_mutation = $false }
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedUserUploadWriteFreezePath -PathType Leaf) {
|
|
if ($null -eq $uploadManifestProof) {
|
|
throw "Cannot prove upload-write freeze ownership during recovery"
|
|
}
|
|
Exit-PublicUploadWriteFreeze `
|
|
-FreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec 30
|
|
} else {
|
|
Assert-PublicUploadWritesAvailable `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec 30
|
|
}
|
|
Exit-PublicRuntimeTaskMaintenance `
|
|
-Snapshot $freshTaskMaintenanceSnapshot
|
|
$freshTaskMaintenanceEntered = $false
|
|
$freshTasksRestored = $true
|
|
$rollbackSucceeded = $true
|
|
} catch {
|
|
$rollbackFailureType = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
if (-not $rollbackSucceeded -and $freshTaskMaintenanceEntered) {
|
|
try {
|
|
Suspend-PublicRuntimeTasks `
|
|
-Snapshot $freshTaskMaintenanceSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$tasksRemainDisabled = $true
|
|
} catch {
|
|
$rollbackFailureType = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
|
|
try {
|
|
$taskTruth = Get-PublicRuntimeTaskMaintenanceState `
|
|
-Snapshot $freshTaskMaintenanceSnapshot
|
|
$taskStateVerified = [bool]$taskTruth.verified
|
|
$tasksRemainDisabled = (
|
|
$taskStateVerified -and
|
|
[bool]$taskTruth.all_disabled_and_idle
|
|
)
|
|
$tasksRestored = (
|
|
$taskStateVerified -and
|
|
[bool]$taskTruth.restored_to_snapshot
|
|
)
|
|
$freshTasksRestored = $tasksRestored
|
|
} catch {
|
|
$taskStateVerified = $false
|
|
$tasksRemainDisabled = $false
|
|
$tasksRestored = $false
|
|
}
|
|
|
|
$failedEvidencePath = ""
|
|
if ($resolvedRuntimeProvenancePath) {
|
|
try {
|
|
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-FailureStage $freshFailureStage `
|
|
-RollbackAttempted $rollbackAttempted `
|
|
-RollbackSucceeded $rollbackSucceeded `
|
|
-CurrentRuntimeRetained $freshNoRollback `
|
|
-TasksRemainDisabled $tasksRemainDisabled `
|
|
-TasksRestored $tasksRestored `
|
|
-TaskStateVerified $taskStateVerified `
|
|
-RollbackResult $rollbackResult `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-UserUploadRoot $resolvedUserUploadDir
|
|
} catch {
|
|
$failedEvidencePath = "unavailable"
|
|
}
|
|
}
|
|
|
|
if ($freshNoRollback) {
|
|
throw "Fresh public promotion failed at $freshFailureStage after the no-rollback boundary; the new runtime/root was retained and full operational success was not published. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
if ($rollbackSucceeded) {
|
|
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior runtime/write availability was restored and scheduled tasks were re-enabled. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
|
}
|
|
throw $caught
|
|
}
|
|
|
|
if (!(Test-Path $Python)) {
|
|
throw "Python 3.11 not found at $Python"
|
|
}
|
|
if (!(Test-Path $ApiDir)) {
|
|
throw "API directory not found at $ApiDir"
|
|
}
|
|
if (!(Test-Path $WebDir)) {
|
|
throw "Web directory not found at $WebDir"
|
|
}
|
|
foreach ($voiceScript in @($WhisperStartScript, $MeloTtsStartScript, $VoiceSidecarProbe)) {
|
|
if (!(Test-Path -LiteralPath $voiceScript)) {
|
|
throw "Voice sidecar prerequisite not found at $voiceScript"
|
|
}
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
if (!(Test-Path -LiteralPath $Cloudflared -PathType Leaf)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path -LiteralPath $CloudflaredConfig -PathType Leaf)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
if (-not $RuntimeProvenancePath) {
|
|
$RuntimeProvenancePath = Join-Path $Workspace "public-runtime-launch-provenance.log"
|
|
}
|
|
|
|
Assert-FreshPublicProvenanceContract `
|
|
-SourceRoot $Workspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256 `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredSha256 $ExpectedCloudflaredSha256 `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ConfigSha256 $ExpectedCloudflaredConfigSha256
|
|
}
|
|
if ($offlineBootstrapMode) {
|
|
Assert-OfflineBootstrapContract `
|
|
-SourceRoot $Workspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256
|
|
}
|
|
|
|
. $uploadRootContract
|
|
$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-UploadRoot $UserUploadDir `
|
|
-ProbeWritable
|
|
$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-UploadRoot $resolvedUserUploadDir `
|
|
-StatePath $UserUploadManifestPath `
|
|
-RequireFile
|
|
$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-UploadRoot $resolvedUserUploadDir `
|
|
-StatePath $UserUploadWriteFreezePath
|
|
$resolvedOfflineBootstrapQuiescenceReceiptPath = ""
|
|
if ($offlineBootstrapMode) {
|
|
$resolvedOfflineBootstrapQuiescenceReceiptPath = Resolve-PublicRuntimePrivateStatePath `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-UploadRoot $resolvedUserUploadDir `
|
|
-StatePath $OfflineBootstrapQuiescenceReceiptPath `
|
|
-RequireFile
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "task_maintenance"
|
|
Assert-PublicRuntimeCoordinatedTaskNamesExact `
|
|
-TaskNames $CoordinatedTaskNames
|
|
$freshOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot `
|
|
-RequireEnabled
|
|
$freshTaskMaintenanceSnapshot = @(
|
|
Enter-PublicRuntimeTaskMaintenance `
|
|
-TaskNames $CoordinatedTaskNames `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
$freshTaskMaintenanceEntered = $true
|
|
$freshTaskMaintenanceWasEntered = $true
|
|
$freshDisabledOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot
|
|
foreach ($taskDefinition in @($freshDisabledOriginalTaskDefinitionSnapshot.entries)) {
|
|
if ([bool]$taskDefinition.enabled) {
|
|
throw "Original public runtime task definition remained enabled during maintenance"
|
|
}
|
|
}
|
|
}
|
|
$uploadManifestProof = Test-PublicRuntimeUploadManifest `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadManifestProbe `
|
|
-UploadRoot $resolvedUserUploadDir `
|
|
-ManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath
|
|
if (-not $uploadManifestProof.Ok) {
|
|
throw "Public upload migration receipt or current DB inventory is invalid: $($uploadManifestProof.Detail)"
|
|
}
|
|
$manifestPreservedObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-Name "preserved_object_count" `
|
|
-Role "Public upload manifest validator"
|
|
$manifestRequiredObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-Name "required_object_count" `
|
|
-Role "Public upload manifest validator"
|
|
$manifestRequiredReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-Name "required_reference_count" `
|
|
-Role "Public upload manifest validator"
|
|
$manifestCurrentObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-Name "current_object_count" `
|
|
-Role "Public upload manifest validator"
|
|
$manifestCurrentReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-Name "current_reference_count" `
|
|
-Role "Public upload manifest validator"
|
|
if (
|
|
$manifestRequiredObjectCount -gt $manifestRequiredReferenceCount -or
|
|
$manifestCurrentObjectCount -gt $manifestCurrentReferenceCount
|
|
) {
|
|
throw "Public upload manifest validator returned impossible DB inventory counts"
|
|
}
|
|
$manifestPreservedDecodeCounts = Get-PreservedDecodeCountProof `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-PreservedObjectCount $manifestPreservedObjectCount `
|
|
-Role "Public upload manifest validator"
|
|
$manifestRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-RequiredObjectCount $manifestRequiredObjectCount `
|
|
-RequiredReferenceCount $manifestRequiredReferenceCount `
|
|
-Role "Public upload manifest validator"
|
|
$manifestCurrentDecodeCounts = Get-CurrentDecodeInvalidCountProof `
|
|
-Payload $uploadManifestProof.Payload `
|
|
-CurrentObjectCount $manifestCurrentObjectCount `
|
|
-CurrentReferenceCount $manifestCurrentReferenceCount `
|
|
-Role "Public upload manifest validator"
|
|
$expectedDatabaseTargetSha256 = [string]$uploadManifestProof.Payload.database_target_sha256
|
|
if ($expectedDatabaseTargetSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Public upload inventory proof did not return a valid database target identity"
|
|
}
|
|
$offlineBootstrapReceiptProof = $null
|
|
if ($offlineBootstrapMode) {
|
|
$offlineBootstrapReceiptProof = Test-OfflineBootstrapQuiescenceReceipt `
|
|
-PythonPath $Python `
|
|
-ProbePath $offlineQuiescenceProbe `
|
|
-ReceiptPath $resolvedOfflineBootstrapQuiescenceReceiptPath `
|
|
-ExpectedReceiptSha256 $ExpectedOfflineBootstrapQuiescenceReceiptSha256 `
|
|
-ManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-StableSourceRoot $resolvedWorkspace `
|
|
-UploadRoot $resolvedUserUploadDir `
|
|
-ExpectedLegacySourceCommit $ExpectedOfflineBootstrapLegacySourceCommit `
|
|
-ExpectedLegacySourceTree $ExpectedOfflineBootstrapLegacySourceTree
|
|
if (-not $offlineBootstrapReceiptProof.Ok) {
|
|
throw "Offline bootstrap quiescence receipt is invalid"
|
|
}
|
|
$offlinePayload = $offlineBootstrapReceiptProof.Payload
|
|
$offlinePreservedObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $offlinePayload `
|
|
-Name "preserved_object_count" `
|
|
-Role "Offline bootstrap quiescence validator"
|
|
$offlineRequiredObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $offlinePayload `
|
|
-Name "unique_object_count" `
|
|
-Role "Offline bootstrap quiescence validator"
|
|
$offlineRequiredReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $offlinePayload `
|
|
-Name "reference_count" `
|
|
-Role "Offline bootstrap quiescence validator"
|
|
$offlinePreservedDecodeCounts = Get-PreservedDecodeCountProof `
|
|
-Payload $offlinePayload `
|
|
-PreservedObjectCount $offlinePreservedObjectCount `
|
|
-Role "Offline bootstrap quiescence validator"
|
|
$offlineRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof `
|
|
-Payload $offlinePayload `
|
|
-RequiredObjectCount $offlineRequiredObjectCount `
|
|
-RequiredReferenceCount $offlineRequiredReferenceCount `
|
|
-Role "Offline bootstrap quiescence validator"
|
|
if (
|
|
[string]$offlinePayload.database_target_sha256 -cne $expectedDatabaseTargetSha256 -or
|
|
$offlinePreservedObjectCount -ne $manifestPreservedObjectCount -or
|
|
[long]$offlinePayload.preserved_total_size_bytes -ne
|
|
[long]$uploadManifestProof.Payload.preserved_total_size_bytes -or
|
|
[string]$offlinePayload.preserved_inventory_sha256 -cne
|
|
[string]$uploadManifestProof.Payload.preserved_object_set_sha256 -or
|
|
$offlineRequiredReferenceCount -ne $manifestRequiredReferenceCount -or
|
|
$offlineRequiredObjectCount -ne $manifestRequiredObjectCount -or
|
|
$offlineRequiredReferenceCount -ne $manifestCurrentReferenceCount -or
|
|
$offlineRequiredObjectCount -ne $manifestCurrentObjectCount -or
|
|
[string]$offlinePayload.reference_set_sha256 -cne
|
|
[string]$uploadManifestProof.Payload.reference_set_sha256 -or
|
|
[string]$offlinePayload.reference_set_sha256 -cne [string]$uploadManifestProof.Payload.current_reference_set_sha256 -or
|
|
$offlinePreservedDecodeCounts.ValidCount -ne
|
|
$manifestPreservedDecodeCounts.ValidCount -or
|
|
$offlinePreservedDecodeCounts.InvalidCount -ne
|
|
$manifestPreservedDecodeCounts.InvalidCount -or
|
|
$offlineRequiredDecodeCounts.ObjectCount -ne
|
|
$manifestRequiredDecodeCounts.ObjectCount -or
|
|
$offlineRequiredDecodeCounts.ReferenceCount -ne
|
|
$manifestRequiredDecodeCounts.ReferenceCount -or
|
|
$offlineRequiredDecodeCounts.ObjectCount -ne
|
|
$manifestCurrentDecodeCounts.ObjectCount -or
|
|
$offlineRequiredDecodeCounts.ReferenceCount -ne
|
|
$manifestCurrentDecodeCounts.ReferenceCount -or
|
|
$offlinePayload.listener_absent -ne $true -or
|
|
$offlinePayload.tunnel_absent -ne $true
|
|
) {
|
|
throw "Offline bootstrap quiescence receipt drifted from the current database inventory"
|
|
}
|
|
}
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
$resolvedRuntimeProvenancePath = Initialize-RuntimeProvenanceOutput `
|
|
-OutputPath $RuntimeProvenancePath
|
|
$resolvedTaskRecoveryReceiptPath = Initialize-RuntimeProvenanceOutput `
|
|
-OutputPath "$resolvedRuntimeProvenancePath.tasks-restored.log"
|
|
|
|
# 승격 모드에서 config를 재작성하면 사전 pin과 실제 tunnel 입력이 달라진다.
|
|
# exact ingress가 이미 들어 있는 경우에만 이후 프로세스 mutation으로 진행한다.
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort `
|
|
-RequireUnchanged
|
|
|
|
$priorApiListenerProof = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadRootProbe `
|
|
-ExpectedUploadRoot $resolvedUserUploadDir `
|
|
-ExpectedApiCwd $ApiDir `
|
|
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
|
-ApiPort $ApiPort
|
|
if (-not $priorApiListenerProof.Ok) {
|
|
throw "Fresh public promotion requires one exact prior API listener identity: $($priorApiListenerProof.Detail)"
|
|
}
|
|
$priorApiProcess = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $($priorApiListenerProof.ListenerPid)" `
|
|
-ErrorAction SilentlyContinue
|
|
if ($null -eq $priorApiProcess) {
|
|
throw "Fresh public promotion lost the exact prior API listener before capture"
|
|
}
|
|
$priorCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
|
|
-ExactPath
|
|
)
|
|
if ($priorCloudflaredProcesses.Count -ne 1) {
|
|
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
|
|
}
|
|
$freshPriorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiListenerProof.ListenerPid `
|
|
-Role "prior api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
|
|
-Role "prior cloudflared" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$null = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorLocalVoiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorLocalVoiceHealth
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
|
|
}
|
|
$null = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 30
|
|
$priorPublicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
|
-Health $priorPublicVoiceHealth
|
|
$null = Assert-PublicUploadWriteFreezeReady `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 `
|
|
-TimeoutSec 30
|
|
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
|
|
-Names $freshManagedEnvironmentNames
|
|
}
|
|
|
|
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
|
# 프로세스는 살아 있는데 그 프로세스의 claude 세션만 죽은 상태는 /health를 통과하므로,
|
|
# /health 기준으로는 복구가 필요한 순간에 오히려 재기동을 건너뛴다(2026-08-07 사고).
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
$engineReady = $false
|
|
if ($null -ne $engineHealth -and $engineHealth.ok) {
|
|
$engineReady = Test-EngineReady -Port $EnginePort
|
|
}
|
|
if ($SkipEngineRestart) {
|
|
if (-not $engineReady) {
|
|
throw "Engine gateway is not ready on http://127.0.0.1:$EnginePort/ready"
|
|
}
|
|
} elseif (-not $engineReady) {
|
|
$null = @(
|
|
Stop-UvicornByPort `
|
|
-AppImport "engine_gateway.gateway:app" `
|
|
-Port $EnginePort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
|
|
# Start-Process는 리다이렉트 대상 로그를 덮어쓴다. 직전 사고 로그를 보존해야
|
|
# 재기동 후에도 원인을 추적할 수 있다.
|
|
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
|
|
if (Test-Path $logFile) {
|
|
# stable detached source의 provenance gate는 untracked 파일도 차단한다.
|
|
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
|
|
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @("-m", "uvicorn", "engine_gateway.gateway:app", "--host", "127.0.0.1", "--port", "$EnginePort") `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $EngineOutLog `
|
|
-RedirectStandardError $EngineErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
$engineReady = Wait-EngineReady -Port $EnginePort -TimeoutSec 90
|
|
if (-not $engineReady) {
|
|
Write-Warning "Engine gateway is still degraded; continuing admin/auth recovery"
|
|
}
|
|
$engineHealth = Get-JsonHealth -Uri "http://127.0.0.1:$EnginePort/health"
|
|
}
|
|
|
|
$env:ENVIRONMENT = "prod"
|
|
$env:ENGINE_URL = "http://127.0.0.1:$EnginePort"
|
|
$env:ENGINE_MODE = "claude_cli"
|
|
$env:VIGNETTE_LIVE_CLIENT_PROVIDER = "claude_cli"
|
|
$env:AUTH_DEV_LOGIN_ENABLED = "false"
|
|
$env:AUTO_SEED_PERSONAS = "false"
|
|
$env:ALLOW_SEED_PERSONA_FALLBACK = "false"
|
|
$env:VIGNETTE_VOICE_POC_SAMPLE_TTS = "false"
|
|
$env:VIGNETTE_VOICE_STT_PROVIDER = "local_whisper"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_URL = "ws://127.0.0.1:$WhisperPort/v1/listen"
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_MODEL = $WhisperModel
|
|
$env:VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE = $WhisperLanguage
|
|
$env:VIGNETTE_VOICE_TTS_PROVIDER = "melotts"
|
|
$env:VIGNETTE_MELOTTS_TTS_URL = "http://127.0.0.1:$MeloTtsPort"
|
|
$env:FRONTEND_BASE_URL = "https://vignette.chanpaca.net"
|
|
$env:USER_UPLOAD_DIR = $resolvedUserUploadDir
|
|
$env:USER_UPLOAD_MANIFEST_REQUIRED = "true"
|
|
$env:USER_UPLOAD_MANIFEST_PATH = $resolvedUserUploadManifestPath
|
|
$env:USER_UPLOAD_MANIFEST_SHA256 = $ExpectedUserUploadManifestSha256
|
|
$env:USER_UPLOAD_WRITE_FREEZE_PATH = $resolvedUserUploadWriteFreezePath
|
|
$env:PUBLIC_RUNTIME_DB_TARGET_SHA256 = $expectedDatabaseTargetSha256
|
|
$frontendOrigins = @("https://vignette.chanpaca.net", "https://vnet.18ka.net", "https://vignette-b1q.pages.dev")
|
|
$localViteOrigins = @()
|
|
foreach ($port in 5170..5180) {
|
|
$localViteOrigins += "http://localhost:$port"
|
|
$localViteOrigins += "http://127.0.0.1:$port"
|
|
}
|
|
$env:CORS_ORIGINS = ConvertTo-CompactJson -Value ($frontendOrigins + $localViteOrigins)
|
|
$env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
|
|
"api-vignette.chanpaca.net" = "https://vignette.chanpaca.net"
|
|
"api-vnet.18ka.net" = "https://vnet.18ka.net"
|
|
})
|
|
|
|
# 포트 리스너만으로는 올바른 provider/model을 증명하지 못한다. 첫 WS ready
|
|
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
|
|
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
|
if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) {
|
|
throw "Public upload-root promotion will not mutate local_whisper; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $WhisperPort) {
|
|
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
|
|
}
|
|
& $WhisperStartScript `
|
|
-Port $WhisperPort `
|
|
-Model $WhisperModel `
|
|
-Device $WhisperDevice `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "stt" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "local_whisper/$WhisperModel/$WhisperDevice did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
if (-not (Test-VoiceSidecarReady -Component "tts")) {
|
|
if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) {
|
|
throw "Public upload-root promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
|
|
}
|
|
if (Test-PortListener -Port $MeloTtsPort) {
|
|
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
|
|
}
|
|
& $MeloTtsStartScript `
|
|
-Port $MeloTtsPort `
|
|
-Language $MeloTtsLanguage `
|
|
-Device "cpu" `
|
|
-WaitReadySeconds 0
|
|
if (-not (Wait-VoiceSidecarReady -Component "tts" -TimeoutSec $VoiceSidecarReadySeconds)) {
|
|
throw "melotts/$MeloTtsModel did not become exactly ready before the API restart gate"
|
|
}
|
|
}
|
|
|
|
# 두 sidecar를 한 번 더 함께 검사해 개별 probe 사이의 TOCTOU를 닫는다.
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Voice sidecar readiness changed before the API restart gate"
|
|
}
|
|
|
|
$health = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/health"
|
|
$voiceHealth = Get-JsonHealth -Uri "http://127.0.0.1:$ApiPort/voice/health"
|
|
$apiUploadRootHealth = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadRootProbe `
|
|
-ExpectedUploadRoot $resolvedUserUploadDir `
|
|
-ExpectedApiCwd $ApiDir `
|
|
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
|
-ApiPort $ApiPort
|
|
$apiControlPlaneReady = (
|
|
$null -ne $health -and
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true -and
|
|
(Test-VoiceApiReady -Health $voiceHealth) -and
|
|
$apiUploadRootHealth.Ok
|
|
)
|
|
$proc = $null
|
|
$apiStoppedProcessIds = @()
|
|
$apiLaunchIdentity = $null
|
|
if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
|
Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
|
|
} else {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "api_cutover"
|
|
$freshMutationStarted = $true
|
|
# 공개 ingress를 먼저 닫은 뒤 API를 교체한다. 이 시점까지 freeze가
|
|
# drained 상태이므로 rollback 전후 어느 root에도 새 DB/file write가 없다.
|
|
$freshStoppedTunnelIds = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $priorCloudflaredProcesses `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "prior cloudflared before upload-root cutover"
|
|
)
|
|
}
|
|
$apiListenerProcess = Get-ExactLoopbackListenerProcess `
|
|
-Port $ApiPort `
|
|
-Role "public API"
|
|
if ($offlineBootstrapMode -and $null -ne $apiListenerProcess) {
|
|
throw "Offline bootstrap API listener reappeared after the quiescence receipt; refusing to stop an unpinned process"
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
if (
|
|
$null -eq $apiListenerProcess -or
|
|
[int]$apiListenerProcess.ProcessId -ne [int]$priorApiListenerProof.ListenerPid
|
|
) {
|
|
throw "Fresh public API listener changed before the exact stop boundary"
|
|
}
|
|
}
|
|
if ($null -eq $apiListenerProcess) {
|
|
# No socket owner means no API process is ours to stop. Command-line decoys
|
|
# that merely mention uvicorn/8001 are deliberately left untouched.
|
|
$apiStoppedProcessIds = @()
|
|
} else {
|
|
$apiStoppedProcessIds = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes @($apiListenerProcess) `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "exact public API listener"
|
|
)
|
|
}
|
|
if (@(Get-ListenerProcessIds -Port $ApiPort).Count -ne 0) {
|
|
throw "Public API listener remains on 127.0.0.1:$ApiPort after bounded stop"
|
|
}
|
|
|
|
$proc = Start-Process -WindowStyle Hidden -FilePath $Python `
|
|
-ArgumentList @(
|
|
"-m", "uvicorn", "app.main:app",
|
|
"--host", "127.0.0.1",
|
|
"--port", "$ApiPort",
|
|
"--workers", "1",
|
|
"--ws", "websockets",
|
|
"--ws-max-queue", "4"
|
|
) `
|
|
-WorkingDirectory $ApiDir `
|
|
-RedirectStandardOutput $OutLog `
|
|
-RedirectStandardError $ErrLog `
|
|
-PassThru
|
|
|
|
if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) {
|
|
if ($apiStoppedProcessIds -contains $proc.Id) {
|
|
throw "Public API cutover did not receive a replacement PID"
|
|
}
|
|
$apiLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $proc.Id `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($apiLaunchIdentity.executable_sha256 -ne $ExpectedPythonSha256.ToLowerInvariant()) {
|
|
throw "Public API executable SHA256 does not match the pinned Python"
|
|
}
|
|
foreach ($requiredArgument in @("uvicorn", "app.main:app", "--port", "$ApiPort", "--workers", "1", "--ws-max-queue", "4")) {
|
|
if ($apiLaunchIdentity.command_line.IndexOf($requiredArgument, [System.StringComparison]::Ordinal) -lt 0) {
|
|
throw "Public API command line is missing required argument: $requiredArgument"
|
|
}
|
|
}
|
|
}
|
|
|
|
Start-Sleep -Seconds 3
|
|
$health = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true -and
|
|
(Test-PublicUploadManifestHealth `
|
|
-Health $health `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256)
|
|
} `
|
|
-TimeoutSec $ApiReadySeconds
|
|
$voiceHealth = Wait-JsonHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec $VoiceApiReadySeconds
|
|
if ($RequireFreshPublicProvenance -or $offlineBootstrapMode) {
|
|
$null = Assert-PublicUploadWriteFreezeReady `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 `
|
|
-TimeoutSec 30
|
|
}
|
|
}
|
|
if (
|
|
$health.environment -ne "prod" -or
|
|
-not $health.db -or
|
|
-not $health.engine -or
|
|
-not (Test-PublicUploadManifestHealth `
|
|
-Health $health `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256)
|
|
) {
|
|
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
|
|
}
|
|
$apiUploadRootHealth = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadRootProbe `
|
|
-ExpectedUploadRoot $resolvedUserUploadDir `
|
|
-ExpectedApiCwd $ApiDir `
|
|
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
|
-ApiPort $ApiPort
|
|
if (-not $apiUploadRootHealth.Ok) {
|
|
throw "Public API does not use the pinned persistent upload root: $($apiUploadRootHealth.Detail)"
|
|
}
|
|
if (-not $RequireFreshPublicProvenance -and -not $offlineBootstrapMode) {
|
|
Assert-PublicUploadWritesAvailable `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec 30
|
|
}
|
|
if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
|
|
throw "Public voice API does not match the exact local provider/model contract: $($voiceHealth | ConvertTo-Json -Compress)"
|
|
}
|
|
|
|
if (!$SkipWebRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "web_preview"
|
|
}
|
|
Stop-NodeByPortHint -Port $WebPort
|
|
|
|
$build = Start-Process -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run build") `
|
|
-WorkingDirectory $WebDir `
|
|
-NoNewWindow `
|
|
-PassThru
|
|
# Windows PowerShell 5.1의 Start-Process -PassThru는 Process.Handle을 먼저
|
|
# 열지 않으면 timed WaitForExit 뒤에도 ExitCode가 $null로 남을 수 있다.
|
|
# 성공한 빌드를 실패로 오판하지 않도록 핸들을 대기 전에 확보한다.
|
|
$null = $build.Handle
|
|
if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) {
|
|
$null = @(
|
|
Stop-ProcessTreeBounded `
|
|
-RootProcess $build `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "web build"
|
|
)
|
|
throw "Web build timed out after $WebBuildTimeoutSeconds seconds"
|
|
}
|
|
$build.Refresh()
|
|
if ($null -eq $build.ExitCode) {
|
|
throw "Web build exit code is unavailable"
|
|
}
|
|
if ($build.ExitCode -ne 0) {
|
|
throw "Web build failed with exit code $($build.ExitCode)"
|
|
}
|
|
|
|
Start-Process -WindowStyle Hidden -FilePath "cmd.exe" `
|
|
-ArgumentList @("/c", "npm run preview -- --host 127.0.0.1 --port $WebPort") `
|
|
-WorkingDirectory $WebDir `
|
|
-RedirectStandardOutput $WebOutLog `
|
|
-RedirectStandardError $WebErrLog `
|
|
-PassThru | Out-Null
|
|
|
|
Wait-HttpStatus -Uri "http://127.0.0.1:$WebPort/" -TimeoutSec 30 | Out-Null
|
|
}
|
|
|
|
$cloudflaredProcess = $null
|
|
$cloudflaredLaunchIdentity = $null
|
|
$cloudflaredStoppedProcessIds = @($freshStoppedTunnelIds)
|
|
if (!$SkipCloudflaredRestart) {
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "cloudflared_cutover"
|
|
}
|
|
if (!(Test-Path $Cloudflared)) {
|
|
throw "cloudflared not found at $Cloudflared"
|
|
}
|
|
if (!(Test-Path $CloudflaredConfig)) {
|
|
throw "cloudflared config not found at $CloudflaredConfig"
|
|
}
|
|
|
|
if ($RouteCloudflareDns) {
|
|
foreach ($hostname in ($PublicWebHostnames + $PublicApiHostnames)) {
|
|
& $Cloudflared tunnel route dns $CloudflareTunnelName $hostname
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Warning "cloudflared DNS route failed for $hostname"
|
|
}
|
|
}
|
|
}
|
|
|
|
if (-not $RequireFreshPublicProvenance) {
|
|
Set-CloudflaredIngress `
|
|
-ConfigPath $CloudflaredConfig `
|
|
-ApiHostnames $PublicApiHostnames `
|
|
-WebHostnames $PublicWebHostnames `
|
|
-ApiPortValue $ApiPort `
|
|
-WebPortValue $WebPort
|
|
}
|
|
|
|
$resolvedCloudflaredConfig = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
if ($RequireFreshPublicProvenance) {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
if ($existingCloudflaredProcesses.Count -ne 0) {
|
|
throw "Fresh public tunnel config was reacquired by an unpinned process before launch"
|
|
}
|
|
$additionalCloudflaredStoppedProcessIds = @()
|
|
} else {
|
|
$existingCloudflaredProcesses = @(
|
|
Get-CloudflaredProcessesForConfig -ConfigPath $resolvedCloudflaredConfig
|
|
)
|
|
$additionalCloudflaredStoppedProcessIds = @(
|
|
Stop-ProcessesBounded `
|
|
-Processes $existingCloudflaredProcesses `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds `
|
|
-Role "cloudflared for $resolvedCloudflaredConfig"
|
|
)
|
|
}
|
|
$cloudflaredStoppedProcessIds = @(
|
|
@($cloudflaredStoppedProcessIds) + @($additionalCloudflaredStoppedProcessIds) |
|
|
Sort-Object -Unique
|
|
)
|
|
|
|
$cloudflaredProcess = Start-Process -WindowStyle Hidden -FilePath $Cloudflared `
|
|
-ArgumentList @("tunnel", "--config", $resolvedCloudflaredConfig, "run") `
|
|
-WorkingDirectory $Workspace `
|
|
-RedirectStandardOutput (Join-Path $Workspace "cloudflared.public.out.log") `
|
|
-RedirectStandardError (Join-Path $Workspace "cloudflared.public.err.log") `
|
|
-PassThru
|
|
|
|
if ($cloudflaredStoppedProcessIds -contains $cloudflaredProcess.Id) {
|
|
throw "Cloudflared did not receive a replacement PID"
|
|
}
|
|
if ($RequireFreshPublicProvenance) {
|
|
$cloudflaredLaunchIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredProcess.Id `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($cloudflaredLaunchIdentity.executable_sha256 -ne $ExpectedCloudflaredSha256.ToLowerInvariant()) {
|
|
throw "Fresh cloudflared executable SHA256 does not match its pin"
|
|
}
|
|
if ($cloudflaredLaunchIdentity.command_line.IndexOf($resolvedCloudflaredConfig, [System.StringComparison]::OrdinalIgnoreCase) -lt 0) {
|
|
throw "Fresh cloudflared command line is not pinned to the expected config"
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($RequireFreshPublicProvenance) {
|
|
$freshFailureStage = "identity_revalidation"
|
|
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
|
|
throw "Fresh public promotion did not produce both API and cloudflared identities"
|
|
}
|
|
|
|
$apiFinalListenerProof = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadRootProbe `
|
|
-ExpectedUploadRoot $resolvedUserUploadDir `
|
|
-ExpectedApiCwd $ApiDir `
|
|
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
|
-ApiPort $ApiPort
|
|
if (
|
|
-not $apiFinalListenerProof.Ok -or
|
|
[int]$apiFinalListenerProof.ListenerPid -ne [int]$apiLaunchIdentity.pid
|
|
) {
|
|
throw "Fresh public API launch PID is not the exact validated 127.0.0.1:$ApiPort listener"
|
|
}
|
|
$apiFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $apiFinalListenerProof.ListenerPid `
|
|
-Role "api" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$cloudflaredFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredLaunchIdentity.pid `
|
|
-Role "cloudflared" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$finalCloudflaredOwners = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
if (
|
|
$finalCloudflaredOwners.Count -ne 1 -or
|
|
[int]$finalCloudflaredOwners[0].ProcessId -ne [int]$cloudflaredFinalIdentity.pid
|
|
) {
|
|
throw "Fresh public tunnel config is not owned by the one pinned cloudflared PID"
|
|
}
|
|
foreach ($identityPair in @(
|
|
[pscustomobject]@{ Role = "api"; Launch = $apiLaunchIdentity; Final = $apiFinalIdentity },
|
|
[pscustomobject]@{ Role = "cloudflared"; Launch = $cloudflaredLaunchIdentity; Final = $cloudflaredFinalIdentity }
|
|
)) {
|
|
foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($identityPair.Launch[$field].ToString() -cne $identityPair.Final[$field].ToString()) {
|
|
throw "Fresh $($identityPair.Role) provenance drifted before receipt: $field"
|
|
}
|
|
}
|
|
}
|
|
|
|
$finalConfigSha256 = (Get-FileHash -LiteralPath $CloudflaredConfig -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
|
|
throw "Pinned cloudflared config drifted before provenance receipt"
|
|
}
|
|
$freshFailureStage = "public_health_validation"
|
|
$publicHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
} `
|
|
-TimeoutSec 60
|
|
$publicVoiceHealth = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicVoiceHealthUrl `
|
|
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
|
-TimeoutSec 30
|
|
$publicOpenApi = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicOpenApiUrl `
|
|
-IsHealthy {
|
|
param($document)
|
|
Test-RequiredOpenApiPaths `
|
|
-Document $document `
|
|
-RequiredPaths $RequiredPublicApiPaths
|
|
} `
|
|
-TimeoutSec 30
|
|
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
|
throw "Exact local voice sidecars changed before provenance receipt"
|
|
}
|
|
$receiptApiListenerProof = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $Python `
|
|
-ProbePath $uploadRootProbe `
|
|
-ExpectedUploadRoot $resolvedUserUploadDir `
|
|
-ExpectedApiCwd $ApiDir `
|
|
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
|
-ApiPort $ApiPort
|
|
if (
|
|
-not $receiptApiListenerProof.Ok -or
|
|
[int]$receiptApiListenerProof.ListenerPid -ne [int]$apiLaunchIdentity.pid
|
|
) {
|
|
throw "Fresh public API listener drifted before provenance receipt"
|
|
}
|
|
$apiFinalIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $receiptApiListenerProof.ListenerPid `
|
|
-Role "api before receipt" `
|
|
-ExpectedCwd $ApiDir `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
foreach ($field in @(
|
|
"pid",
|
|
"started_at_utc",
|
|
"executable_sha256",
|
|
"command_line_sha256",
|
|
"cwd"
|
|
)) {
|
|
if ($apiLaunchIdentity[$field].ToString() -cne $apiFinalIdentity[$field].ToString()) {
|
|
throw "Fresh API listener provenance drifted before receipt: $field"
|
|
}
|
|
}
|
|
$receiptTunnelOwners = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
if (
|
|
$receiptTunnelOwners.Count -ne 1 -or
|
|
[int]$receiptTunnelOwners[0].ProcessId -ne [int]$cloudflaredLaunchIdentity.pid
|
|
) {
|
|
throw "Fresh public tunnel config ownership drifted before provenance receipt"
|
|
}
|
|
$cloudflaredReceiptIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $cloudflaredLaunchIdentity.pid `
|
|
-Role "cloudflared before receipt" `
|
|
-ExpectedCwd $Workspace `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
foreach ($field in @(
|
|
"pid",
|
|
"started_at_utc",
|
|
"executable_sha256",
|
|
"command_line_sha256",
|
|
"cwd"
|
|
)) {
|
|
if (
|
|
$cloudflaredLaunchIdentity[$field].ToString() -cne
|
|
$cloudflaredReceiptIdentity[$field].ToString()
|
|
) {
|
|
throw "Fresh cloudflared provenance drifted before receipt: $field"
|
|
}
|
|
}
|
|
$cloudflaredFinalIdentity = $cloudflaredReceiptIdentity
|
|
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
|
|
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
|
|
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
|
|
throw "Could not record the psutil version used for process provenance"
|
|
}
|
|
$safeApiIdentity = ConvertTo-SafeProcessIdentity -Identity $apiFinalIdentity
|
|
$safeCloudflaredIdentity = ConvertTo-SafeProcessIdentity -Identity $cloudflaredFinalIdentity
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $freshDisabledOriginalTaskDefinitionSnapshot
|
|
|
|
$provenance = [ordered]@{
|
|
schema_version = "vignette.public-runtime-launch-provenance.v1"
|
|
status = "passed"
|
|
scope = "runtime_storage_commit_only"
|
|
operational_success = $false
|
|
task_recovery_receipt_path_sha256 = Get-Utf8Sha256 `
|
|
-Value $resolvedTaskRecoveryReceiptPath
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
source = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_commit = $ExpectedSourceCommit.ToLowerInvariant()
|
|
git_tree = $ExpectedSourceTree.ToLowerInvariant()
|
|
launcher_sha256 = (Get-FileHash -LiteralPath $PSCommandPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
clean_detached_head = $true
|
|
}
|
|
config = [ordered]@{
|
|
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
|
sha256 = $finalConfigSha256
|
|
}
|
|
storage = [ordered]@{
|
|
user_upload_root = $resolvedUserUploadDir
|
|
migration_manifest_sha256 = $ExpectedUserUploadManifestSha256
|
|
initialized_object_count = [int]$uploadManifestProof.Payload.required_object_count
|
|
initialized_reference_count = [int]$uploadManifestProof.Payload.required_reference_count
|
|
initialized_reference_set_sha256 = [string]$uploadManifestProof.Payload.reference_set_sha256
|
|
preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount
|
|
preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount
|
|
required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount
|
|
required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount
|
|
current_object_count = [int]$uploadManifestProof.Payload.current_object_count
|
|
current_reference_count = [int]$uploadManifestProof.Payload.current_reference_count
|
|
current_reference_set_sha256 = [string]$uploadManifestProof.Payload.current_reference_set_sha256
|
|
current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount
|
|
current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount
|
|
write_freeze_path_sha256 = [string]$health.upload_write_freeze.path_sha256
|
|
}
|
|
task_definitions = [ordered]@{
|
|
original_set_sha256 = [string]$freshOriginalTaskDefinitionSnapshot.set_sha256
|
|
disabled_pre_cutover_set_sha256 = [string]$freshDisabledOriginalTaskDefinitionSnapshot.set_sha256
|
|
}
|
|
public_validation = [ordered]@{
|
|
health_url = $CanonicalPublicHealthUrl
|
|
health = $true
|
|
voice_health_url = $CanonicalPublicVoiceHealthUrl
|
|
voice_health = $true
|
|
openapi_url = $CanonicalPublicOpenApiUrl
|
|
required_openapi_paths = @($RequiredPublicApiPaths)
|
|
openapi = $true
|
|
local_voice_sidecars = $true
|
|
}
|
|
replacement = [ordered]@{
|
|
api_stopped_pids = @($apiStoppedProcessIds)
|
|
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
|
|
api_new_pid = [int]$apiFinalIdentity.pid
|
|
cloudflared_new_pid = [int]$cloudflaredFinalIdentity.pid
|
|
}
|
|
processes = [ordered]@{
|
|
api = $safeApiIdentity
|
|
cloudflared = $safeCloudflaredIdentity
|
|
}
|
|
topology_inputs = [ordered]@{
|
|
repo_root = (Resolve-Path -LiteralPath $Workspace).Path
|
|
git_sha = $ExpectedSourceCommit.ToLowerInvariant()
|
|
api_pid = [int]$apiFinalIdentity.pid
|
|
api_started_at_utc = $apiFinalIdentity.started_at_utc
|
|
api_executable_name = $apiFinalIdentity.executable_name
|
|
api_executable_sha256 = $apiFinalIdentity.executable_sha256
|
|
api_command_line_sha256 = $apiFinalIdentity.command_line_sha256
|
|
api_cwd = $apiFinalIdentity.cwd
|
|
api_listen_port = $ApiPort
|
|
cloudflared_pid = [int]$cloudflaredFinalIdentity.pid
|
|
cloudflared_started_at_utc = $cloudflaredFinalIdentity.started_at_utc
|
|
cloudflared_executable_name = $cloudflaredFinalIdentity.executable_name
|
|
cloudflared_executable_sha256 = $cloudflaredFinalIdentity.executable_sha256
|
|
cloudflared_command_line_sha256 = $cloudflaredFinalIdentity.command_line_sha256
|
|
cloudflared_cwd = $cloudflaredFinalIdentity.cwd
|
|
psutil_version = $psutilVersion
|
|
}
|
|
}
|
|
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
|
|
$boundaryTunnelOwners = @(
|
|
Get-CloudflaredProcessesForConfig `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ExactPath
|
|
)
|
|
if (
|
|
$boundaryTunnelOwners.Count -ne 1 -or
|
|
[int]$boundaryTunnelOwners[0].ProcessId -ne [int]$cloudflaredFinalIdentity.pid
|
|
) {
|
|
throw "Fresh public tunnel config ownership drifted at the write-release boundary"
|
|
}
|
|
$freshFailureStage = "upload_write_release"
|
|
# 이 지점부터는 새 API/root를 외부에 유지한다. Exit 함수는 센티널을 지운 뒤
|
|
# health에서 write-unfrozen을 확인하므로, 호출 중 오류가 나도 새 root에 쓰기가
|
|
# 시작됐을 수 있다. 따라서 release 시도 전에 rollback 금지 경계를 확정한다.
|
|
$freshNoRollback = $true
|
|
Exit-PublicUploadWriteFreeze `
|
|
-FreezePath $resolvedUserUploadWriteFreezePath `
|
|
-ExpectedTokenSha256 $uploadManifestProof.Payload.write_freeze_token_sha256 `
|
|
-HealthUri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec 30
|
|
$null = Wait-JsonHealth `
|
|
-Uri $CanonicalPublicHealthUrl `
|
|
-IsHealthy {
|
|
param($health)
|
|
$freeze = $health.upload_write_freeze
|
|
$manifest = $health.upload_manifest
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true -and
|
|
$null -ne $freeze -and
|
|
$freeze.capable -eq $true -and
|
|
$freeze.active -eq $false -and
|
|
$freeze.valid -eq $true -and
|
|
[int]$freeze.in_flight -eq 0 -and
|
|
$null -ne $manifest -and
|
|
$manifest.required -eq $true -and
|
|
$manifest.validated -eq $true -and
|
|
[string]$manifest.manifest_sha256 -ceq $ExpectedUserUploadManifestSha256
|
|
} `
|
|
-TimeoutSec 60
|
|
$freshFailureStage = "receipt_publish"
|
|
try {
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $freshDisabledOriginalTaskDefinitionSnapshot
|
|
Write-Utf8TextAtomically `
|
|
-OutputPath $resolvedRuntimeProvenancePath `
|
|
-Value ($provenanceJson + [Environment]::NewLine)
|
|
} catch {
|
|
# write release 뒤에는 prior root로 되돌릴 수 없다. passed receipt 없이 새
|
|
# runtime/root와 disabled tasks를 유지하고 trap이 별도 failed evidence를 쓴다.
|
|
throw "Fresh public promotion retained the new runtime after write release, but no atomic passed receipt was published. $($_.Exception.Message)"
|
|
}
|
|
$freshFailureStage = "task_definition_cutover"
|
|
Assert-FreshPublicProvenanceContract `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256 `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredSha256 $ExpectedCloudflaredSha256 `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ConfigSha256 $ExpectedCloudflaredConfigSha256
|
|
$bootInstallAction = {
|
|
& $bootTaskInstaller `
|
|
-StableSourceRoot $resolvedWorkspace `
|
|
-Python $Python `
|
|
-Cloudflared $Cloudflared `
|
|
-CloudflaredConfig $resolvedCloudflaredConfig `
|
|
-UserUploadDir $resolvedUserUploadDir `
|
|
-UserUploadManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-TaskName "VignettePublicRuntime" `
|
|
-InitiallyDisabled
|
|
}.GetNewClosure()
|
|
$watchdogInstallAction = {
|
|
& $watchdogTaskInstaller `
|
|
-StableSourceRoot $resolvedWorkspace `
|
|
-TaskName "VignettePublicRuntimeWatchdog" `
|
|
-Python $Python `
|
|
-UserUploadDir $resolvedUserUploadDir `
|
|
-UserUploadManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-Cloudflared $Cloudflared `
|
|
-CloudflaredConfig $resolvedCloudflaredConfig `
|
|
-PublicHealthUrl $CanonicalPublicHealthUrl `
|
|
-InitiallyDisabled
|
|
}.GetNewClosure()
|
|
Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled `
|
|
-BootInstaller $bootInstallAction `
|
|
-WatchdogInstaller $watchdogInstallAction `
|
|
-MaintenanceSnapshot $freshTaskMaintenanceSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-PublicRuntimeTasksDisabledAndIdle `
|
|
-Snapshot $freshTaskMaintenanceSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$freshNewDisabledTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot
|
|
$null = Assert-NewPublicRuntimeTaskDefinitionsPinned `
|
|
-Snapshot $freshNewDisabledTaskDefinitionSnapshot `
|
|
-StableSourceRoot $resolvedWorkspace `
|
|
-ExpectedSourceCommit $ExpectedSourceCommit `
|
|
-ExpectedSourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-UserUploadDir $resolvedUserUploadDir `
|
|
-UserUploadManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredConfigPath $resolvedCloudflaredConfig `
|
|
-PublicHealthUrl $CanonicalPublicHealthUrl
|
|
Assert-FreshPublicProvenanceContract `
|
|
-SourceRoot $resolvedWorkspace `
|
|
-SourceCommit $ExpectedSourceCommit `
|
|
-SourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-PythonSha256 $ExpectedPythonSha256 `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredSha256 $ExpectedCloudflaredSha256 `
|
|
-ConfigPath $resolvedCloudflaredConfig `
|
|
-ConfigSha256 $ExpectedCloudflaredConfigSha256
|
|
|
|
$freshFailureStage = "task_maintenance_exit"
|
|
$freshOperationalTaskDefinitionSnapshot = Enable-NewPublicRuntimeTaskDefinitions `
|
|
-DisabledSnapshot $freshNewDisabledTaskDefinitionSnapshot
|
|
$null = Assert-NewPublicRuntimeTaskDefinitionsPinned `
|
|
-Snapshot $freshOperationalTaskDefinitionSnapshot `
|
|
-StableSourceRoot $resolvedWorkspace `
|
|
-ExpectedSourceCommit $ExpectedSourceCommit `
|
|
-ExpectedSourceTree $ExpectedSourceTree `
|
|
-PythonPath $Python `
|
|
-UserUploadDir $resolvedUserUploadDir `
|
|
-UserUploadManifestPath $resolvedUserUploadManifestPath `
|
|
-ExpectedUserUploadManifestSha256 $ExpectedUserUploadManifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
|
-CloudflaredPath $Cloudflared `
|
|
-CloudflaredConfigPath $resolvedCloudflaredConfig `
|
|
-PublicHealthUrl $CanonicalPublicHealthUrl `
|
|
-AllowEnabled
|
|
$restoredTaskTruth = Get-PublicRuntimeTaskMaintenanceState `
|
|
-Snapshot $freshTaskMaintenanceSnapshot
|
|
if (
|
|
-not [bool]$restoredTaskTruth.verified -or
|
|
-not [bool]$restoredTaskTruth.restored_to_snapshot
|
|
) {
|
|
throw "Public runtime task maintenance exit could not prove the original task state"
|
|
}
|
|
$freshTasksRestored = $true
|
|
$freshFailureStage = "task_recovery_receipt_publish"
|
|
$runtimeReceiptSha256 = (
|
|
Get-FileHash `
|
|
-LiteralPath $resolvedRuntimeProvenancePath `
|
|
-Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
$taskRecoveryReceipt = [ordered]@{
|
|
schema_version = "vignette.public-runtime-task-recovery.v1"
|
|
status = "passed"
|
|
operational_success = $true
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
|
runtime_receipt_sha256 = $runtimeReceiptSha256
|
|
preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount
|
|
preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount
|
|
required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount
|
|
required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount
|
|
current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount
|
|
current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount
|
|
task_definitions = [ordered]@{
|
|
original_set_sha256 = [string]$freshOriginalTaskDefinitionSnapshot.set_sha256
|
|
installed_disabled_set_sha256 = [string]$freshNewDisabledTaskDefinitionSnapshot.set_sha256
|
|
operational_set_sha256 = [string]$freshOperationalTaskDefinitionSnapshot.set_sha256
|
|
}
|
|
task_maintenance = [ordered]@{
|
|
restored = $freshTasksRestored
|
|
state_verified = [bool]$restoredTaskTruth.verified
|
|
task_names = @($CoordinatedTaskNames | Sort-Object -Unique)
|
|
}
|
|
}
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $freshOperationalTaskDefinitionSnapshot
|
|
Write-Utf8TextAtomically `
|
|
-OutputPath $resolvedTaskRecoveryReceiptPath `
|
|
-Value ((ConvertTo-Json -InputObject $taskRecoveryReceipt -Depth 5) + [Environment]::NewLine)
|
|
$freshTaskMaintenanceEntered = $false
|
|
$freshPromotionCommitted = $true
|
|
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
|
|
Write-Output "Fresh public task recovery: $resolvedTaskRecoveryReceiptPath"
|
|
}
|
|
|
|
if ($engineReady) {
|
|
Write-Output "Engine gateway ready (real generation proven) on http://127.0.0.1:$EnginePort"
|
|
} else {
|
|
Write-Warning "Engine gateway degraded; admin/auth control plane remains available"
|
|
}
|
|
if ($null -ne $proc) {
|
|
Write-Output "Public API running on http://127.0.0.1:$ApiPort with PID $($proc.Id)"
|
|
} else {
|
|
Write-Output "Public API kept running on http://127.0.0.1:$ApiPort"
|
|
}
|
|
if (!$SkipWebRestart) {
|
|
Write-Output "Public vnet web preview running on http://127.0.0.1:$WebPort"
|
|
}
|
|
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
|
|
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
|
|
Write-Output "User upload root: $resolvedUserUploadDir"
|
|
} finally {
|
|
if ($null -ne $recoveryLock) {
|
|
try {
|
|
$recoveryLock.Dispose()
|
|
} catch {
|
|
Write-Warning "Public runtime recovery lock release failed: $($_.Exception.Message)"
|
|
}
|
|
}
|
|
}
|