공개 런타임 롤백 경계 강화
This commit is contained in:
Yun Chan
parent
ff3c79dfc2
commit
aaebe4450e
7 changed files with 1271 additions and 30 deletions
|
|
@ -15,6 +15,7 @@
|
|||
[switch]$RouteCloudflareDns,
|
||||
[string]$CloudflareTunnelName = "vignette",
|
||||
[switch]$SkipCloudflaredRestart,
|
||||
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
||||
[switch]$RequireFreshPublicProvenance,
|
||||
[string]$ExpectedSourceCommit = "",
|
||||
[string]$ExpectedSourceTree = "",
|
||||
|
|
@ -51,6 +52,15 @@ $WhisperLanguage = "ko"
|
|||
$WhisperDevice = "cpu"
|
||||
$MeloTtsModel = "melotts-korean"
|
||||
$MeloTtsLanguage = "KR"
|
||||
$CanonicalPublicHealthUrl = "https://api-vignette.chanpaca.net/health"
|
||||
$CanonicalPublicVoiceHealthUrl = "https://api-vignette.chanpaca.net/voice/health"
|
||||
$CanonicalPublicOpenApiUrl = "https://api-vignette.chanpaca.net/openapi.json"
|
||||
$RequiredPublicApiPaths = @(
|
||||
"/health",
|
||||
"/voice/health",
|
||||
"/voice/speech",
|
||||
"/admin/voice-runtime"
|
||||
)
|
||||
$PublicApiHostnames = @("api-vignette.chanpaca.net", "api-vnet.18ka.net")
|
||||
$PublicWebHostnames = @("vnet.18ka.net")
|
||||
|
||||
|
|
@ -97,6 +107,18 @@ function Test-PortListener {
|
|||
return $null -ne $listener
|
||||
}
|
||||
|
||||
function Get-ListenerProcessIds {
|
||||
param([int]$Port)
|
||||
|
||||
return @(
|
||||
Get-NetTCPConnection `
|
||||
-State Listen `
|
||||
-LocalPort $Port `
|
||||
-ErrorAction SilentlyContinue |
|
||||
Select-Object -ExpandProperty OwningProcess -Unique
|
||||
)
|
||||
}
|
||||
|
||||
function Test-VoiceSidecarReady {
|
||||
param(
|
||||
[ValidateSet("stt", "tts")]
|
||||
|
|
@ -156,6 +178,74 @@ function Test-VoiceApiReady {
|
|||
)
|
||||
}
|
||||
|
||||
function Test-PriorVoiceApiReady {
|
||||
param([object]$Health)
|
||||
|
||||
return (
|
||||
$null -ne $Health -and
|
||||
$Health.status -eq "ok" -and
|
||||
$Health.available -eq $true -and
|
||||
$Health.stt_available -eq $true -and
|
||||
$Health.tts_available -eq $true -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_provider) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.stt_model) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_provider) -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$Health.tts_model)
|
||||
)
|
||||
}
|
||||
|
||||
function ConvertTo-SafeVoiceHealthContract {
|
||||
param([object]$Health)
|
||||
|
||||
return [ordered]@{
|
||||
status = [string]$Health.status
|
||||
available = [bool]$Health.available
|
||||
stt_available = [bool]$Health.stt_available
|
||||
tts_available = [bool]$Health.tts_available
|
||||
stt_provider = [string]$Health.stt_provider
|
||||
stt_model = [string]$Health.stt_model
|
||||
tts_provider = [string]$Health.tts_provider
|
||||
tts_model = [string]$Health.tts_model
|
||||
uvicorn_ws_max_queue = [int]$Health.limits.uvicorn_ws_max_queue
|
||||
}
|
||||
}
|
||||
|
||||
function Test-VoiceHealthContract {
|
||||
param(
|
||||
[object]$Health,
|
||||
[System.Collections.IDictionary]$Expected
|
||||
)
|
||||
|
||||
if (-not (Test-PriorVoiceApiReady -Health $Health)) {
|
||||
return $false
|
||||
}
|
||||
$actual = ConvertTo-SafeVoiceHealthContract -Health $Health
|
||||
foreach ($name in $Expected.Keys) {
|
||||
if ($actual[$name].ToString() -cne $Expected[$name].ToString()) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-RequiredOpenApiPaths {
|
||||
param(
|
||||
[object]$Document,
|
||||
[string[]]$RequiredPaths
|
||||
)
|
||||
|
||||
if ($null -eq $Document -or $null -eq $Document.paths) {
|
||||
return $false
|
||||
}
|
||||
$actualPaths = @($Document.paths.PSObject.Properties.Name)
|
||||
foreach ($requiredPath in $RequiredPaths) {
|
||||
if ($actualPaths -notcontains $requiredPath) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-EngineReady {
|
||||
param(
|
||||
[int]$Port,
|
||||
|
|
@ -247,16 +337,15 @@ function Stop-ProcessesBounded {
|
|||
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
|
||||
}
|
||||
|
||||
function Stop-UvicornByPort {
|
||||
function Get-UvicornProcessesByPort {
|
||||
param(
|
||||
[string]$AppImport,
|
||||
[int]$Port,
|
||||
[int]$TimeoutSec = 15
|
||||
[int]$Port
|
||||
)
|
||||
|
||||
# Name 조건이 없으면 같은 문자열을 인자로 들고 있는 셸/래퍼 프로세스까지 매칭해
|
||||
# 호출자 자신을 죽일 수 있다. 대상은 항상 python 프로세스다.
|
||||
$processes = @(
|
||||
return @(
|
||||
Get-CimInstance Win32_Process |
|
||||
Where-Object {
|
||||
$_.Name -like "python*" -and
|
||||
|
|
@ -265,6 +354,16 @@ function Stop-UvicornByPort {
|
|||
$_.CommandLine -like "*--port $Port*"
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function Stop-UvicornByPort {
|
||||
param(
|
||||
[string]$AppImport,
|
||||
[int]$Port,
|
||||
[int]$TimeoutSec = 15
|
||||
)
|
||||
|
||||
$processes = @(Get-UvicornProcessesByPort -AppImport $AppImport -Port $Port)
|
||||
return @(
|
||||
Stop-ProcessesBounded `
|
||||
-Processes $processes `
|
||||
|
|
@ -298,7 +397,7 @@ function Wait-ProcessIdentity {
|
|||
param(
|
||||
[int]$ProcessId,
|
||||
[string]$Role,
|
||||
[string]$ExpectedCwd,
|
||||
[string]$ExpectedCwd = "",
|
||||
[int]$TimeoutSec = 15
|
||||
)
|
||||
|
||||
|
|
@ -314,20 +413,26 @@ function Wait-ProcessIdentity {
|
|||
) {
|
||||
$identityProbeArgs = @(
|
||||
"-X", "utf8", "-c",
|
||||
"import hashlib,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(p.cmdline()).encode('utf-8', errors='strict')).hexdigest())",
|
||||
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00', 'Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8', errors='strict')).hexdigest()); print(json.dumps(argv[1:], ensure_ascii=True, separators=(',', ':'))); print(json.dumps(p.environ(), ensure_ascii=True, separators=(',', ':')))",
|
||||
"$ProcessId"
|
||||
)
|
||||
$identityProbe = @(& $Python @identityProbeArgs)
|
||||
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 3) {
|
||||
if ($LASTEXITCODE -ne 0 -or $identityProbe.Count -ne 5) {
|
||||
throw "Could not prove $Role psutil identity for PID $ProcessId"
|
||||
}
|
||||
$actualCwd = $identityProbe[0].Trim()
|
||||
$startedAtUtc = $identityProbe[1].Trim()
|
||||
$commandLineSha256 = $identityProbe[2].Trim().ToLowerInvariant()
|
||||
$argumentList = @($identityProbe[3] | ConvertFrom-Json)
|
||||
$environmentObject = $identityProbe[4] | ConvertFrom-Json
|
||||
$processEnvironment = [ordered]@{}
|
||||
foreach ($property in $environmentObject.PSObject.Properties) {
|
||||
$processEnvironment[$property.Name] = [string]$property.Value
|
||||
}
|
||||
if (-not $actualCwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
||||
throw "$Role psutil identity is incomplete for PID $ProcessId"
|
||||
}
|
||||
if (-not [string]::Equals(
|
||||
if ($ExpectedCwd -and -not [string]::Equals(
|
||||
[System.IO.Path]::GetFullPath($actualCwd),
|
||||
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
|
|
@ -343,6 +448,8 @@ function Wait-ProcessIdentity {
|
|||
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
command_line = $process.CommandLine
|
||||
command_line_sha256 = $commandLineSha256
|
||||
argument_list = $argumentList
|
||||
environment = $processEnvironment
|
||||
cwd = $actualCwd
|
||||
}
|
||||
}
|
||||
|
|
@ -367,6 +474,274 @@ function ConvertTo-SafeProcessIdentity {
|
|||
}
|
||||
}
|
||||
|
||||
function Save-ManagedEnvironment {
|
||||
param([string[]]$Names)
|
||||
|
||||
$snapshot = [ordered]@{}
|
||||
foreach ($name in $Names) {
|
||||
$value = [System.Environment]::GetEnvironmentVariable(
|
||||
$name,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
$snapshot[$name] = [ordered]@{
|
||||
present = $null -ne $value
|
||||
value = $value
|
||||
}
|
||||
}
|
||||
return $snapshot
|
||||
}
|
||||
|
||||
function Restore-ManagedEnvironment {
|
||||
param([System.Collections.IDictionary]$Snapshot)
|
||||
|
||||
foreach ($name in $Snapshot.Keys) {
|
||||
$entry = $Snapshot[$name]
|
||||
if ($entry.present) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
[string]$entry.value,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
} else {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
$null,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Save-CompleteProcessEnvironment {
|
||||
$snapshot = [ordered]@{}
|
||||
$environment = [System.Environment]::GetEnvironmentVariables(
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
foreach ($name in $environment.Keys) {
|
||||
$snapshot[[string]$name] = [string]$environment[$name]
|
||||
}
|
||||
return $snapshot
|
||||
}
|
||||
|
||||
function Set-CompleteProcessEnvironment {
|
||||
param([System.Collections.IDictionary]$Environment)
|
||||
|
||||
$current = [System.Environment]::GetEnvironmentVariables(
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
foreach ($name in @($current.Keys)) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
[string]$name,
|
||||
$null,
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
foreach ($name in $Environment.Keys) {
|
||||
[System.Environment]::SetEnvironmentVariable(
|
||||
[string]$name,
|
||||
[string]$Environment[$name],
|
||||
[System.EnvironmentVariableTarget]::Process
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertTo-WindowsCommandLineArgument {
|
||||
param([AllowEmptyString()][string]$Argument)
|
||||
|
||||
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
||||
return $Argument
|
||||
}
|
||||
|
||||
$builder = New-Object System.Text.StringBuilder
|
||||
$null = $builder.Append('"')
|
||||
$backslashes = 0
|
||||
foreach ($character in $Argument.ToCharArray()) {
|
||||
if ($character -eq '\') {
|
||||
$backslashes++
|
||||
continue
|
||||
}
|
||||
if ($character -eq '"') {
|
||||
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
||||
$null = $builder.Append('"')
|
||||
$backslashes = 0
|
||||
continue
|
||||
}
|
||||
if ($backslashes -gt 0) {
|
||||
$null = $builder.Append(('\' * $backslashes))
|
||||
$backslashes = 0
|
||||
}
|
||||
$null = $builder.Append($character)
|
||||
}
|
||||
if ($backslashes -gt 0) {
|
||||
$null = $builder.Append(('\' * ($backslashes * 2)))
|
||||
}
|
||||
$null = $builder.Append('"')
|
||||
return $builder.ToString()
|
||||
}
|
||||
|
||||
function Join-WindowsArgumentList {
|
||||
param([object[]]$ArgumentList)
|
||||
|
||||
return (@(
|
||||
foreach ($argument in $ArgumentList) {
|
||||
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
||||
}
|
||||
) -join ' ')
|
||||
}
|
||||
|
||||
function Start-PinnedPriorProcess {
|
||||
param(
|
||||
[System.Collections.IDictionary]$Identity,
|
||||
[string]$Role,
|
||||
[string]$StdoutLog,
|
||||
[string]$StderrLog
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Identity.executable_path -PathType Leaf)) {
|
||||
throw "Prior $Role executable is unavailable"
|
||||
}
|
||||
$actualExecutableSha256 = (
|
||||
Get-FileHash -LiteralPath $Identity.executable_path -Algorithm SHA256
|
||||
).Hash.ToLowerInvariant()
|
||||
if ($actualExecutableSha256 -ne $Identity.executable_sha256) {
|
||||
throw "Prior $Role executable SHA256 drift"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
||||
throw "Prior $Role working directory is unavailable"
|
||||
}
|
||||
if (@($Identity.argument_list).Count -eq 0) {
|
||||
throw "Prior $Role argument list is unavailable"
|
||||
}
|
||||
if ($null -eq $Identity.environment -or $Identity.environment.Count -eq 0) {
|
||||
throw "Prior $Role environment is unavailable"
|
||||
}
|
||||
|
||||
$callerEnvironment = Save-CompleteProcessEnvironment
|
||||
try {
|
||||
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
||||
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
||||
return Start-Process -WindowStyle Hidden `
|
||||
-FilePath $Identity.executable_path `
|
||||
-ArgumentList $argumentString `
|
||||
-WorkingDirectory $Identity.cwd `
|
||||
-RedirectStandardOutput $StdoutLog `
|
||||
-RedirectStandardError $StderrLog `
|
||||
-PassThru
|
||||
} finally {
|
||||
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
||||
}
|
||||
}
|
||||
|
||||
function Restore-PriorPublicRuntime {
|
||||
param(
|
||||
[System.Collections.IDictionary]$PriorApi,
|
||||
[System.Collections.IDictionary]$PriorCloudflared,
|
||||
[System.Collections.IDictionary]$PriorLocalVoiceContract,
|
||||
[System.Collections.IDictionary]$PriorPublicVoiceContract,
|
||||
[System.Collections.IDictionary]$EnvironmentSnapshot,
|
||||
[string]$ConfigPath,
|
||||
[int]$ApiPortValue,
|
||||
[string]$HealthUrl,
|
||||
[string]$VoiceHealthUrl,
|
||||
[int]$TimeoutSec
|
||||
)
|
||||
|
||||
$null = @(
|
||||
Stop-UvicornByPort `
|
||||
-AppImport "app.main:app" `
|
||||
-Port $ApiPortValue `
|
||||
-TimeoutSec $TimeoutSec
|
||||
)
|
||||
Restore-ManagedEnvironment -Snapshot $EnvironmentSnapshot
|
||||
$priorApiProcess = Start-PinnedPriorProcess `
|
||||
-Identity $PriorApi `
|
||||
-Role "api" `
|
||||
-StdoutLog (Join-Path $PriorApi.cwd "api.public.rollback.out.log") `
|
||||
-StderrLog (Join-Path $PriorApi.cwd "api.public.rollback.err.log")
|
||||
$priorApiIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorApiProcess.Id `
|
||||
-Role "restored prior api" `
|
||||
-ExpectedCwd $PriorApi.cwd `
|
||||
-TimeoutSec $TimeoutSec
|
||||
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
||||
if ($PriorApi[$field].ToString() -cne $priorApiIdentity[$field].ToString()) {
|
||||
throw "Restored prior API identity drift: $field"
|
||||
}
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPortValue/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPortValue/voice/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
Test-VoiceHealthContract -Health $health -Expected $PriorLocalVoiceContract
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Restored prior runtime does not have the exact local voice sidecars"
|
||||
}
|
||||
|
||||
$resolvedConfigPath = (Resolve-Path -LiteralPath $ConfigPath).Path
|
||||
$currentCloudflared = @(
|
||||
Get-CloudflaredProcessesForConfig `
|
||||
-ConfigPath $resolvedConfigPath `
|
||||
-ExactPath
|
||||
)
|
||||
$null = @(
|
||||
Stop-ProcessesBounded `
|
||||
-Processes $currentCloudflared `
|
||||
-TimeoutSec $TimeoutSec `
|
||||
-Role "failed fresh cloudflared"
|
||||
)
|
||||
$priorCloudflaredProcess = Start-PinnedPriorProcess `
|
||||
-Identity $PriorCloudflared `
|
||||
-Role "cloudflared" `
|
||||
-StdoutLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.out.log") `
|
||||
-StderrLog (Join-Path $PriorCloudflared.cwd "cloudflared.public.rollback.err.log")
|
||||
$priorCloudflaredIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorCloudflaredProcess.Id `
|
||||
-Role "restored prior cloudflared" `
|
||||
-ExpectedCwd $PriorCloudflared.cwd `
|
||||
-TimeoutSec $TimeoutSec
|
||||
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
||||
if ($PriorCloudflared[$field].ToString() -cne $priorCloudflaredIdentity[$field].ToString()) {
|
||||
throw "Restored prior cloudflared identity drift: $field"
|
||||
}
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $HealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $VoiceHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
Test-VoiceHealthContract -Health $health -Expected $PriorPublicVoiceContract
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
|
||||
return [ordered]@{
|
||||
api = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
||||
cloudflared = ConvertTo-SafeProcessIdentity -Identity $priorCloudflaredIdentity
|
||||
local_health = $true
|
||||
local_voice_health = $true
|
||||
public_health = $true
|
||||
public_voice_health = $true
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-NodeByPortHint {
|
||||
param([int]$Port)
|
||||
|
||||
|
|
@ -515,6 +890,39 @@ function Write-Utf8TextAtomically {
|
|||
}
|
||||
}
|
||||
|
||||
function Write-FailedFreshPromotionEvidence {
|
||||
param(
|
||||
[string]$OutputPath,
|
||||
[string]$FailureStage,
|
||||
[bool]$RollbackSucceeded,
|
||||
[object]$RollbackResult,
|
||||
[string]$SourceCommit,
|
||||
[string]$SourceTree
|
||||
)
|
||||
|
||||
# Default receipt는 stable detached root의 *.log 경계에 놓일 수 있다. 실패 증거도
|
||||
# 최종 suffix를 .log로 유지해야 rollback 직후 source-clean provenance를 깨지 않는다.
|
||||
$failedPath = "$OutputPath.failed.log"
|
||||
$payload = [ordered]@{
|
||||
schema_version = "vignette.public-runtime-launch-failure.v1"
|
||||
status = if ($RollbackSucceeded) { "failed_rolled_back" } else { "failed_rollback" }
|
||||
captured_at_utc = (Get-Date).ToUniversalTime().ToString("o")
|
||||
failure_stage = $FailureStage
|
||||
source = [ordered]@{
|
||||
git_commit = $SourceCommit.ToLowerInvariant()
|
||||
git_tree = $SourceTree.ToLowerInvariant()
|
||||
}
|
||||
rollback = [ordered]@{
|
||||
attempted = $true
|
||||
succeeded = $RollbackSucceeded
|
||||
result = $RollbackResult
|
||||
}
|
||||
}
|
||||
$json = ConvertTo-Json -InputObject $payload -Depth 8
|
||||
Write-Utf8TextAtomically -OutputPath $failedPath -Value ($json + [Environment]::NewLine)
|
||||
return $failedPath
|
||||
}
|
||||
|
||||
function Assert-FreshPublicProvenanceContract {
|
||||
param(
|
||||
[string]$SourceRoot,
|
||||
|
|
@ -534,6 +942,22 @@ function Assert-FreshPublicProvenanceContract {
|
|||
if ($SkipCloudflaredRestart) {
|
||||
throw "-RequireFreshPublicProvenance forbids -SkipCloudflaredRestart"
|
||||
}
|
||||
if (-not $SkipEngineRestart) {
|
||||
throw "-RequireFreshPublicProvenance requires -SkipEngineRestart; engine is an unchanged precondition"
|
||||
}
|
||||
if (-not $SkipWebRestart) {
|
||||
throw "-RequireFreshPublicProvenance requires -SkipWebRestart; web preview is outside the API/tunnel transaction"
|
||||
}
|
||||
if ($RouteCloudflareDns) {
|
||||
throw "-RequireFreshPublicProvenance forbids DNS route mutation"
|
||||
}
|
||||
if (-not [string]::Equals(
|
||||
$PublicHealthUrl,
|
||||
$CanonicalPublicHealthUrl,
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
)) {
|
||||
throw "Fresh public promotion requires the canonical HTTPS public health URL"
|
||||
}
|
||||
foreach ($sourcePin in @($SourceCommit, $SourceTree)) {
|
||||
if ($sourcePin -notmatch "^[0-9a-fA-F]{40}$") {
|
||||
throw "Fresh public provenance requires exact source commit and tree pins"
|
||||
|
|
@ -657,6 +1081,85 @@ function Set-CloudflaredIngress {
|
|||
}
|
||||
}
|
||||
|
||||
$freshMutationStarted = $false
|
||||
$freshPromotionCommitted = $false
|
||||
$freshFailureStage = "preflight"
|
||||
$freshPriorApiIdentity = $null
|
||||
$freshPriorCloudflaredIdentity = $null
|
||||
$freshPriorLocalVoiceContract = $null
|
||||
$freshPriorPublicVoiceContract = $null
|
||||
$freshEnvironmentSnapshot = $null
|
||||
$resolvedRuntimeProvenancePath = $null
|
||||
$freshManagedEnvironmentNames = @(
|
||||
"ENVIRONMENT",
|
||||
"ENGINE_URL",
|
||||
"ENGINE_MODE",
|
||||
"VIGNETTE_LIVE_CLIENT_PROVIDER",
|
||||
"AUTH_DEV_LOGIN_ENABLED",
|
||||
"AUTO_SEED_PERSONAS",
|
||||
"ALLOW_SEED_PERSONA_FALLBACK",
|
||||
"VIGNETTE_VOICE_POC_SAMPLE_TTS",
|
||||
"VIGNETTE_VOICE_STT_PROVIDER",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_URL",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_MODEL",
|
||||
"VIGNETTE_LOCAL_WHISPER_STT_LANGUAGE",
|
||||
"VIGNETTE_VOICE_TTS_PROVIDER",
|
||||
"VIGNETTE_MELOTTS_TTS_URL",
|
||||
"FRONTEND_BASE_URL",
|
||||
"CORS_ORIGINS",
|
||||
"FRONTEND_ORIGIN_MAP"
|
||||
)
|
||||
|
||||
trap {
|
||||
$caught = $_
|
||||
if (
|
||||
$RequireFreshPublicProvenance -and
|
||||
$freshMutationStarted -and
|
||||
-not $freshPromotionCommitted
|
||||
) {
|
||||
$rollbackResult = $null
|
||||
$rollbackSucceeded = $false
|
||||
$rollbackFailureType = "none"
|
||||
try {
|
||||
$rollbackResult = Restore-PriorPublicRuntime `
|
||||
-PriorApi $freshPriorApiIdentity `
|
||||
-PriorCloudflared $freshPriorCloudflaredIdentity `
|
||||
-PriorLocalVoiceContract $freshPriorLocalVoiceContract `
|
||||
-PriorPublicVoiceContract $freshPriorPublicVoiceContract `
|
||||
-EnvironmentSnapshot $freshEnvironmentSnapshot `
|
||||
-ConfigPath $CloudflaredConfig `
|
||||
-ApiPortValue $ApiPort `
|
||||
-HealthUrl $PublicHealthUrl `
|
||||
-VoiceHealthUrl $CanonicalPublicVoiceHealthUrl `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$rollbackSucceeded = $true
|
||||
} catch {
|
||||
$rollbackFailureType = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$failedEvidencePath = ""
|
||||
if ($resolvedRuntimeProvenancePath) {
|
||||
try {
|
||||
$failedEvidencePath = Write-FailedFreshPromotionEvidence `
|
||||
-OutputPath $resolvedRuntimeProvenancePath `
|
||||
-FailureStage $freshFailureStage `
|
||||
-RollbackSucceeded $rollbackSucceeded `
|
||||
-RollbackResult $rollbackResult `
|
||||
-SourceCommit $ExpectedSourceCommit `
|
||||
-SourceTree $ExpectedSourceTree
|
||||
} catch {
|
||||
$failedEvidencePath = "unavailable"
|
||||
}
|
||||
}
|
||||
|
||||
if ($rollbackSucceeded) {
|
||||
throw "Fresh public promotion failed at $freshFailureStage; the pinned prior API and tunnel were restored. failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
||||
}
|
||||
throw "Fresh public promotion failed at $freshFailureStage and prior-runtime rollback failed closed ($rollbackFailureType). failure_evidence=$failedEvidencePath cause=$($caught.Exception.Message)"
|
||||
}
|
||||
throw $caught
|
||||
}
|
||||
|
||||
if (!(Test-Path $Python)) {
|
||||
throw "Python 3.11 not found at $Python"
|
||||
}
|
||||
|
|
@ -705,6 +1208,64 @@ if ($RequireFreshPublicProvenance) {
|
|||
-ApiPortValue $ApiPort `
|
||||
-WebPortValue $WebPort `
|
||||
-RequireUnchanged
|
||||
|
||||
$priorApiProcesses = @(
|
||||
Get-UvicornProcessesByPort -AppImport "app.main:app" -Port $ApiPort
|
||||
)
|
||||
if ($priorApiProcesses.Count -ne 1) {
|
||||
throw "Fresh public promotion requires exactly one prior API process for transactional rollback"
|
||||
}
|
||||
$priorCloudflaredProcesses = @(
|
||||
Get-CloudflaredProcessesForConfig `
|
||||
-ConfigPath (Resolve-Path -LiteralPath $CloudflaredConfig).Path `
|
||||
-ExactPath
|
||||
)
|
||||
if ($priorCloudflaredProcesses.Count -ne 1) {
|
||||
throw "Fresh public promotion requires exactly one prior cloudflared process for transactional rollback"
|
||||
}
|
||||
$freshPriorApiIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorApiProcesses[0].ProcessId `
|
||||
-Role "prior api" `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$freshPriorCloudflaredIdentity = Wait-ProcessIdentity `
|
||||
-ProcessId $priorCloudflaredProcesses[0].ProcessId `
|
||||
-Role "prior cloudflared" `
|
||||
-TimeoutSec $ProcessStopTimeoutSeconds
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
$priorLocalVoiceHealth = Wait-JsonHealth `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$freshPriorLocalVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
||||
-Health $priorLocalVoiceHealth
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Fresh public promotion requires exact healthy voice sidecars as an unchanged precondition"
|
||||
}
|
||||
$null = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
$priorPublicVoiceHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicVoiceHealthUrl `
|
||||
-IsHealthy { param($health) Test-PriorVoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$freshPriorPublicVoiceContract = ConvertTo-SafeVoiceHealthContract `
|
||||
-Health $priorPublicVoiceHealth
|
||||
$freshEnvironmentSnapshot = Save-ManagedEnvironment `
|
||||
-Names $freshManagedEnvironmentNames
|
||||
}
|
||||
|
||||
# 재기동 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
||||
|
|
@ -732,7 +1293,9 @@ if ($SkipEngineRestart) {
|
|||
$rotateStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
foreach ($logFile in @($EngineOutLog, $EngineErrLog)) {
|
||||
if (Test-Path $logFile) {
|
||||
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak" -Force -ErrorAction SilentlyContinue
|
||||
# stable detached source의 provenance gate는 untracked 파일도 차단한다.
|
||||
# suffix를 .log로 유지해 회전 산출물이 기존 *.log ignore 경계 안에 머물게 한다.
|
||||
Move-Item -LiteralPath $logFile -Destination "$logFile.$rotateStamp.bak.log" -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -781,6 +1344,9 @@ $env:FRONTEND_ORIGIN_MAP = ConvertTo-CompactJson -Value ([ordered]@{
|
|||
# 프레임과 MeloTTS health metadata가 운영 계약과 정확히 일치할 때만 API를
|
||||
# 유지하거나 재시작한다. 잘못된 기존 리스너는 소유권을 추측해 종료하지 않는다.
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
throw "Fresh public promotion will not mutate local_whisper; restore the exact sidecar before retrying"
|
||||
}
|
||||
if (Test-PortListener -Port $WhisperPort) {
|
||||
throw "Port $WhisperPort is occupied but does not expose the exact local_whisper/$WhisperModel/$WhisperDevice protocol"
|
||||
}
|
||||
|
|
@ -795,6 +1361,9 @@ if (-not (Test-VoiceSidecarReady -Component "stt")) {
|
|||
}
|
||||
|
||||
if (-not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
throw "Fresh public promotion will not mutate MeloTTS; restore the exact sidecar before retrying"
|
||||
}
|
||||
if (Test-PortListener -Port $MeloTtsPort) {
|
||||
throw "Port $MeloTtsPort is occupied but does not expose the exact melotts/$MeloTtsModel health contract"
|
||||
}
|
||||
|
|
@ -828,6 +1397,10 @@ $apiLaunchIdentity = $null
|
|||
if ($apiControlPlaneReady -and -not $ForceApiRestart) {
|
||||
Write-Output "Production API and exact local voice stack already healthy; skipping API restart"
|
||||
} else {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "api_cutover"
|
||||
$freshMutationStarted = $true
|
||||
}
|
||||
$apiStoppedProcessIds = @(
|
||||
Stop-UvicornByPort `
|
||||
-AppImport "app.main:app" `
|
||||
|
|
@ -890,6 +1463,9 @@ if (-not (Test-VoiceApiReady -Health $voiceHealth)) {
|
|||
}
|
||||
|
||||
if (!$SkipWebRestart) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "web_preview"
|
||||
}
|
||||
Stop-NodeByPortHint -Port $WebPort
|
||||
|
||||
$build = Start-Process -FilePath "cmd.exe" `
|
||||
|
|
@ -916,6 +1492,9 @@ $cloudflaredProcess = $null
|
|||
$cloudflaredLaunchIdentity = $null
|
||||
$cloudflaredStoppedProcessIds = @()
|
||||
if (!$SkipCloudflaredRestart) {
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "cloudflared_cutover"
|
||||
}
|
||||
if (!(Test-Path $Cloudflared)) {
|
||||
throw "cloudflared not found at $Cloudflared"
|
||||
}
|
||||
|
|
@ -986,6 +1565,7 @@ if (!$SkipCloudflaredRestart) {
|
|||
}
|
||||
|
||||
if ($RequireFreshPublicProvenance) {
|
||||
$freshFailureStage = "identity_revalidation"
|
||||
if ($null -eq $apiLaunchIdentity -or $null -eq $cloudflaredLaunchIdentity) {
|
||||
throw "Fresh public promotion did not produce both API and cloudflared identities"
|
||||
}
|
||||
|
|
@ -1015,6 +1595,32 @@ if ($RequireFreshPublicProvenance) {
|
|||
if ($finalConfigSha256 -ne $ExpectedCloudflaredConfigSha256.ToLowerInvariant()) {
|
||||
throw "Pinned cloudflared config drifted before provenance receipt"
|
||||
}
|
||||
$freshFailureStage = "public_health_validation"
|
||||
$publicHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicHealthUrl `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -eq $true -and
|
||||
$health.engine -eq $true
|
||||
} `
|
||||
-TimeoutSec 60
|
||||
$publicVoiceHealth = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicVoiceHealthUrl `
|
||||
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
|
||||
-TimeoutSec 30
|
||||
$publicOpenApi = Wait-JsonHealth `
|
||||
-Uri $CanonicalPublicOpenApiUrl `
|
||||
-IsHealthy {
|
||||
param($document)
|
||||
Test-RequiredOpenApiPaths `
|
||||
-Document $document `
|
||||
-RequiredPaths $RequiredPublicApiPaths
|
||||
} `
|
||||
-TimeoutSec 30
|
||||
if (-not (Test-VoiceSidecarReady -Component "stt") -or -not (Test-VoiceSidecarReady -Component "tts")) {
|
||||
throw "Exact local voice sidecars changed before provenance receipt"
|
||||
}
|
||||
$psutilVersionArgs = @("-X", "utf8", "-c", "import importlib.metadata; print(importlib.metadata.version('psutil'))")
|
||||
$psutilVersion = (@(& $Python @psutilVersionArgs) -join [Environment]::NewLine).Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or -not $psutilVersion) {
|
||||
|
|
@ -1038,6 +1644,16 @@ if ($RequireFreshPublicProvenance) {
|
|||
path = (Resolve-Path -LiteralPath $CloudflaredConfig).Path
|
||||
sha256 = $finalConfigSha256
|
||||
}
|
||||
public_validation = [ordered]@{
|
||||
health_url = $CanonicalPublicHealthUrl
|
||||
health = $true
|
||||
voice_health_url = $CanonicalPublicVoiceHealthUrl
|
||||
voice_health = $true
|
||||
openapi_url = $CanonicalPublicOpenApiUrl
|
||||
required_openapi_paths = @($RequiredPublicApiPaths)
|
||||
openapi = $true
|
||||
local_voice_sidecars = $true
|
||||
}
|
||||
replacement = [ordered]@{
|
||||
api_stopped_pids = @($apiStoppedProcessIds)
|
||||
cloudflared_stopped_pids = @($cloudflaredStoppedProcessIds)
|
||||
|
|
@ -1068,6 +1684,7 @@ if ($RequireFreshPublicProvenance) {
|
|||
}
|
||||
}
|
||||
$provenanceJson = ConvertTo-Json -InputObject $provenance -Depth 8
|
||||
$freshFailureStage = "receipt_publish"
|
||||
try {
|
||||
Write-Utf8TextAtomically `
|
||||
-OutputPath $resolvedRuntimeProvenancePath `
|
||||
|
|
@ -1077,6 +1694,7 @@ if ($RequireFreshPublicProvenance) {
|
|||
# 승격 성공으로 간주할 수 없다. 기존 receipt는 보존되고 호출은 non-zero로 끝난다.
|
||||
throw "Fresh public promotion failed closed after runtime replacement: no atomic passed receipt was published. Re-run the pinned promotion after fixing the receipt destination. $($_.Exception.Message)"
|
||||
}
|
||||
$freshPromotionCommitted = $true
|
||||
Write-Output "Fresh public provenance: $resolvedRuntimeProvenancePath"
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue