아바타 저장소 승격 계약을 완성
This commit is contained in:
Yun Chan
parent
ac9b702688
commit
ccdcfcd2f5
36 changed files with 14734 additions and 222 deletions
|
|
@ -13,6 +13,12 @@
|
|||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[0-9a-fA-F]{64}$")]
|
||||
[string]$ExpectedStartScriptSha256,
|
||||
[ValidatePattern("^$|^[0-9a-fA-F]{64}$")]
|
||||
[string]$ExpectedPythonSha256 = "",
|
||||
[ValidatePattern("^$|^[0-9a-fA-F]{64}$")]
|
||||
[string]$ExpectedCloudflaredSha256 = "",
|
||||
[ValidatePattern("^$|^[0-9a-fA-F]{64}$")]
|
||||
[string]$ExpectedCloudflaredConfigSha256 = "",
|
||||
[int]$ApiPort = 8001,
|
||||
[int]$WebPort = 5174,
|
||||
[int]$EnginePort = 9099,
|
||||
|
|
@ -23,6 +29,15 @@
|
|||
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
|
||||
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
||||
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$UserUploadDir,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$UserUploadManifestPath,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[0-9a-f]{64}$")]
|
||||
[string]$ExpectedUserUploadManifestSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$UserUploadWriteFreezePath,
|
||||
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
||||
[string[]]$AdditionalPublicHealthUrls = @(),
|
||||
# 게이트웨이가 shared secret으로 떠 있으면 /ready는 인증이 필요하다(/health만 면제).
|
||||
|
|
@ -41,6 +56,10 @@ $resolvedSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|||
$expectedWatchdogPath = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1"
|
||||
$startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1"
|
||||
$voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py"
|
||||
$uploadRootContract = Join-Path $resolvedSourceRoot "scripts\public-runtime-upload-root.ps1"
|
||||
$uploadRootProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-runtime-upload-root.py"
|
||||
$uploadManifestProbe = Join-Path $resolvedSourceRoot "scripts\validate-public-runtime-upload-manifest.py"
|
||||
$databaseIdentityHelper = Join-Path $resolvedSourceRoot "scripts\public_runtime_database_identity.py"
|
||||
|
||||
function Invoke-GitText {
|
||||
param([string[]]$Arguments)
|
||||
|
|
@ -62,6 +81,11 @@ function Assert-StableSourceProvenance {
|
|||
if (-not (Test-Path -LiteralPath $voiceSidecarProbe -PathType Leaf)) {
|
||||
throw "Pinned voice sidecar probe not found at $voiceSidecarProbe"
|
||||
}
|
||||
foreach ($uploadContractFile in @($uploadRootContract, $uploadRootProbe, $uploadManifestProbe, $databaseIdentityHelper)) {
|
||||
if (-not (Test-Path -LiteralPath $uploadContractFile -PathType Leaf)) {
|
||||
throw "Pinned upload-root contract file not found at $uploadContractFile"
|
||||
}
|
||||
}
|
||||
|
||||
$runningWatchdogPath = (Resolve-Path -LiteralPath $PSCommandPath).Path
|
||||
if (-not [string]::Equals(
|
||||
|
|
@ -108,7 +132,13 @@ function Assert-StableSourceProvenance {
|
|||
foreach ($relativePath in @(
|
||||
"scripts/watch-public-runtime.ps1",
|
||||
"scripts/start-public-runtime.ps1",
|
||||
"scripts/probe-public-voice-sidecars.py"
|
||||
"scripts/probe-public-voice-sidecars.py",
|
||||
"scripts/public-runtime-upload-root.ps1",
|
||||
"scripts/probe-public-runtime-upload-root.py",
|
||||
"scripts/validate-public-runtime-upload-manifest.py",
|
||||
"scripts/public_runtime_database_identity.py",
|
||||
"apps/api/app/upload_storage.py",
|
||||
"apps/api/app/upload_runtime.py"
|
||||
)) {
|
||||
Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null
|
||||
}
|
||||
|
|
@ -121,11 +151,45 @@ function Assert-StableSourceProvenance {
|
|||
if ($actualStartScriptSha256 -ne $ExpectedStartScriptSha256.ToLowerInvariant()) {
|
||||
throw "Pinned start script SHA256 drift"
|
||||
}
|
||||
foreach ($pin in @(
|
||||
[pscustomobject]@{ Path = $Python; Expected = $ExpectedPythonSha256; Role = "Python" },
|
||||
[pscustomobject]@{ Path = $Cloudflared; Expected = $ExpectedCloudflaredSha256; Role = "cloudflared" },
|
||||
[pscustomobject]@{ Path = $CloudflaredConfig; Expected = $ExpectedCloudflaredConfigSha256; Role = "cloudflared config" }
|
||||
)) {
|
||||
if ([string]::IsNullOrWhiteSpace([string]$pin.Expected)) {
|
||||
continue
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $pin.Path -PathType Leaf)) {
|
||||
throw "Pinned $($pin.Role) is unavailable"
|
||||
}
|
||||
$actual = (Get-FileHash -LiteralPath $pin.Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actual -cne ([string]$pin.Expected).ToLowerInvariant()) {
|
||||
throw "Pinned $($pin.Role) SHA256 drift"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# health probe, failcount 기록, 프로세스 재기동보다 먼저 source provenance를 닫는다.
|
||||
# 검증 실패는 운영 프로세스를 그대로 보존한 채 non-zero로 끝난다.
|
||||
Assert-StableSourceProvenance
|
||||
. $uploadRootContract
|
||||
$uploadRootResolveArgs = @{
|
||||
SourceRoot = $resolvedSourceRoot
|
||||
UploadRoot = $UserUploadDir
|
||||
}
|
||||
if (-not $CheckOnly) {
|
||||
$uploadRootResolveArgs["ProbeWritable"] = $true
|
||||
}
|
||||
$resolvedUserUploadDir = Resolve-PublicRuntimeUploadRoot @uploadRootResolveArgs
|
||||
$resolvedUserUploadManifestPath = Resolve-PublicRuntimePrivateStatePath `
|
||||
-SourceRoot $resolvedSourceRoot `
|
||||
-UploadRoot $resolvedUserUploadDir `
|
||||
-StatePath $UserUploadManifestPath `
|
||||
-RequireFile
|
||||
$resolvedUserUploadWriteFreezePath = Resolve-PublicRuntimePrivateStatePath `
|
||||
-SourceRoot $resolvedSourceRoot `
|
||||
-UploadRoot $resolvedUserUploadDir `
|
||||
-StatePath $UserUploadWriteFreezePath
|
||||
|
||||
if (!$LogPath) {
|
||||
$LogPath = Join-Path $resolvedSourceRoot "public-runtime-watchdog.log"
|
||||
|
|
@ -337,7 +401,11 @@ function Test-PublicRuntimeHardDown {
|
|||
($FailedNames -contains "web-preview") -and
|
||||
($FailedNames -contains "voice-sidecars")
|
||||
)
|
||||
return ($FailedNames -contains "cloudflared") -or $allLocalSurfacesDown
|
||||
return (
|
||||
($FailedNames -contains "cloudflared") -or
|
||||
($FailedNames -contains "api-upload-root") -or
|
||||
$allLocalSurfacesDown
|
||||
)
|
||||
}
|
||||
|
||||
# engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
|
||||
|
|
@ -345,6 +413,57 @@ function Test-PublicRuntimeHardDown {
|
|||
# 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0).
|
||||
# /ready는 게이트웨이 readiness 캐시(ENGINE_READY_TTL_SECONDS)를 그대로 쓰므로
|
||||
# 매 주기 LLM 호출로 이어지지 않는다. 콜드 스폰 여유로 타임아웃만 넉넉히 준다.
|
||||
$uploadManifestCheck = Test-PublicRuntimeUploadManifest `
|
||||
-PythonPath $Python `
|
||||
-ProbePath $uploadManifestProbe `
|
||||
-UploadRoot $resolvedUserUploadDir `
|
||||
-ManifestPath $resolvedUserUploadManifestPath `
|
||||
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
||||
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath
|
||||
$expectedDatabaseTargetSha256 = "0000000000000000000000000000000000000000000000000000000000000000"
|
||||
if ($uploadManifestCheck.Ok) {
|
||||
$candidateDatabaseTargetSha256 = [string]$uploadManifestCheck.Payload.database_target_sha256
|
||||
if ($candidateDatabaseTargetSha256 -match "^[0-9a-f]{64}$") {
|
||||
$expectedDatabaseTargetSha256 = $candidateDatabaseTargetSha256
|
||||
}
|
||||
}
|
||||
if (Test-Path -LiteralPath $resolvedUserUploadWriteFreezePath -PathType Leaf) {
|
||||
$promotionHealth = $null
|
||||
try {
|
||||
$promotionHealth = Invoke-RestMethod `
|
||||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-TimeoutSec 5
|
||||
} catch {
|
||||
$promotionHealth = $null
|
||||
}
|
||||
$promotionFreeze = $null
|
||||
if ($null -ne $promotionHealth) {
|
||||
$promotionFreeze = $promotionHealth.upload_write_freeze
|
||||
}
|
||||
if (
|
||||
$uploadManifestCheck.Ok -and
|
||||
$null -ne $promotionFreeze -and
|
||||
$promotionFreeze.capable -eq $true -and
|
||||
$promotionFreeze.active -eq $true -and
|
||||
$promotionFreeze.valid -eq $true -and
|
||||
[int]$promotionFreeze.in_flight -eq 0 -and
|
||||
[string]$promotionFreeze.token_sha256 -ceq
|
||||
[string]$uploadManifestCheck.Payload.write_freeze_token_sha256
|
||||
) {
|
||||
if ($CheckOnly) {
|
||||
Write-Output "promotion-in-progress: valid drained upload freeze is active; runtime mutation skipped"
|
||||
} else {
|
||||
Write-WatchdogLog "promotion-in-progress: valid drained upload freeze is active; runtime mutation skipped"
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
if ($CheckOnly) {
|
||||
Write-Output "unhealthy: upload freeze sentinel exists without exact active/drained API proof; runtime mutation refused"
|
||||
} else {
|
||||
Write-WatchdogLog "ERROR: upload freeze sentinel exists without exact active/drained API proof; refusing runtime mutation"
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
$checks = @(
|
||||
(Test-JsonHealth `
|
||||
-Name "engine" `
|
||||
|
|
@ -355,8 +474,27 @@ $checks = @(
|
|||
(Test-JsonHealth `
|
||||
-Name "api" `
|
||||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -and
|
||||
$health.engine -and
|
||||
$health.upload_write_freeze.capable -eq $true -and
|
||||
$health.upload_write_freeze.active -eq $false -and
|
||||
$health.upload_write_freeze.valid -eq $true
|
||||
} `
|
||||
-TimeoutSec 60),
|
||||
$uploadManifestCheck,
|
||||
(Test-PublicRuntimeApiUploadRoot `
|
||||
-PythonPath $Python `
|
||||
-ProbePath $uploadRootProbe `
|
||||
-ExpectedUploadRoot $resolvedUserUploadDir `
|
||||
-ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") `
|
||||
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
||||
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
||||
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
||||
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
||||
-ApiPort $ApiPort),
|
||||
(Test-JsonHealth `
|
||||
-Name "voice-api" `
|
||||
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
|
||||
|
|
@ -389,7 +527,7 @@ if (!$SkipPublicHealth) {
|
|||
$failed = @($checks | Where-Object { -not $_.Ok })
|
||||
if ($CheckOnly) {
|
||||
if ($failed.Count -eq 0) {
|
||||
Write-Output "healthy: $($checks.Name -join ', ')"
|
||||
Write-Output "healthy: $($checks.Name -join ', '); user_upload_root=$resolvedUserUploadDir"
|
||||
exit 0
|
||||
}
|
||||
Write-Output "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
|
||||
|
|
@ -418,6 +556,10 @@ if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown) {
|
|||
if ($hardDown -and $failCount -lt $FailuresBeforeRestart) {
|
||||
Write-WatchdogLog "immediate restart: public runtime hard-down detected"
|
||||
}
|
||||
if ($failedNames -contains "api-upload-manifest") {
|
||||
Write-WatchdogLog "ERROR: immutable upload migration receipt or current DB inventory is invalid; refusing runtime mutation"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당).
|
||||
# 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에
|
||||
|
|
@ -442,6 +584,10 @@ $startArgs = @{
|
|||
Python = $Python
|
||||
Cloudflared = $Cloudflared
|
||||
CloudflaredConfig = $CloudflaredConfig
|
||||
UserUploadDir = $resolvedUserUploadDir
|
||||
UserUploadManifestPath = $resolvedUserUploadManifestPath
|
||||
ExpectedUserUploadManifestSha256 = $ExpectedUserUploadManifestSha256
|
||||
UserUploadWriteFreezePath = $resolvedUserUploadWriteFreezePath
|
||||
}
|
||||
if (($checks | Where-Object { $_.Name -eq "web-preview" }).Ok) {
|
||||
$startArgs["SkipWebRestart"] = $true
|
||||
|
|
@ -463,11 +609,46 @@ try {
|
|||
$apiAfter = Test-JsonHealth `
|
||||
-Name "api" `
|
||||
-Uri "http://127.0.0.1:$ApiPort/health" `
|
||||
-IsHealthy { param($health) $health.environment -eq "prod" -and $health.db -and $health.engine } `
|
||||
-IsHealthy {
|
||||
param($health)
|
||||
$health.environment -eq "prod" -and
|
||||
$health.db -and
|
||||
$health.engine -and
|
||||
$health.upload_write_freeze.capable -eq $true -and
|
||||
$health.upload_write_freeze.active -eq $false -and
|
||||
$health.upload_write_freeze.valid -eq $true
|
||||
} `
|
||||
-TimeoutSec 90
|
||||
if (!$apiAfter.Ok) {
|
||||
throw "Public API still unhealthy after restart: $($apiAfter.Detail)"
|
||||
}
|
||||
$uploadManifestAfter = Test-PublicRuntimeUploadManifest `
|
||||
-PythonPath $Python `
|
||||
-ProbePath $uploadManifestProbe `
|
||||
-UploadRoot $resolvedUserUploadDir `
|
||||
-ManifestPath $resolvedUserUploadManifestPath `
|
||||
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
||||
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath
|
||||
if (-not $uploadManifestAfter.Ok) {
|
||||
throw "Public upload migration receipt or current DB inventory is invalid after restart"
|
||||
}
|
||||
$expectedDatabaseTargetSha256 = [string]$uploadManifestAfter.Payload.database_target_sha256
|
||||
if ($expectedDatabaseTargetSha256 -notmatch "^[0-9a-f]{64}$") {
|
||||
throw "Public upload inventory proof did not return a valid database target identity after restart"
|
||||
}
|
||||
$apiUploadRootAfter = Test-PublicRuntimeApiUploadRoot `
|
||||
-PythonPath $Python `
|
||||
-ProbePath $uploadRootProbe `
|
||||
-ExpectedUploadRoot $resolvedUserUploadDir `
|
||||
-ExpectedApiCwd (Join-Path $resolvedSourceRoot "apps\api") `
|
||||
-ExpectedManifestPath $resolvedUserUploadManifestPath `
|
||||
-ExpectedManifestSha256 $ExpectedUserUploadManifestSha256 `
|
||||
-ExpectedWriteFreezePath $resolvedUserUploadWriteFreezePath `
|
||||
-ExpectedDatabaseTargetSha256 $expectedDatabaseTargetSha256 `
|
||||
-ApiPort $ApiPort
|
||||
if (-not $apiUploadRootAfter.Ok) {
|
||||
throw "Public API still uses the wrong upload root after restart: $($apiUploadRootAfter.Detail)"
|
||||
}
|
||||
|
||||
$voiceApiAfter = Test-JsonHealth `
|
||||
-Name "voice-api" `
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue