2670 lines
95 KiB
PowerShell
2670 lines
95 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$StableSourceRoot,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{40}$")]
|
|
[string]$ExpectedSourceCommit,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{40}$")]
|
|
[string]$ExpectedSourceTree,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{40}$")]
|
|
[string]$ExpectedLegacySourceCommit,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{40}$")]
|
|
[string]$ExpectedLegacySourceTree,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$PythonPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{64}$")]
|
|
[string]$ExpectedPythonSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$CloudflaredPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{64}$")]
|
|
[string]$ExpectedCloudflaredSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$CloudflaredConfigPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{64}$")]
|
|
[string]$ExpectedCloudflaredConfigSha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$UserUploadDir,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$ManifestStateDir,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$UserUploadWriteFreezePath,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateRange(0, 2147483647)]
|
|
[int]$ExpectedReferenceCount,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateRange(1, 2147483647)]
|
|
[int]$ExpectedPreservedObjectCount,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateRange(1, 9223372036854775807)]
|
|
[long]$ExpectedPreservedTotalSizeBytes,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^[0-9a-f]{64}$")]
|
|
[string]$ExpectedPreservedInventorySha256,
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[string[]]$AllowedLegacySourceUploadDir,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$CutoverReceiptPath,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$TaskRecoveryReceiptPath,
|
|
[int]$ApiPort = 8001,
|
|
[int]$WhisperPort = 9882,
|
|
[int]$MeloTtsPort = 9883,
|
|
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
|
[string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock",
|
|
[ValidateRange(10, 300)]
|
|
[int]$HealthTimeoutSeconds = 60,
|
|
[ValidateRange(5, 120)]
|
|
[int]$ProcessStopTimeoutSeconds = 30,
|
|
[string[]]$CoordinatedTaskNames = @(
|
|
"VignettePublicRuntimeWatchdog",
|
|
"VignettePublicRuntime"
|
|
)
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)
|
|
$OutputEncoding = [System.Text.UTF8Encoding]::new($false)
|
|
|
|
function Get-Utf8Sha256 {
|
|
param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value)
|
|
|
|
$bytes = [System.Text.UTF8Encoding]::new($false).GetBytes($Value)
|
|
$hasher = [System.Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
return ([BitConverter]::ToString($hasher.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$hasher.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-RequiredPrivacySafeCount {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][string]$Name,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$property = $Payload.PSObject.Properties[$Name]
|
|
if ($null -eq $property -or $null -eq $property.Value) {
|
|
throw "$Role did not return privacy-safe decode count $Name"
|
|
}
|
|
$typeCode = [System.Type]::GetTypeCode($property.Value.GetType())
|
|
if (@(
|
|
[System.TypeCode]::Byte,
|
|
[System.TypeCode]::SByte,
|
|
[System.TypeCode]::Int16,
|
|
[System.TypeCode]::UInt16,
|
|
[System.TypeCode]::Int32,
|
|
[System.TypeCode]::UInt32,
|
|
[System.TypeCode]::Int64,
|
|
[System.TypeCode]::UInt64
|
|
) -notcontains $typeCode) {
|
|
throw "$Role returned non-integer privacy-safe decode count $Name"
|
|
}
|
|
$value = [decimal]$property.Value
|
|
if ($value -lt 0 -or $value -gt [int]::MaxValue) {
|
|
throw "$Role returned out-of-range privacy-safe decode count $Name"
|
|
}
|
|
return [int]$value
|
|
}
|
|
|
|
function Get-PreservedDecodeCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$PreservedObjectCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$validCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "preserved_decode_valid_count" `
|
|
-Role $Role
|
|
$invalidCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "preserved_decode_invalid_count" `
|
|
-Role $Role
|
|
if (($validCount + $invalidCount) -ne $PreservedObjectCount) {
|
|
throw "$Role preserved decode counts do not cover the exact object inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ValidCount = $validCount
|
|
InvalidCount = $invalidCount
|
|
}
|
|
}
|
|
|
|
function Get-RequiredDecodeInvalidCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$RequiredObjectCount,
|
|
[Parameter(Mandatory = $true)][int]$RequiredReferenceCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$objectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "required_decode_invalid_object_count" `
|
|
-Role $Role
|
|
$referenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "required_decode_invalid_reference_count" `
|
|
-Role $Role
|
|
if (
|
|
$objectCount -gt $RequiredObjectCount -or
|
|
$referenceCount -gt $RequiredReferenceCount
|
|
) {
|
|
throw "$Role required decode-invalid counts exceed the bound DB inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ObjectCount = $objectCount
|
|
ReferenceCount = $referenceCount
|
|
}
|
|
}
|
|
|
|
function Get-CurrentDecodeInvalidCountProof {
|
|
param(
|
|
[Parameter(Mandatory = $true)][object]$Payload,
|
|
[Parameter(Mandatory = $true)][int]$CurrentObjectCount,
|
|
[Parameter(Mandatory = $true)][int]$CurrentReferenceCount,
|
|
[Parameter(Mandatory = $true)][string]$Role
|
|
)
|
|
|
|
$objectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "current_decode_invalid_object_count" `
|
|
-Role $Role
|
|
$referenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $Payload `
|
|
-Name "current_decode_invalid_reference_count" `
|
|
-Role $Role
|
|
if (
|
|
$objectCount -gt $CurrentObjectCount -or
|
|
$referenceCount -gt $CurrentReferenceCount
|
|
) {
|
|
throw "$Role current decode-invalid counts exceed the bound DB inventory"
|
|
}
|
|
return [pscustomobject]@{
|
|
ObjectCount = $objectCount
|
|
ReferenceCount = $referenceCount
|
|
}
|
|
}
|
|
|
|
function Get-CanonicalPathSha256 {
|
|
param([Parameter(Mandatory = $true)][string]$Path)
|
|
|
|
$resolved = (Resolve-Path -LiteralPath $Path).Path
|
|
$identity = $resolved.Replace("\", "/").ToLowerInvariant()
|
|
return Get-Utf8Sha256 -Value $identity
|
|
}
|
|
|
|
function Invoke-PinnedGitText {
|
|
param(
|
|
[string]$Root,
|
|
[string[]]$Arguments
|
|
)
|
|
|
|
$output = @(& git.exe -C $Root @Arguments)
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Stable source Git proof failed"
|
|
}
|
|
return ((@($output) -join [Environment]::NewLine).Trim())
|
|
}
|
|
|
|
function Assert-BootstrapSourceProvenance {
|
|
param([string]$Root)
|
|
|
|
$gitRoot = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--show-toplevel")
|
|
if (-not [string]::Equals(
|
|
(Resolve-Path -LiteralPath $gitRoot).Path,
|
|
$Root,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Bootstrap source is not its Git toplevel"
|
|
}
|
|
$symbolicHead = @(& git.exe -C $Root symbolic-ref --quiet HEAD)
|
|
$symbolicExit = $LASTEXITCODE
|
|
if ($symbolicExit -eq 0 -or $symbolicHead.Count -gt 0) {
|
|
throw "Bootstrap source must use detached HEAD"
|
|
}
|
|
if ($symbolicExit -ne 1) {
|
|
throw "Bootstrap detached HEAD proof failed"
|
|
}
|
|
$actualCommit = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--verify", "HEAD")
|
|
$actualTree = Invoke-PinnedGitText -Root $Root -Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($actualCommit -cne $ExpectedSourceCommit -or $actualTree -cne $ExpectedSourceTree) {
|
|
throw "Bootstrap source commit or tree drift"
|
|
}
|
|
$dirty = Invoke-PinnedGitText -Root $Root -Arguments @(
|
|
"status", "--porcelain=v1", "--untracked-files=normal"
|
|
)
|
|
if ($dirty) {
|
|
throw "Bootstrap source must be clean"
|
|
}
|
|
foreach ($relativePath in @(
|
|
"scripts/bootstrap-legacy-public-runtime-upload-root.ps1",
|
|
"scripts/initialize-public-runtime-upload-root.ps1",
|
|
"scripts/initialize-public-runtime-upload-root.py",
|
|
"scripts/validate-public-runtime-offline-quiescence.py",
|
|
"scripts/start-public-runtime.ps1",
|
|
"scripts/public-runtime-task-maintenance.ps1",
|
|
"scripts/public-runtime-task-definition-cutover.ps1",
|
|
"scripts/register-boot-task.ps1",
|
|
"scripts/install-public-runtime-task.ps1",
|
|
"scripts/public-runtime-upload-root.ps1",
|
|
"scripts/probe-public-runtime-upload-root.py",
|
|
"scripts/validate-public-runtime-upload-manifest.py",
|
|
"scripts/public_runtime_database_identity.py",
|
|
"scripts/probe-public-runtime-database-identity.py",
|
|
"apps/api/app/upload_storage.py",
|
|
"apps/api/app/upload_runtime.py"
|
|
)) {
|
|
$null = Invoke-PinnedGitText `
|
|
-Root $Root `
|
|
-Arguments @("ls-files", "--error-unmatch", "--", $relativePath)
|
|
}
|
|
}
|
|
|
|
function Assert-LegacyApiSourceProvenance {
|
|
param([System.Collections.IDictionary]$ApiIdentity)
|
|
|
|
$cwd = (Resolve-Path -LiteralPath ([string]$ApiIdentity.cwd)).Path
|
|
$legacyGitRoot = Invoke-PinnedGitText `
|
|
-Root $cwd `
|
|
-Arguments @("rev-parse", "--show-toplevel")
|
|
$resolvedLegacyGitRoot = (Resolve-Path -LiteralPath $legacyGitRoot).Path
|
|
$symbolicHead = @(& git.exe -C $resolvedLegacyGitRoot symbolic-ref --quiet HEAD)
|
|
$symbolicExit = $LASTEXITCODE
|
|
if ($symbolicExit -eq 0 -or $symbolicHead.Count -gt 0 -or $symbolicExit -ne 1) {
|
|
throw "Legacy API source must use detached HEAD"
|
|
}
|
|
$commit = Invoke-PinnedGitText `
|
|
-Root $resolvedLegacyGitRoot `
|
|
-Arguments @("rev-parse", "--verify", "HEAD")
|
|
$tree = Invoke-PinnedGitText `
|
|
-Root $resolvedLegacyGitRoot `
|
|
-Arguments @("rev-parse", "--verify", "HEAD^{tree}")
|
|
if ($commit -cne $ExpectedLegacySourceCommit -or $tree -cne $ExpectedLegacySourceTree) {
|
|
throw "Legacy API source commit or tree drift"
|
|
}
|
|
$dirty = Invoke-PinnedGitText `
|
|
-Root $resolvedLegacyGitRoot `
|
|
-Arguments @("status", "--porcelain=v1", "--untracked-files=normal")
|
|
if ($dirty) {
|
|
throw "Legacy API source must be clean"
|
|
}
|
|
$expectedApiCwd = Join-Path $resolvedLegacyGitRoot "apps\api"
|
|
if (-not [string]::Equals(
|
|
$cwd,
|
|
[System.IO.Path]::GetFullPath($expectedApiCwd),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Legacy API cwd is not bound to its Git root"
|
|
}
|
|
return [ordered]@{
|
|
commit = $commit
|
|
tree = $tree
|
|
}
|
|
}
|
|
|
|
function Assert-FileSha256 {
|
|
param(
|
|
[string]$Path,
|
|
[string]$ExpectedSha256,
|
|
[string]$Role
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Role is unavailable"
|
|
}
|
|
$actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actual -cne $ExpectedSha256) {
|
|
throw "$Role SHA256 drift"
|
|
}
|
|
}
|
|
|
|
function Resolve-PrivateBootstrapPath {
|
|
param(
|
|
[string]$Path,
|
|
[string]$StateRoot,
|
|
[switch]$RequireFile
|
|
)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($Path)) {
|
|
throw "Bootstrap private state path must be absolute"
|
|
}
|
|
$full = Get-PublicRuntimeCanonicalPath -Path $Path
|
|
$parent = [System.IO.Path]::GetDirectoryName($full)
|
|
if (-not [string]::Equals(
|
|
$parent,
|
|
$StateRoot,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Bootstrap receipt and freeze files must be direct private-state children"
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $full
|
|
if ($RequireFile) {
|
|
if (-not (Test-Path -LiteralPath $full -PathType Leaf)) {
|
|
throw "Bootstrap private state file is unavailable"
|
|
}
|
|
return (Resolve-Path -LiteralPath $full).Path
|
|
}
|
|
return $full
|
|
}
|
|
|
|
function Resolve-PrivateBootstrapStateDirectory {
|
|
param(
|
|
[string]$Path,
|
|
[string]$SourceRoot,
|
|
[string]$UploadRoot
|
|
)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($Path)) {
|
|
throw "Bootstrap private state directory must be absolute"
|
|
}
|
|
$full = Get-PublicRuntimeCanonicalPath -Path $Path
|
|
foreach ($boundary in @($SourceRoot, $UploadRoot)) {
|
|
if (
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $full -Parent $boundary) -or
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $boundary -Parent $full)
|
|
) {
|
|
throw "Bootstrap private state must be disjoint from source and public roots"
|
|
}
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $full
|
|
if (-not (Test-Path -LiteralPath $full)) {
|
|
[System.IO.Directory]::CreateDirectory($full) | Out-Null
|
|
}
|
|
if (-not (Test-Path -LiteralPath $full -PathType Container)) {
|
|
throw "Bootstrap private state directory is unavailable"
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $full
|
|
return (Resolve-Path -LiteralPath $full).Path
|
|
}
|
|
|
|
function Get-ExactLoopbackListenerPid {
|
|
param([int]$Port)
|
|
|
|
$listeners = @(
|
|
Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue
|
|
)
|
|
if ($listeners.Count -eq 0) {
|
|
throw "Expected API listener is absent"
|
|
}
|
|
foreach ($listener in $listeners) {
|
|
if ([string]$listener.LocalAddress -cne "127.0.0.1") {
|
|
throw "API port has a non-loopback listener"
|
|
}
|
|
}
|
|
$processIds = @($listeners | ForEach-Object { [int]$_.OwningProcess } | Sort-Object -Unique)
|
|
if ($processIds.Count -ne 1) {
|
|
throw "API listener does not have one exact owner"
|
|
}
|
|
return [int]$processIds[0]
|
|
}
|
|
|
|
function Assert-LoopbackListenerAbsent {
|
|
param(
|
|
[int]$Port,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$listeners = @(
|
|
Get-NetTCPConnection `
|
|
-State Listen `
|
|
-LocalPort $Port `
|
|
-ErrorAction SilentlyContinue
|
|
)
|
|
if ($listeners.Count -eq 0) {
|
|
return
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "API listener absence was not proven"
|
|
}
|
|
|
|
function Wait-ProcessIdentity {
|
|
param(
|
|
[int]$ProcessId,
|
|
[string]$Role,
|
|
[string]$ExpectedCwd = "",
|
|
[int]$TimeoutSec = 15
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$process = Get-CimInstance Win32_Process `
|
|
-Filter "ProcessId = $ProcessId" `
|
|
-ErrorAction SilentlyContinue
|
|
if ($null -ne $process -and $process.ExecutablePath -and $process.CommandLine) {
|
|
$probeArgs = @(
|
|
"-X", "utf8", "-c",
|
|
"import hashlib,json,psutil,sys; from datetime import UTC,datetime; p=psutil.Process(int(sys.argv[1])); argv=p.cmdline(); print(p.cwd()); print(datetime.fromtimestamp(p.create_time(), UTC).isoformat().replace('+00:00','Z')); print(hashlib.sha256(chr(0).join(argv).encode('utf-8',errors='strict')).hexdigest()); print(json.dumps(argv[1:],ensure_ascii=True,separators=(',',':'))); print(json.dumps(p.environ(),ensure_ascii=True,separators=(',',':')))",
|
|
"$ProcessId"
|
|
)
|
|
$probe = @(& $resolvedPythonPath @probeArgs)
|
|
if ($LASTEXITCODE -ne 0 -or $probe.Count -ne 5) {
|
|
throw "$Role identity probe failed"
|
|
}
|
|
$cwd = $probe[0].Trim()
|
|
$startedAtUtc = $probe[1].Trim()
|
|
$commandLineSha256 = $probe[2].Trim().ToLowerInvariant()
|
|
$argumentList = @($probe[3] | ConvertFrom-Json)
|
|
$environmentObject = $probe[4] | ConvertFrom-Json
|
|
$environment = [ordered]@{}
|
|
foreach ($property in $environmentObject.PSObject.Properties) {
|
|
$environment[$property.Name] = [string]$property.Value
|
|
}
|
|
if (-not $cwd -or $startedAtUtc -notmatch "Z$" -or $commandLineSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "$Role identity is incomplete"
|
|
}
|
|
if ($ExpectedCwd -and -not [string]::Equals(
|
|
[System.IO.Path]::GetFullPath($cwd),
|
|
[System.IO.Path]::GetFullPath($ExpectedCwd),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "$Role working directory drift"
|
|
}
|
|
return [ordered]@{
|
|
pid = [int]$process.ProcessId
|
|
started_at_utc = $startedAtUtc
|
|
executable_path = [string]$process.ExecutablePath
|
|
executable_sha256 = (Get-FileHash -LiteralPath $process.ExecutablePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
command_line_sha256 = $commandLineSha256
|
|
argument_list = $argumentList
|
|
environment = $environment
|
|
cwd = $cwd
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "$Role identity timed out"
|
|
}
|
|
|
|
function Test-ArgumentPair {
|
|
param(
|
|
[object[]]$Arguments,
|
|
[string]$Name,
|
|
[string]$Value
|
|
)
|
|
|
|
$nameCount = 0
|
|
$matchingValueCount = 0
|
|
for ($index = 0; $index -lt $Arguments.Count; $index++) {
|
|
if ([string]$Arguments[$index] -cne $Name) {
|
|
continue
|
|
}
|
|
$nameCount++
|
|
if (
|
|
$index -lt ($Arguments.Count - 1) -and
|
|
[string]::Equals(
|
|
[string]$Arguments[$index + 1],
|
|
$Value,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)
|
|
) {
|
|
$matchingValueCount++
|
|
}
|
|
}
|
|
return $nameCount -eq 1 -and $matchingValueCount -eq 1
|
|
}
|
|
|
|
function Assert-ApiIdentityContract {
|
|
param([System.Collections.IDictionary]$Identity)
|
|
|
|
$arguments = @($Identity.argument_list)
|
|
if (
|
|
$arguments -notcontains "uvicorn" -or
|
|
$arguments -notcontains "app.main:app" -or
|
|
-not (Test-ArgumentPair -Arguments $arguments -Name "--port" -Value "$ApiPort") -or
|
|
-not (Test-ArgumentPair -Arguments $arguments -Name "--workers" -Value "1")
|
|
) {
|
|
throw "API listener command contract drift"
|
|
}
|
|
}
|
|
|
|
function Get-TunnelIdentitiesForConfig {
|
|
param([string]$ConfigPath)
|
|
|
|
$candidateProcesses = @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0
|
|
}
|
|
)
|
|
$identities = New-Object System.Collections.Generic.List[object]
|
|
foreach ($candidate in $candidateProcesses) {
|
|
$identity = Wait-ProcessIdentity `
|
|
-ProcessId ([int]$candidate.ProcessId) `
|
|
-Role "cloudflared" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$arguments = @($identity.argument_list)
|
|
if (
|
|
(Test-ArgumentPair -Arguments $arguments -Name "--config" -Value $ConfigPath) -and
|
|
$arguments -contains "tunnel" -and
|
|
$arguments -contains "run"
|
|
) {
|
|
$identities.Add($identity)
|
|
}
|
|
}
|
|
return @($identities)
|
|
}
|
|
|
|
function Get-ExactTunnelIdentity {
|
|
param([string]$ConfigPath)
|
|
|
|
$identities = @(
|
|
Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath
|
|
)
|
|
if ($identities.Count -ne 1) {
|
|
throw "Expected exactly one tunnel identity"
|
|
}
|
|
return $identities[0]
|
|
}
|
|
|
|
function Assert-TunnelAbsent {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$matches = @(
|
|
Get-CimInstance Win32_Process |
|
|
Where-Object {
|
|
$_.Name -eq "cloudflared.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine.IndexOf($ConfigPath, [System.StringComparison]::OrdinalIgnoreCase) -ge 0
|
|
}
|
|
)
|
|
if ($matches.Count -eq 0) {
|
|
return
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Tunnel absence was not proven"
|
|
}
|
|
|
|
function Assert-CurrentTunnelIdentity {
|
|
param(
|
|
[string]$ConfigPath,
|
|
[System.Collections.IDictionary]$Expected
|
|
)
|
|
|
|
$current = @(Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath)
|
|
if (
|
|
$current.Count -ne 1 -or
|
|
-not (Test-IdentityExactlyMatches -Expected $Expected -Actual $current[0])
|
|
) {
|
|
throw "Current tunnel identity is not the unique launched tunnel"
|
|
}
|
|
}
|
|
|
|
function Assert-IdentityUnchanged {
|
|
param(
|
|
[System.Collections.IDictionary]$Expected,
|
|
[System.Collections.IDictionary]$Actual,
|
|
[string]$Role
|
|
)
|
|
|
|
foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($Expected[$field].ToString() -cne $Actual[$field].ToString()) {
|
|
throw "$Role identity drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-IdentityExactlyMatches {
|
|
param(
|
|
[System.Collections.IDictionary]$Expected,
|
|
[System.Collections.IDictionary]$Actual
|
|
)
|
|
|
|
if ($null -eq $Expected -or $null -eq $Actual) {
|
|
return $false
|
|
}
|
|
foreach ($field in @("pid", "started_at_utc", "executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($Expected[$field].ToString() -cne $Actual[$field].ToString()) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Stop-VerifiedIdentity {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role
|
|
)
|
|
|
|
$current = Wait-ProcessIdentity `
|
|
-ProcessId ([int]$Identity.pid) `
|
|
-Role $Role `
|
|
-ExpectedCwd ([string]$Identity.cwd) `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-IdentityUnchanged -Expected $Identity -Actual $current -Role $Role
|
|
Stop-Process -Id ([int]$Identity.pid) -Force -ErrorAction Stop
|
|
$deadline = (Get-Date).AddSeconds($ProcessStopTimeoutSeconds)
|
|
do {
|
|
if ($null -eq (Get-Process -Id ([int]$Identity.pid) -ErrorAction SilentlyContinue)) {
|
|
return
|
|
}
|
|
Start-Sleep -Milliseconds 200
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "$Role did not stop within the bounded interval"
|
|
}
|
|
|
|
function ConvertTo-SafeProcessIdentity {
|
|
param([System.Collections.IDictionary]$Identity)
|
|
|
|
return [ordered]@{
|
|
pid = [int]$Identity.pid
|
|
started_at_utc = [string]$Identity.started_at_utc
|
|
executable_sha256 = [string]$Identity.executable_sha256
|
|
command_line_sha256 = [string]$Identity.command_line_sha256
|
|
cwd_sha256 = Get-CanonicalPathSha256 -Path ([string]$Identity.cwd)
|
|
}
|
|
}
|
|
|
|
function Save-CompleteProcessEnvironment {
|
|
$snapshot = [ordered]@{}
|
|
$environment = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in $environment.Keys) {
|
|
$snapshot[[string]$name] = [string]$environment[$name]
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Get-IdentityEnvironmentValue {
|
|
param(
|
|
[System.Collections.IDictionary]$Environment,
|
|
[string]$Name,
|
|
[switch]$AllowEmpty
|
|
)
|
|
|
|
foreach ($key in $Environment.Keys) {
|
|
if ([string]$key -ieq $Name) {
|
|
$value = [string]$Environment[$key]
|
|
if (-not $AllowEmpty -and [string]::IsNullOrWhiteSpace($value)) {
|
|
throw "Required API environment value is empty"
|
|
}
|
|
return $value
|
|
}
|
|
}
|
|
throw "Required API environment key is missing"
|
|
}
|
|
|
|
function Invoke-EffectiveApiSettingsProbe {
|
|
param(
|
|
[string]$PythonPath,
|
|
[string]$ApiCwd,
|
|
[System.Collections.IDictionary]$Environment
|
|
)
|
|
|
|
$code = "import json; from app.config import settings as s; j=lambda v:json.dumps(v,ensure_ascii=True,separators=(',',':')); d={'DATABASE_URL':str(s.database_url),'SESSION_SECRET':s.session_secret,'ENGINE_URL':str(s.engine_url),'ENGINE_GATEWAY_SHARED_SECRET':s.engine_gateway_shared_secret.get_secret_value(),'OAUTH_GOOGLE_CLIENT_ID':s.oauth_google_client_id,'OAUTH_GOOGLE_CLIENT_SECRET':s.oauth_google_client_secret,'OAUTH_REDIRECT_URI':s.oauth_redirect_uri,'AUTH_ALLOWED_EMAIL_DOMAINS':j(s.auth_allowed_email_domains),'AUTH_TEACHER_EMAILS':j(s.auth_teacher_emails),'AUTH_ADMIN_EMAILS':j(s.auth_admin_emails),'AUTH_SUPER_ADMIN_EMAILS':j(s.auth_super_admin_emails),'AUTH_APPROVED_EMAILS':j(s.auth_approved_emails),'AUTH_NEW_USER_DEFAULT_STATUS':s.auth_new_user_default_status,'AUTH_EMAIL_COHORT_MAP':j(s.auth_email_cohort_map),'AUTH_DOMAIN_COHORT_MAP':j(s.auth_domain_cohort_map),'DEFAULT_AFFILIATION':s.default_affiliation}; print(json.dumps(d,ensure_ascii=True,separators=(',',':')))"
|
|
$callerEnvironment = Save-CompleteProcessEnvironment
|
|
try {
|
|
Set-CompleteProcessEnvironment -Environment $Environment
|
|
Push-Location $ApiCwd
|
|
try {
|
|
$probeOutput = @(& $PythonPath @("-X", "utf8", "-c", $code) 2>$null)
|
|
$probeExit = $LASTEXITCODE
|
|
} finally {
|
|
Pop-Location
|
|
}
|
|
} finally {
|
|
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
|
}
|
|
if ($probeExit -ne 0 -or $probeOutput.Count -ne 1) {
|
|
throw "API effective settings could not be loaded"
|
|
}
|
|
$effective = $probeOutput[0] | ConvertFrom-Json
|
|
$values = [ordered]@{}
|
|
foreach ($property in $effective.PSObject.Properties) {
|
|
$name = [string]$property.Name
|
|
$value = [string]$property.Value
|
|
$values[$name] = $value
|
|
}
|
|
return $values
|
|
}
|
|
|
|
function Set-RequiredApiEnvironmentFromIdentity {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$ExpectedEnvironmentFileSha256
|
|
)
|
|
|
|
$priorEnvPath = Join-Path ([string]$Identity.cwd) ".env"
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvPath `
|
|
-ExpectedSha256 $ExpectedEnvironmentFileSha256 `
|
|
-Role "prior API"
|
|
$values = Invoke-EffectiveApiSettingsProbe `
|
|
-PythonPath ([string]$Identity.executable_path) `
|
|
-ApiCwd ([string]$Identity.cwd) `
|
|
-Environment $Identity.environment
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvPath `
|
|
-ExpectedSha256 $ExpectedEnvironmentFileSha256 `
|
|
-Role "prior API after settings load"
|
|
foreach ($name in @(
|
|
"DATABASE_URL", "SESSION_SECRET", "OAUTH_GOOGLE_CLIENT_ID",
|
|
"OAUTH_GOOGLE_CLIENT_SECRET", "OAUTH_REDIRECT_URI"
|
|
)) {
|
|
if (-not $values.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$values[$name])) {
|
|
throw "Prior API effective authentication/database setting is empty"
|
|
}
|
|
}
|
|
foreach ($name in $values.Keys) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
[string]$values[$name],
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
return $values
|
|
}
|
|
|
|
function Ensure-ReleaseEnvironmentFile {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApiIdentity,
|
|
[string]$ExpectedSourceSha256
|
|
)
|
|
|
|
$source = (Resolve-Path -LiteralPath (Join-Path $PriorApiIdentity.cwd ".env")).Path
|
|
$target = Join-Path $resolvedStableSourceRoot "apps\api\.env"
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $source
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $target
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $source `
|
|
-ExpectedSha256 $ExpectedSourceSha256 `
|
|
-Role "prior API before environment copy"
|
|
$ignored = @(
|
|
& git.exe -C $resolvedStableSourceRoot check-ignore --quiet -- "apps/api/.env"
|
|
)
|
|
if ($LASTEXITCODE -ne 0 -or $ignored.Count -gt 0) {
|
|
throw "New release private environment file is not ignored"
|
|
}
|
|
if ([string]::Equals(
|
|
$source,
|
|
[System.IO.Path]::GetFullPath($target),
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
return
|
|
}
|
|
$sourceSha256 = $ExpectedSourceSha256
|
|
if (Test-Path -LiteralPath $target) {
|
|
if (-not (Test-Path -LiteralPath $target -PathType Leaf)) {
|
|
throw "New release private environment target is not a regular file"
|
|
}
|
|
$targetSha256 = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($targetSha256 -cne $sourceSha256) {
|
|
throw "New release private environment target differs from the prior runtime"
|
|
}
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $source `
|
|
-ExpectedSha256 $ExpectedSourceSha256 `
|
|
-Role "prior API after existing environment verification"
|
|
return
|
|
}
|
|
$input = $null
|
|
$output = $null
|
|
try {
|
|
$input = [System.IO.File]::Open(
|
|
$source,
|
|
[System.IO.FileMode]::Open,
|
|
[System.IO.FileAccess]::Read,
|
|
[System.IO.FileShare]::Read
|
|
)
|
|
$output = [System.IO.File]::Open(
|
|
$target,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
$input.CopyTo($output)
|
|
$output.Flush($true)
|
|
} finally {
|
|
if ($null -ne $output) {
|
|
$output.Dispose()
|
|
}
|
|
if ($null -ne $input) {
|
|
$input.Dispose()
|
|
}
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $target
|
|
$targetSha256 = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($targetSha256 -cne $sourceSha256) {
|
|
throw "New release private environment copy verification failed"
|
|
}
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $source `
|
|
-ExpectedSha256 $ExpectedSourceSha256 `
|
|
-Role "prior API after environment copy"
|
|
}
|
|
|
|
function Get-FutureLauncherRequiredApiSettings {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApiIdentity,
|
|
[string[]]$RequiredNames,
|
|
[int]$EnginePort
|
|
)
|
|
|
|
# Scheduled boot/watchdog launches do not inherit bootstrap-only secret
|
|
# overrides. Rebuild the exact persistent input: prior non-required process
|
|
# environment + start-public-runtime's fixed production flags + target .env.
|
|
$futureEnvironment = [ordered]@{}
|
|
foreach ($key in $PriorApiIdentity.environment.Keys) {
|
|
$isRequired = $false
|
|
foreach ($requiredName in $RequiredNames) {
|
|
if ([string]$key -ieq $requiredName) {
|
|
$isRequired = $true
|
|
break
|
|
}
|
|
}
|
|
if (-not $isRequired) {
|
|
$futureEnvironment[[string]$key] = [string]$PriorApiIdentity.environment[$key]
|
|
}
|
|
}
|
|
$futureEnvironment["ENVIRONMENT"] = "prod"
|
|
$futureEnvironment["ENGINE_URL"] = "http://127.0.0.1:$EnginePort"
|
|
$futureEnvironment["ENGINE_MODE"] = "claude_cli"
|
|
$futureEnvironment["VIGNETTE_LIVE_CLIENT_PROVIDER"] = "claude_cli"
|
|
$futureEnvironment["AUTH_DEV_LOGIN_ENABLED"] = "false"
|
|
$futureEnvironment["AUTO_SEED_PERSONAS"] = "false"
|
|
$futureEnvironment["ALLOW_SEED_PERSONA_FALLBACK"] = "false"
|
|
$futureEnvironment["VIGNETTE_VOICE_POC_SAMPLE_TTS"] = "false"
|
|
$futureEnvironment["FRONTEND_BASE_URL"] = "https://vignette.chanpaca.net"
|
|
$futureEnvironment["CORS_ORIGINS"] = '["https://vignette.chanpaca.net","https://vnet.18ka.net","https://vignette-b1q.pages.dev"]'
|
|
$futureEnvironment["FRONTEND_ORIGIN_MAP"] = '{"api-vignette.chanpaca.net":"https://vignette.chanpaca.net","api-vnet.18ka.net":"https://vnet.18ka.net"}'
|
|
|
|
return Invoke-EffectiveApiSettingsProbe `
|
|
-PythonPath $resolvedPythonPath `
|
|
-ApiCwd (Join-Path $resolvedStableSourceRoot "apps\api") `
|
|
-Environment $futureEnvironment
|
|
}
|
|
|
|
function Assert-RequiredApiSettingsEqual {
|
|
param(
|
|
[System.Collections.IDictionary]$Expected,
|
|
[System.Collections.IDictionary]$Actual,
|
|
[string]$Role
|
|
)
|
|
|
|
if ($Expected.Count -ne $Actual.Count) {
|
|
throw "$Role effective settings key count drift"
|
|
}
|
|
foreach ($name in $Expected.Keys) {
|
|
if (-not $Actual.Contains($name) -or [string]$Expected[$name] -cne [string]$Actual[$name]) {
|
|
throw "$Role effective settings digest drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Assert-EnvironmentFilePinned {
|
|
param(
|
|
[string]$Path,
|
|
[string]$ExpectedSha256,
|
|
[string]$Role
|
|
)
|
|
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $Path
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Role environment file is unavailable"
|
|
}
|
|
$actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actual -cne $ExpectedSha256) {
|
|
throw "$Role environment file SHA256 drift"
|
|
}
|
|
}
|
|
|
|
function Save-SelectedProcessEnvironment {
|
|
param([string[]]$Names)
|
|
|
|
$snapshot = [ordered]@{}
|
|
foreach ($name in $Names) {
|
|
$value = [System.Environment]::GetEnvironmentVariable(
|
|
$name,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
$snapshot[$name] = [ordered]@{
|
|
present = $null -ne $value
|
|
value = $value
|
|
}
|
|
}
|
|
return $snapshot
|
|
}
|
|
|
|
function Restore-SelectedProcessEnvironment {
|
|
param([System.Collections.IDictionary]$Snapshot)
|
|
|
|
foreach ($name in $Snapshot.Keys) {
|
|
$entry = $Snapshot[$name]
|
|
$value = $null
|
|
if ([bool]$entry.present) {
|
|
$value = [string]$entry.value
|
|
}
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
$value,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
|
|
function Assert-NewApiEnvironmentMatches {
|
|
param(
|
|
[System.Collections.IDictionary]$Expected,
|
|
[System.Collections.IDictionary]$ActualIdentity
|
|
)
|
|
|
|
foreach ($name in $Expected.Keys) {
|
|
$actual = Get-IdentityEnvironmentValue `
|
|
-Environment $ActualIdentity.environment `
|
|
-Name ([string]$name) `
|
|
-AllowEmpty
|
|
if ([string]$Expected[$name] -cne $actual) {
|
|
throw "New API required environment digest drift"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-RequiredEnvironmentDigest {
|
|
param([System.Collections.IDictionary]$Environment)
|
|
|
|
$parts = @()
|
|
foreach ($name in @($Environment.Keys | Sort-Object)) {
|
|
$parts += ([string]$name + "=" + (Get-Utf8Sha256 -Value ([string]$Environment[$name])))
|
|
}
|
|
return Get-Utf8Sha256 -Value (@($parts) -join "`n")
|
|
}
|
|
|
|
function Get-ConnectedDatabaseTargetSha256 {
|
|
param([string]$ProbePath)
|
|
|
|
$probeOutput = @(& $resolvedPythonPath @("-X", "utf8", $ProbePath))
|
|
$probeExit = $LASTEXITCODE
|
|
if ($probeExit -ne 0 -or $probeOutput.Count -ne 1) {
|
|
throw "Connected database target identity could not be proven"
|
|
}
|
|
try {
|
|
$payload = $probeOutput[0] | ConvertFrom-Json
|
|
} catch {
|
|
throw "Connected database target identity proof is invalid"
|
|
}
|
|
$digest = [string]$payload.database_target_sha256
|
|
if ($payload.status -ne "passed" -or $digest -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Connected database target identity proof failed"
|
|
}
|
|
return $digest
|
|
}
|
|
|
|
function Set-CompleteProcessEnvironment {
|
|
param([System.Collections.IDictionary]$Environment)
|
|
|
|
foreach ($name in @("SystemRoot", "windir", "SystemDrive", "ComSpec")) {
|
|
$missing = -not $Environment.Contains($name) -or [string]::IsNullOrWhiteSpace([string]$Environment[$name])
|
|
if ($missing) {
|
|
$machineValue = [System.Environment]::GetEnvironmentVariable($name, "Machine")
|
|
if ($machineValue) {
|
|
$Environment[$name] = $machineValue
|
|
}
|
|
}
|
|
}
|
|
$current = [System.Environment]::GetEnvironmentVariables(
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
foreach ($name in @($current.Keys)) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
$null,
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
foreach ($name in $Environment.Keys) {
|
|
[System.Environment]::SetEnvironmentVariable(
|
|
[string]$name,
|
|
[string]$Environment[$name],
|
|
[System.EnvironmentVariableTarget]::Process
|
|
)
|
|
}
|
|
}
|
|
|
|
function ConvertTo-WindowsCommandLineArgument {
|
|
param([AllowEmptyString()][string]$Argument)
|
|
|
|
if ($Argument.Length -gt 0 -and $Argument -notmatch '[\s"]') {
|
|
return $Argument
|
|
}
|
|
$builder = New-Object System.Text.StringBuilder
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
foreach ($character in $Argument.ToCharArray()) {
|
|
if ($character -eq '\') {
|
|
$backslashes++
|
|
continue
|
|
}
|
|
if ($character -eq '"') {
|
|
$null = $builder.Append(('\' * (($backslashes * 2) + 1)))
|
|
$null = $builder.Append('"')
|
|
$backslashes = 0
|
|
continue
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * $backslashes))
|
|
$backslashes = 0
|
|
}
|
|
$null = $builder.Append($character)
|
|
}
|
|
if ($backslashes -gt 0) {
|
|
$null = $builder.Append(('\' * ($backslashes * 2)))
|
|
}
|
|
$null = $builder.Append('"')
|
|
return $builder.ToString()
|
|
}
|
|
|
|
function Join-WindowsArgumentList {
|
|
param([object[]]$ArgumentList)
|
|
|
|
return (@(
|
|
foreach ($argument in $ArgumentList) {
|
|
ConvertTo-WindowsCommandLineArgument -Argument ([string]$argument)
|
|
}
|
|
) -join " ")
|
|
}
|
|
|
|
function Start-PinnedPriorProcess {
|
|
param(
|
|
[System.Collections.IDictionary]$Identity,
|
|
[string]$Role,
|
|
[string]$LogDirectory
|
|
)
|
|
|
|
Assert-FileSha256 `
|
|
-Path ([string]$Identity.executable_path) `
|
|
-ExpectedSha256 ([string]$Identity.executable_sha256) `
|
|
-Role $Role
|
|
if (-not (Test-Path -LiteralPath $Identity.cwd -PathType Container)) {
|
|
throw "$Role working directory is unavailable"
|
|
}
|
|
if (@($Identity.argument_list).Count -eq 0 -or $Identity.environment.Count -eq 0) {
|
|
throw "$Role restart inputs are incomplete"
|
|
}
|
|
$callerEnvironment = Save-CompleteProcessEnvironment
|
|
$suffix = [Guid]::NewGuid().ToString("N")
|
|
try {
|
|
Set-CompleteProcessEnvironment -Environment $Identity.environment
|
|
$argumentString = Join-WindowsArgumentList -ArgumentList @($Identity.argument_list)
|
|
return Start-Process `
|
|
-WindowStyle Hidden `
|
|
-FilePath $Identity.executable_path `
|
|
-ArgumentList $argumentString `
|
|
-WorkingDirectory $Identity.cwd `
|
|
-RedirectStandardOutput (Join-Path $LogDirectory "$Role-rollback-$suffix.out.log") `
|
|
-RedirectStandardError (Join-Path $LogDirectory "$Role-rollback-$suffix.err.log") `
|
|
-PassThru
|
|
} finally {
|
|
Set-CompleteProcessEnvironment -Environment $callerEnvironment
|
|
}
|
|
}
|
|
|
|
function Get-JsonHealth {
|
|
param([string]$Uri)
|
|
|
|
try {
|
|
return Invoke-RestMethod -Uri $Uri -Method Get -UseBasicParsing -TimeoutSec 5
|
|
} catch {
|
|
return $null
|
|
}
|
|
}
|
|
|
|
function Wait-ApiHealth {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec,
|
|
[string]$ExpectedManifestSha256 = "",
|
|
[Nullable[bool]]$ExpectedFreezeActive = $null,
|
|
[string]$ExpectedFreezeTokenSha256 = ""
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$health = Get-JsonHealth -Uri $Uri
|
|
$healthy = (
|
|
$null -ne $health -and
|
|
$health.environment -eq "prod" -and
|
|
$health.db -eq $true -and
|
|
$health.engine -eq $true
|
|
)
|
|
if ($healthy -and $ExpectedManifestSha256) {
|
|
$manifest = $health.upload_manifest
|
|
$healthy = (
|
|
$null -ne $manifest -and
|
|
$manifest.required -eq $true -and
|
|
$manifest.validated -eq $true -and
|
|
[string]$manifest.manifest_sha256 -ceq $ExpectedManifestSha256
|
|
)
|
|
}
|
|
if ($healthy -and $null -ne $ExpectedFreezeActive) {
|
|
$freeze = $health.upload_write_freeze
|
|
$healthy = (
|
|
$null -ne $freeze -and
|
|
$freeze.capable -eq $true -and
|
|
[bool]$freeze.active -eq [bool]$ExpectedFreezeActive -and
|
|
$freeze.valid -eq $true -and
|
|
[int]$freeze.in_flight -eq 0
|
|
)
|
|
if ($healthy -and [bool]$ExpectedFreezeActive) {
|
|
$healthy = [string]$freeze.token_sha256 -ceq $ExpectedFreezeTokenSha256
|
|
}
|
|
}
|
|
if ($healthy) {
|
|
return $health
|
|
}
|
|
Start-Sleep -Milliseconds 500
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Runtime health contract was not proven"
|
|
}
|
|
|
|
function Wait-GoogleAuthContract {
|
|
param(
|
|
[string]$Uri,
|
|
[int]$TimeoutSec
|
|
)
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
do {
|
|
$config = Get-JsonHealth -Uri $Uri
|
|
if ($null -ne $config -and $config.google_oauth_configured -eq $true) {
|
|
$enabledProviders = @(
|
|
$config.providers |
|
|
Where-Object { $_.enabled -eq $true } |
|
|
ForEach-Object { [string]$_.provider }
|
|
)
|
|
$google = @(
|
|
$config.providers |
|
|
Where-Object { [string]$_.provider -eq "google" }
|
|
)
|
|
if (
|
|
$google.Count -eq 1 -and
|
|
$google[0].configured -eq $true -and
|
|
$google[0].enabled -eq $true -and
|
|
$enabledProviders.Count -eq 1 -and
|
|
$enabledProviders[0] -eq "google" -and
|
|
$config.dev_login_enabled -eq $false
|
|
) {
|
|
return
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 500
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Google-only authentication contract was not proven"
|
|
}
|
|
|
|
function Write-PrivacySafeReceiptCreateOnly {
|
|
param(
|
|
[string]$Path,
|
|
[System.Collections.IDictionary]$Payload
|
|
)
|
|
|
|
if ([System.IO.File]::Exists($Path)) {
|
|
throw "Bootstrap receipt target already exists"
|
|
}
|
|
$json = ConvertTo-Json -InputObject $Payload -Depth 10 -Compress
|
|
$bytes = [System.Text.UTF8Encoding]::new($false).GetBytes(
|
|
$json + [Environment]::NewLine
|
|
)
|
|
$stream = $null
|
|
try {
|
|
$stream = [System.IO.File]::Open(
|
|
$Path,
|
|
[System.IO.FileMode]::CreateNew,
|
|
[System.IO.FileAccess]::Write,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
} finally {
|
|
if ($null -ne $stream) {
|
|
$stream.Dispose()
|
|
}
|
|
}
|
|
$actual = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actual -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Bootstrap receipt verification failed"
|
|
}
|
|
return $actual
|
|
}
|
|
|
|
function Enter-InheritedBootstrapRecoveryLock {
|
|
param(
|
|
[string]$Path,
|
|
[string]$SourceRoot,
|
|
[string]$UploadRoot
|
|
)
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($Path)) {
|
|
throw "Bootstrap recovery lock path must be absolute"
|
|
}
|
|
$full = Get-PublicRuntimeCanonicalPath -Path $Path
|
|
foreach ($boundary in @($SourceRoot, $UploadRoot)) {
|
|
if (
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $full -Parent $boundary) -or
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $boundary -Parent $full)
|
|
) {
|
|
throw "Bootstrap recovery lock must be private and disjoint"
|
|
}
|
|
}
|
|
$parent = [System.IO.Path]::GetDirectoryName($full)
|
|
[System.IO.Directory]::CreateDirectory($parent) | Out-Null
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $full
|
|
$stream = [System.IO.File]::Open(
|
|
$full,
|
|
[System.IO.FileMode]::OpenOrCreate,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::Read
|
|
)
|
|
try {
|
|
$nonce = [Guid]::NewGuid().ToString("N") + [Guid]::NewGuid().ToString("N")
|
|
$ownerStartedAtUtc = (
|
|
Get-Process -Id $PID -ErrorAction Stop
|
|
).StartTime.ToUniversalTime().ToString("o")
|
|
$payload = [ordered]@{
|
|
schema_version = "vignette.public-runtime-inherited-lock.v1"
|
|
status = "held"
|
|
owner_pid = $PID
|
|
owner_started_at_utc = $ownerStartedAtUtc
|
|
nonce_sha256 = Get-Utf8Sha256 -Value $nonce
|
|
source_commit = $ExpectedSourceCommit
|
|
source_tree = $ExpectedSourceTree
|
|
}
|
|
$json = ConvertTo-Json -InputObject $payload -Compress
|
|
$bytes = [System.Text.UTF8Encoding]::new($false).GetBytes(
|
|
$json + [Environment]::NewLine
|
|
)
|
|
$stream.SetLength(0)
|
|
$stream.Position = 0
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
$receiptSha256 = Get-Utf8Sha256 -Value ($json + [Environment]::NewLine)
|
|
return [pscustomobject]@{
|
|
Stream = $stream
|
|
Path = $full
|
|
ReceiptSha256 = $receiptSha256
|
|
}
|
|
} catch {
|
|
$stream.Dispose()
|
|
throw
|
|
}
|
|
}
|
|
|
|
function Get-PublicRuntimeTaskTruth {
|
|
param(
|
|
[object[]]$Snapshot,
|
|
[switch]$ExpectedRestored
|
|
)
|
|
|
|
$enabledCount = 0
|
|
$runningCount = 0
|
|
$restored = $true
|
|
foreach ($entry in @($Snapshot)) {
|
|
if (-not [bool]$entry.exists) {
|
|
continue
|
|
}
|
|
$task = Get-ScheduledTask `
|
|
-TaskName ([string]$entry.task_name) `
|
|
-ErrorAction Stop
|
|
$enabled = [bool]$task.Settings.Enabled
|
|
if ($enabled) {
|
|
$enabledCount++
|
|
}
|
|
if ([string]$task.State -eq "Running") {
|
|
$runningCount++
|
|
}
|
|
if ($ExpectedRestored -and $enabled -ne [bool]$entry.was_enabled) {
|
|
$restored = $false
|
|
}
|
|
}
|
|
return [ordered]@{
|
|
enabled_count = $enabledCount
|
|
running_count = $runningCount
|
|
all_disabled_and_idle = ($enabledCount -eq 0 -and $runningCount -eq 0)
|
|
expected_state_restored = $restored
|
|
}
|
|
}
|
|
|
|
function Assert-OfflinedRuntimeRaceGate {
|
|
param(
|
|
[object[]]$TaskSnapshot,
|
|
[string]$ConfigPath
|
|
)
|
|
|
|
Assert-PublicRuntimeTasksDisabledAndIdle `
|
|
-Snapshot $TaskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-LoopbackListenerAbsent `
|
|
-Port $ApiPort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-TunnelAbsent `
|
|
-ConfigPath $ConfigPath `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
}
|
|
|
|
function Assert-ListenerIdentityUnchanged {
|
|
param(
|
|
[int]$Port,
|
|
[System.Collections.IDictionary]$Expected,
|
|
[string]$Role
|
|
)
|
|
|
|
$listenerProcessId = Get-ExactLoopbackListenerPid -Port $Port
|
|
if ($listenerProcessId -ne [int]$Expected.pid) {
|
|
throw "$Role listener PID drift"
|
|
}
|
|
$actual = Wait-ProcessIdentity `
|
|
-ProcessId $listenerProcessId `
|
|
-Role $Role `
|
|
-ExpectedCwd ([string]$Expected.cwd) `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-IdentityUnchanged -Expected $Expected -Actual $actual -Role $Role
|
|
}
|
|
|
|
function Get-OwnedFreezeTokenSha256 {
|
|
param([string]$Path)
|
|
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $Path
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "Owned bootstrap write-freeze is unavailable"
|
|
}
|
|
$payload = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
if (
|
|
$null -eq $payload -or
|
|
$payload.schema_version -ne "vignette.public-upload-write-freeze.v1" -or
|
|
[string]::IsNullOrWhiteSpace([string]$payload.token)
|
|
) {
|
|
throw "Owned bootstrap write-freeze is invalid"
|
|
}
|
|
return Get-Utf8Sha256 -Value ([string]$payload.token)
|
|
}
|
|
|
|
function Remove-OwnedFreezeByHash {
|
|
param(
|
|
[string]$Path,
|
|
[string]$ExpectedTokenSha256
|
|
)
|
|
|
|
if (-not [System.IO.File]::Exists($Path)) {
|
|
return
|
|
}
|
|
$payload = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
if (
|
|
$null -eq $payload -or
|
|
$payload.schema_version -ne "vignette.public-upload-write-freeze.v1" -or
|
|
[string]::IsNullOrWhiteSpace([string]$payload.token) -or
|
|
(Get-Utf8Sha256 -Value ([string]$payload.token)) -cne $ExpectedTokenSha256
|
|
) {
|
|
throw "Bootstrap write-freeze ownership proof failed"
|
|
}
|
|
[System.IO.File]::Delete($Path)
|
|
if ([System.IO.File]::Exists($Path)) {
|
|
throw "Bootstrap write-freeze deletion failed"
|
|
}
|
|
}
|
|
|
|
function Get-ValidatedExplicitLegacySourceRoots {
|
|
param([string[]]$AllowedRoots)
|
|
|
|
if ($AllowedRoots.Count -ne 3) {
|
|
throw "Legacy bootstrap requires exactly three explicit source roots"
|
|
}
|
|
$resolvedAllowedRoots = @()
|
|
foreach ($allowed in $AllowedRoots) {
|
|
if (-not [System.IO.Path]::IsPathRooted($allowed)) {
|
|
throw "Allowed legacy upload root is not absolute"
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $allowed
|
|
if (-not (Test-Path -LiteralPath $allowed -PathType Container)) {
|
|
throw "Allowed legacy upload root is unavailable"
|
|
}
|
|
$resolvedAllowed = (Resolve-Path -LiteralPath $allowed).Path
|
|
foreach ($existingAllowed in $resolvedAllowedRoots) {
|
|
if ([string]::Equals(
|
|
$existingAllowed,
|
|
$resolvedAllowed,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
throw "Allowed legacy upload roots contain a duplicate"
|
|
}
|
|
}
|
|
$resolvedAllowedRoots += $resolvedAllowed
|
|
}
|
|
return $resolvedAllowedRoots
|
|
}
|
|
|
|
function Get-ValidatedLegacySourceRoots {
|
|
param(
|
|
[System.Collections.IDictionary]$ApiIdentity,
|
|
[string[]]$AllowedRoots
|
|
)
|
|
|
|
$declared = $null
|
|
$declaredPresent = $false
|
|
foreach ($entry in $ApiIdentity.environment.Keys) {
|
|
if ([string]$entry -ieq "USER_UPLOAD_DIR") {
|
|
$declaredPresent = $true
|
|
$candidate = [string]$ApiIdentity.environment[$entry]
|
|
if ([string]::IsNullOrWhiteSpace($candidate)) {
|
|
throw "Legacy USER_UPLOAD_DIR is present but empty"
|
|
}
|
|
$declared = $candidate
|
|
break
|
|
}
|
|
}
|
|
if (-not $declaredPresent) {
|
|
$declared = Join-Path ([string]$ApiIdentity.cwd) "uploads"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($declared)) {
|
|
throw "Legacy upload source is not absolute"
|
|
}
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $declared
|
|
if (-not (Test-Path -LiteralPath $declared -PathType Container)) {
|
|
throw "Legacy upload source is unavailable"
|
|
}
|
|
$resolvedDeclared = (Resolve-Path -LiteralPath $declared).Path
|
|
$resolvedAllowedRoots = @(
|
|
Get-ValidatedExplicitLegacySourceRoots -AllowedRoots $AllowedRoots
|
|
)
|
|
$matches = @()
|
|
foreach ($resolvedAllowed in $resolvedAllowedRoots) {
|
|
if ([string]::Equals(
|
|
$resolvedAllowed,
|
|
$resolvedDeclared,
|
|
[System.StringComparison]::OrdinalIgnoreCase
|
|
)) {
|
|
$matches += $resolvedAllowed
|
|
}
|
|
}
|
|
if ($matches.Count -ne 1) {
|
|
throw "Legacy upload source is not one explicitly allowed root"
|
|
}
|
|
return $resolvedAllowedRoots
|
|
}
|
|
|
|
function Restore-LegacyRuntime {
|
|
param(
|
|
[System.Collections.IDictionary]$PriorApi,
|
|
[System.Collections.IDictionary]$PriorTunnel,
|
|
[string]$ConfigPath,
|
|
[string]$FreezePath,
|
|
[string]$FreezeTokenSha256,
|
|
[string]$PrivateStateRoot,
|
|
[string]$PriorEnvironmentPath,
|
|
[string]$PriorEnvironmentSha256,
|
|
[System.Collections.IDictionary]$OwnedNewApi,
|
|
[System.Collections.IDictionary]$OwnedNewTunnel
|
|
)
|
|
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $PriorEnvironmentPath `
|
|
-ExpectedSha256 $PriorEnvironmentSha256 `
|
|
-Role "prior API rollback"
|
|
# 공개 ingress를 먼저 닫아 rollback 중 교체 API가 외부 요청을 받지 않게 한다.
|
|
$currentTunnels = @(Get-TunnelIdentitiesForConfig -ConfigPath $ConfigPath)
|
|
if ($currentTunnels.Count -gt 1) {
|
|
throw "Rollback found ambiguous tunnel identities"
|
|
}
|
|
$restoredTunnel = $null
|
|
if ($currentTunnels.Count -eq 1) {
|
|
if (Test-IdentityExactlyMatches -Expected $PriorTunnel -Actual $currentTunnels[0]) {
|
|
$restoredTunnel = $currentTunnels[0]
|
|
} elseif (Test-IdentityExactlyMatches -Expected $OwnedNewTunnel -Actual $currentTunnels[0]) {
|
|
Stop-VerifiedIdentity -Identity $OwnedNewTunnel -Role "owned new tunnel"
|
|
} else {
|
|
throw "Rollback refuses to stop an unowned tunnel"
|
|
}
|
|
}
|
|
if ($null -eq $restoredTunnel) {
|
|
Assert-TunnelAbsent -ConfigPath $ConfigPath -TimeoutSec $ProcessStopTimeoutSeconds
|
|
}
|
|
|
|
$listeners = @(
|
|
Get-NetTCPConnection -State Listen -LocalPort $ApiPort -ErrorAction SilentlyContinue
|
|
)
|
|
$listenerPids = @($listeners | ForEach-Object { [int]$_.OwningProcess } | Sort-Object -Unique)
|
|
if ($listenerPids.Count -gt 1) {
|
|
throw "Rollback found ambiguous API listeners"
|
|
}
|
|
$restoredApi = $null
|
|
if ($listenerPids.Count -eq 1) {
|
|
$currentApi = Wait-ProcessIdentity `
|
|
-ProcessId ([int]$listenerPids[0]) `
|
|
-Role "rollback API listener" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if (Test-IdentityExactlyMatches -Expected $PriorApi -Actual $currentApi) {
|
|
$restoredApi = $currentApi
|
|
} elseif (Test-IdentityExactlyMatches -Expected $OwnedNewApi -Actual $currentApi) {
|
|
Stop-VerifiedIdentity -Identity $OwnedNewApi -Role "owned new API"
|
|
} else {
|
|
throw "Rollback refuses to stop an unowned API listener"
|
|
}
|
|
}
|
|
if ($null -eq $restoredApi) {
|
|
Assert-LoopbackListenerAbsent -Port $ApiPort -TimeoutSec $ProcessStopTimeoutSeconds
|
|
}
|
|
|
|
if ([System.IO.File]::Exists($FreezePath)) {
|
|
if ($FreezeTokenSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Rollback cannot prove write-freeze ownership"
|
|
}
|
|
Remove-OwnedFreezeByHash `
|
|
-Path $FreezePath `
|
|
-ExpectedTokenSha256 $FreezeTokenSha256
|
|
}
|
|
|
|
if ($null -eq $restoredApi) {
|
|
$apiProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorApi `
|
|
-Role "legacy-api" `
|
|
-LogDirectory $PrivateStateRoot
|
|
$restoredApi = Wait-ProcessIdentity `
|
|
-ProcessId $apiProcess.Id `
|
|
-Role "restored legacy API" `
|
|
-ExpectedCwd ([string]$PriorApi.cwd) `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
}
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorApi[$field].ToString() -cne $restoredApi[$field].ToString()) {
|
|
throw "Restored legacy API identity drift"
|
|
}
|
|
}
|
|
$null = Wait-ApiHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
Wait-GoogleAuthContract `
|
|
-Uri "http://127.0.0.1:$ApiPort/auth/config" `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
|
|
if ($null -eq $restoredTunnel) {
|
|
$tunnelProcess = Start-PinnedPriorProcess `
|
|
-Identity $PriorTunnel `
|
|
-Role "legacy-tunnel" `
|
|
-LogDirectory $PrivateStateRoot
|
|
$restoredTunnel = Wait-ProcessIdentity `
|
|
-ProcessId $tunnelProcess.Id `
|
|
-Role "restored legacy tunnel" `
|
|
-ExpectedCwd ([string]$PriorTunnel.cwd) `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
}
|
|
foreach ($field in @("executable_sha256", "command_line_sha256", "cwd")) {
|
|
if ($PriorTunnel[$field].ToString() -cne $restoredTunnel[$field].ToString()) {
|
|
throw "Restored legacy tunnel identity drift"
|
|
}
|
|
}
|
|
$null = Wait-ApiHealth -Uri $PublicHealthUrl -TimeoutSec $HealthTimeoutSeconds
|
|
Wait-GoogleAuthContract `
|
|
-Uri $publicAuthConfigUrl `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
if ([System.IO.File]::Exists($FreezePath)) {
|
|
throw "Legacy rollback left the write-freeze sentinel present"
|
|
}
|
|
}
|
|
|
|
$resolvedStableSourceRoot = (Resolve-Path -LiteralPath $StableSourceRoot).Path
|
|
if ($PublicHealthUrl -cne "https://api-vignette.chanpaca.net/health") {
|
|
throw "Legacy bootstrap public health URL must be canonical"
|
|
}
|
|
$uploadContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-upload-root.ps1"
|
|
$taskContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-task-maintenance.ps1"
|
|
$taskDefinitionContract = Join-Path $resolvedStableSourceRoot "scripts\public-runtime-task-definition-cutover.ps1"
|
|
$bootTaskInstaller = Join-Path $resolvedStableSourceRoot "scripts\register-boot-task.ps1"
|
|
$watchdogTaskInstaller = Join-Path $resolvedStableSourceRoot "scripts\install-public-runtime-task.ps1"
|
|
$initializer = Join-Path $resolvedStableSourceRoot "scripts\initialize-public-runtime-upload-root.ps1"
|
|
$initializerWorker = Join-Path $resolvedStableSourceRoot "scripts\initialize-public-runtime-upload-root.py"
|
|
$startScript = Join-Path $resolvedStableSourceRoot "scripts\start-public-runtime.ps1"
|
|
$manifestProbe = Join-Path $resolvedStableSourceRoot "scripts\validate-public-runtime-upload-manifest.py"
|
|
$uploadProbe = Join-Path $resolvedStableSourceRoot "scripts\probe-public-runtime-upload-root.py"
|
|
$databaseIdentityProbe = Join-Path $resolvedStableSourceRoot "scripts\probe-public-runtime-database-identity.py"
|
|
foreach ($required in @(
|
|
$uploadContract,
|
|
$taskContract,
|
|
$taskDefinitionContract,
|
|
$bootTaskInstaller,
|
|
$watchdogTaskInstaller,
|
|
$initializer,
|
|
$initializerWorker,
|
|
$startScript,
|
|
$manifestProbe,
|
|
$uploadProbe,
|
|
$databaseIdentityProbe,
|
|
$PythonPath,
|
|
$CloudflaredPath,
|
|
$CloudflaredConfigPath
|
|
)) {
|
|
if (-not (Test-Path -LiteralPath $required -PathType Leaf)) {
|
|
throw "Legacy bootstrap prerequisite is unavailable"
|
|
}
|
|
}
|
|
. $uploadContract
|
|
. $taskContract
|
|
. $taskDefinitionContract
|
|
|
|
Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot
|
|
$resolvedPythonPath = (Resolve-Path -LiteralPath $PythonPath).Path
|
|
$resolvedCloudflaredPath = (Resolve-Path -LiteralPath $CloudflaredPath).Path
|
|
$resolvedCloudflaredConfigPath = (Resolve-Path -LiteralPath $CloudflaredConfigPath).Path
|
|
Assert-FileSha256 -Path $resolvedPythonPath -ExpectedSha256 $ExpectedPythonSha256 -Role "Python"
|
|
Assert-FileSha256 -Path $resolvedCloudflaredPath -ExpectedSha256 $ExpectedCloudflaredSha256 -Role "cloudflared"
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredConfigPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredConfigSha256 `
|
|
-Role "cloudflared config"
|
|
|
|
$targetCanonical = Get-PublicRuntimeCanonicalPath -Path $UserUploadDir
|
|
$resolvedPrivateStateDir = Resolve-PrivateBootstrapStateDirectory `
|
|
-Path $ManifestStateDir `
|
|
-SourceRoot $resolvedStableSourceRoot `
|
|
-UploadRoot $targetCanonical
|
|
$resolvedFreezePath = Resolve-PrivateBootstrapPath `
|
|
-Path $UserUploadWriteFreezePath `
|
|
-StateRoot $resolvedPrivateStateDir
|
|
$resolvedCutoverReceiptPath = Resolve-PrivateBootstrapPath `
|
|
-Path $CutoverReceiptPath `
|
|
-StateRoot $resolvedPrivateStateDir
|
|
$resolvedTaskRecoveryReceiptPath = Resolve-PrivateBootstrapPath `
|
|
-Path $TaskRecoveryReceiptPath `
|
|
-StateRoot $resolvedPrivateStateDir
|
|
foreach ($absentTarget in @(
|
|
$resolvedFreezePath,
|
|
$resolvedCutoverReceiptPath,
|
|
$resolvedTaskRecoveryReceiptPath
|
|
)) {
|
|
if (Test-Path -LiteralPath $absentTarget) {
|
|
throw "Legacy bootstrap fixed private-state target must be absent at preflight"
|
|
}
|
|
}
|
|
$fixedPrivateTargets = @(
|
|
$resolvedFreezePath,
|
|
$resolvedCutoverReceiptPath,
|
|
$resolvedTaskRecoveryReceiptPath,
|
|
(Get-PublicRuntimeCanonicalPath -Path $RecoveryLockPath)
|
|
)
|
|
if (@($fixedPrivateTargets | Sort-Object -Unique).Count -ne $fixedPrivateTargets.Count) {
|
|
throw "Legacy bootstrap private-state paths must be pairwise distinct"
|
|
}
|
|
$preflightLegacySourceRoots = @(
|
|
Get-ValidatedExplicitLegacySourceRoots `
|
|
-AllowedRoots $AllowedLegacySourceUploadDir
|
|
)
|
|
foreach ($sourceRoot in $preflightLegacySourceRoots) {
|
|
if (
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $sourceRoot -Parent $targetCanonical) -or
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $targetCanonical -Parent $sourceRoot) -or
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $sourceRoot -Parent $resolvedPrivateStateDir) -or
|
|
(Test-PublicRuntimePathIsSameOrChild -Candidate $resolvedPrivateStateDir -Parent $sourceRoot)
|
|
) {
|
|
throw "Legacy source roots must be disjoint from public and private roots"
|
|
}
|
|
}
|
|
$preservedProbeArgs = @(
|
|
"-X", "utf8", "-B", $initializerWorker,
|
|
"probe-preserved-inventory",
|
|
"--expected-preserved-object-count", $ExpectedPreservedObjectCount.ToString(),
|
|
"--expected-preserved-total-size-bytes", $ExpectedPreservedTotalSizeBytes.ToString(),
|
|
"--expected-preserved-inventory-sha256", $ExpectedPreservedInventorySha256
|
|
)
|
|
foreach ($sourceRoot in $preflightLegacySourceRoots) {
|
|
$preservedProbeArgs += @("--source-root", $sourceRoot)
|
|
}
|
|
$preservedProbeOutput = @(& $resolvedPythonPath @preservedProbeArgs)
|
|
$preservedProbeExit = $LASTEXITCODE
|
|
$preservedProbePayload = $null
|
|
if ($preservedProbeExit -eq 0) {
|
|
try {
|
|
$preservedProbePayload = (@($preservedProbeOutput) -join "").Trim() |
|
|
ConvertFrom-Json
|
|
} catch {
|
|
$preservedProbePayload = $null
|
|
}
|
|
}
|
|
if (
|
|
$preservedProbeExit -ne 0 -or
|
|
$null -eq $preservedProbePayload -or
|
|
$preservedProbePayload.status -cne "verified" -or
|
|
[int]$preservedProbePayload.preserved_object_count -ne $ExpectedPreservedObjectCount -or
|
|
[long]$preservedProbePayload.preserved_total_size_bytes -ne
|
|
$ExpectedPreservedTotalSizeBytes -or
|
|
[string]$preservedProbePayload.preserved_inventory_sha256 -cne $ExpectedPreservedInventorySha256
|
|
) {
|
|
throw "Legacy preserved source inventory preflight failed"
|
|
}
|
|
$preservedProbeDecodeCounts = Get-PreservedDecodeCountProof `
|
|
-Payload $preservedProbePayload `
|
|
-PreservedObjectCount $ExpectedPreservedObjectCount `
|
|
-Role "Legacy preserved source inventory preflight"
|
|
$bootstrapLock = Enter-InheritedBootstrapRecoveryLock `
|
|
-Path $RecoveryLockPath `
|
|
-SourceRoot $resolvedStableSourceRoot `
|
|
-UploadRoot $targetCanonical
|
|
|
|
$taskSnapshot = @()
|
|
$taskMaintenanceEntered = $false
|
|
$originalTaskDefinitionSnapshot = $null
|
|
$disabledOriginalTaskDefinitionSnapshot = $null
|
|
$newDisabledTaskDefinitionSnapshot = $null
|
|
$operationalTaskDefinitionSnapshot = $null
|
|
$runtimeMutationStarted = $false
|
|
$noRollback = $false
|
|
$failureStage = "preflight"
|
|
$priorApiIdentity = $null
|
|
$priorTunnelIdentity = $null
|
|
$whisperIdentity = $null
|
|
$meloTtsIdentity = $null
|
|
$newApiIdentity = $null
|
|
$newTunnelIdentity = $null
|
|
$newApiLaunchAttempted = $false
|
|
$newTunnelProcess = $null
|
|
$freezeAbsentBeforeInitializer = $false
|
|
$requiredApiEnvironment = $null
|
|
$callerRequiredEnvironmentSnapshot = $null
|
|
$priorDatabaseTargetSha256 = ""
|
|
$requiredEnvironmentDigest = ""
|
|
$priorEnvironmentFilePath = ""
|
|
$priorEnvironmentFileSha256 = ""
|
|
$stableEnvironmentFilePath = Join-Path $resolvedStableSourceRoot "apps\api\.env"
|
|
$stableEnvironmentFileSha256 = ""
|
|
$publicAuthConfigUrl = $PublicHealthUrl -replace '/health(?:\?.*)?$', '/auth/config'
|
|
$freezeTokenSha256 = ""
|
|
$cutoverReceiptSha256 = ""
|
|
$taskRecoveryReceiptSha256 = ""
|
|
|
|
try {
|
|
# LEGACY_BOOTSTRAP_STAGE:task_maintenance_enter
|
|
$failureStage = "task_maintenance_enter"
|
|
Assert-PublicRuntimeCoordinatedTaskNamesExact `
|
|
-TaskNames $CoordinatedTaskNames
|
|
$originalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot `
|
|
-RequireEnabled
|
|
$taskSnapshot = @(
|
|
Enter-PublicRuntimeTaskMaintenance `
|
|
-TaskNames $CoordinatedTaskNames `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
)
|
|
$taskMaintenanceEntered = $true
|
|
Assert-PublicRuntimeTasksDisabledAndIdle `
|
|
-Snapshot $taskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$disabledOriginalTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot
|
|
foreach ($taskDefinition in @($disabledOriginalTaskDefinitionSnapshot.entries)) {
|
|
if ([bool]$taskDefinition.enabled) {
|
|
throw "Original public runtime task definition remained enabled during maintenance"
|
|
}
|
|
}
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:exact_runtime_capture
|
|
$failureStage = "exact_runtime_capture"
|
|
$priorApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort
|
|
$priorApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $priorApiPid `
|
|
-Role "legacy API" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-ApiIdentityContract -Identity $priorApiIdentity
|
|
$legacySourceProof = Assert-LegacyApiSourceProvenance `
|
|
-ApiIdentity $priorApiIdentity
|
|
$requiredApiEnvironmentNames = @(
|
|
"DATABASE_URL", "SESSION_SECRET", "ENGINE_URL", "ENGINE_GATEWAY_SHARED_SECRET",
|
|
"OAUTH_GOOGLE_CLIENT_ID", "OAUTH_GOOGLE_CLIENT_SECRET", "OAUTH_REDIRECT_URI",
|
|
"AUTH_ALLOWED_EMAIL_DOMAINS", "AUTH_TEACHER_EMAILS", "AUTH_ADMIN_EMAILS",
|
|
"AUTH_SUPER_ADMIN_EMAILS", "AUTH_APPROVED_EMAILS", "AUTH_NEW_USER_DEFAULT_STATUS",
|
|
"AUTH_EMAIL_COHORT_MAP", "AUTH_DOMAIN_COHORT_MAP", "DEFAULT_AFFILIATION"
|
|
)
|
|
$callerRequiredEnvironmentSnapshot = Save-SelectedProcessEnvironment `
|
|
-Names $requiredApiEnvironmentNames
|
|
$priorEnvironmentFilePath = (Resolve-Path -LiteralPath (
|
|
Join-Path $priorApiIdentity.cwd ".env"
|
|
)).Path
|
|
Assert-PublicRuntimePathHasNoReparsePoint -Path $priorEnvironmentFilePath
|
|
$priorEnvironmentFileSha256 = (
|
|
Get-FileHash -LiteralPath $priorEnvironmentFilePath -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
$requiredApiEnvironment = Set-RequiredApiEnvironmentFromIdentity `
|
|
-Identity $priorApiIdentity `
|
|
-ExpectedEnvironmentFileSha256 $priorEnvironmentFileSha256
|
|
Ensure-ReleaseEnvironmentFile `
|
|
-PriorApiIdentity $priorApiIdentity `
|
|
-ExpectedSourceSha256 $priorEnvironmentFileSha256
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvironmentFilePath `
|
|
-ExpectedSha256 $priorEnvironmentFileSha256 `
|
|
-Role "prior API after release environment preparation"
|
|
$stableEnvironmentFilePath = (Resolve-Path -LiteralPath $stableEnvironmentFilePath).Path
|
|
$stableEnvironmentFileSha256 = (
|
|
Get-FileHash -LiteralPath $stableEnvironmentFilePath -Algorithm SHA256
|
|
).Hash.ToLowerInvariant()
|
|
if ($stableEnvironmentFileSha256 -cne $priorEnvironmentFileSha256) {
|
|
throw "New release environment file is not exact to the prior runtime"
|
|
}
|
|
$futureRequiredApiEnvironment = Get-FutureLauncherRequiredApiSettings `
|
|
-PriorApiIdentity $priorApiIdentity `
|
|
-RequiredNames $requiredApiEnvironmentNames `
|
|
-EnginePort $EnginePort
|
|
Assert-RequiredApiSettingsEqual `
|
|
-Expected $requiredApiEnvironment `
|
|
-Actual $futureRequiredApiEnvironment `
|
|
-Role "future boot/watchdog launcher"
|
|
$priorDatabaseTargetSha256 = Get-ConnectedDatabaseTargetSha256 `
|
|
-ProbePath $databaseIdentityProbe
|
|
$requiredEnvironmentDigest = Get-RequiredEnvironmentDigest `
|
|
-Environment $requiredApiEnvironment
|
|
$priorTunnelIdentity = Get-ExactTunnelIdentity `
|
|
-ConfigPath $resolvedCloudflaredConfigPath
|
|
$whisperPid = Get-ExactLoopbackListenerPid -Port $WhisperPort
|
|
$whisperIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $whisperPid `
|
|
-Role "whisper sidecar" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$meloTtsPid = Get-ExactLoopbackListenerPid -Port $MeloTtsPort
|
|
$meloTtsIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $meloTtsPid `
|
|
-Role "MeloTTS sidecar" `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$legacySourceRoots = @(
|
|
Get-ValidatedLegacySourceRoots `
|
|
-ApiIdentity $priorApiIdentity `
|
|
-AllowedRoots $preflightLegacySourceRoots
|
|
)
|
|
$null = Wait-ApiHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
$null = Wait-ApiHealth -Uri $PublicHealthUrl -TimeoutSec $HealthTimeoutSeconds
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:tunnel_quiescence
|
|
$failureStage = "tunnel_quiescence"
|
|
$runtimeMutationStarted = $true
|
|
Stop-VerifiedIdentity -Identity $priorTunnelIdentity -Role "legacy tunnel"
|
|
Assert-TunnelAbsent `
|
|
-ConfigPath $resolvedCloudflaredConfigPath `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:listener_quiescence
|
|
$failureStage = "listener_quiescence"
|
|
Stop-VerifiedIdentity -Identity $priorApiIdentity -Role "legacy API"
|
|
Assert-LoopbackListenerAbsent `
|
|
-Port $ApiPort `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
|
|
$sourceRootSha256s = @()
|
|
foreach ($legacySourceRoot in $legacySourceRoots) {
|
|
$sourceRootSha256s += Get-CanonicalPathSha256 -Path $legacySourceRoot
|
|
}
|
|
$sortedSourceRootSha256s = @($sourceRootSha256s | Sort-Object)
|
|
$sourceRootSetJson = ConvertTo-Json `
|
|
-InputObject @($sortedSourceRootSha256s) `
|
|
-Compress
|
|
$capture = [ordered]@{
|
|
schema_version = "vignette.public-upload-offline-quiescence-capture.v2"
|
|
status = "quiesced"
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ")
|
|
source_commit = [string]$legacySourceProof.commit
|
|
source_tree = [string]$legacySourceProof.tree
|
|
source_root_sha256s = @($sourceRootSha256s)
|
|
source_root_set_sha256 = Get-Utf8Sha256 -Value $sourceRootSetJson
|
|
api_identity = ConvertTo-SafeProcessIdentity -Identity $priorApiIdentity
|
|
tunnel_identity = ConvertTo-SafeProcessIdentity -Identity $priorTunnelIdentity
|
|
listener_absent = $true
|
|
tunnel_absent = $true
|
|
listener_endpoint_sha256 = Get-Utf8Sha256 -Value "tcp://127.0.0.1:$ApiPort"
|
|
tunnel_config_sha256 = $ExpectedCloudflaredConfigSha256
|
|
}
|
|
$captureJson = ConvertTo-Json -InputObject $capture -Depth 8 -Compress
|
|
$captureBase64 = [Convert]::ToBase64String(
|
|
[System.Text.UTF8Encoding]::new($false).GetBytes($captureJson)
|
|
)
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:offline_initializer
|
|
$failureStage = "offline_initializer"
|
|
Assert-OfflinedRuntimeRaceGate `
|
|
-TaskSnapshot $taskSnapshot `
|
|
-ConfigPath $resolvedCloudflaredConfigPath
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvironmentFilePath `
|
|
-ExpectedSha256 $priorEnvironmentFileSha256 `
|
|
-Role "prior API"
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $stableEnvironmentFilePath `
|
|
-ExpectedSha256 $stableEnvironmentFileSha256 `
|
|
-Role "new release"
|
|
if ([System.IO.File]::Exists($resolvedFreezePath)) {
|
|
throw "Offline initializer requires an absent write-freeze target"
|
|
}
|
|
$freezeAbsentBeforeInitializer = $true
|
|
$initializerOutput = @(
|
|
& $initializer `
|
|
-StableSourceRoot $resolvedStableSourceRoot `
|
|
-UserUploadDir $UserUploadDir `
|
|
-ManifestStateDir $resolvedPrivateStateDir `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-ExpectedReferenceCount $ExpectedReferenceCount `
|
|
-ExpectedPreservedObjectCount $ExpectedPreservedObjectCount `
|
|
-ExpectedPreservedTotalSizeBytes $ExpectedPreservedTotalSizeBytes `
|
|
-ExpectedPreservedInventorySha256 $ExpectedPreservedInventorySha256 `
|
|
-SourceUploadDir $legacySourceRoots `
|
|
-OfflineQuiescenceCaptureBase64 $captureBase64 `
|
|
-ExpectedOfflineSourceCommit $ExpectedLegacySourceCommit `
|
|
-ExpectedOfflineSourceTree $ExpectedLegacySourceTree `
|
|
-PythonPath $resolvedPythonPath
|
|
)
|
|
$freezeTokenSha256 = Get-OwnedFreezeTokenSha256 -Path $resolvedFreezePath
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvironmentFilePath `
|
|
-ExpectedSha256 $priorEnvironmentFileSha256 `
|
|
-Role "prior API"
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $stableEnvironmentFilePath `
|
|
-ExpectedSha256 $stableEnvironmentFileSha256 `
|
|
-Role "new release"
|
|
Assert-OfflinedRuntimeRaceGate `
|
|
-TaskSnapshot $taskSnapshot `
|
|
-ConfigPath $resolvedCloudflaredConfigPath
|
|
$initializerPayload = ((@($initializerOutput) -join "").Trim()) | ConvertFrom-Json
|
|
if (
|
|
$null -eq $initializerPayload -or
|
|
$initializerPayload.status -ne "initialized" -or
|
|
[string]$initializerPayload.manifest_sha256 -notmatch "^[0-9a-f]{64}$" -or
|
|
[string]$initializerPayload.offline_quiescence_receipt_sha256 -notmatch "^[0-9a-f]{64}$" -or
|
|
[string]$initializerPayload.write_freeze_token_sha256 -notmatch "^[0-9a-f]{64}$" -or
|
|
[int]$initializerPayload.preserved_object_count -ne $ExpectedPreservedObjectCount -or
|
|
[long]$initializerPayload.preserved_total_size_bytes -ne
|
|
$ExpectedPreservedTotalSizeBytes -or
|
|
[string]$initializerPayload.preserved_inventory_sha256 -cne $ExpectedPreservedInventorySha256
|
|
) {
|
|
throw "Offline initializer did not return its privacy-safe proof"
|
|
}
|
|
$initializerRequiredObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $initializerPayload `
|
|
-Name "required_object_count" `
|
|
-Role "Offline initializer"
|
|
$initializerReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $initializerPayload `
|
|
-Name "database_reference_count" `
|
|
-Role "Offline initializer"
|
|
if (
|
|
$initializerRequiredObjectCount -gt $initializerReferenceCount -or
|
|
$initializerReferenceCount -ne $ExpectedReferenceCount
|
|
) {
|
|
throw "Offline initializer DB inventory counts drifted"
|
|
}
|
|
$initializerPreservedDecodeCounts = Get-PreservedDecodeCountProof `
|
|
-Payload $initializerPayload `
|
|
-PreservedObjectCount $ExpectedPreservedObjectCount `
|
|
-Role "Offline initializer"
|
|
$initializerRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof `
|
|
-Payload $initializerPayload `
|
|
-RequiredObjectCount $initializerRequiredObjectCount `
|
|
-RequiredReferenceCount $initializerReferenceCount `
|
|
-Role "Offline initializer"
|
|
if (
|
|
$initializerPreservedDecodeCounts.ValidCount -ne
|
|
$preservedProbeDecodeCounts.ValidCount -or
|
|
$initializerPreservedDecodeCounts.InvalidCount -ne
|
|
$preservedProbeDecodeCounts.InvalidCount
|
|
) {
|
|
throw "Offline initializer decode inventory drifted from preflight"
|
|
}
|
|
$manifestSha256 = [string]$initializerPayload.manifest_sha256
|
|
$quiescenceReceiptSha256 = [string]$initializerPayload.offline_quiescence_receipt_sha256
|
|
if ([string]$initializerPayload.write_freeze_token_sha256 -cne $freezeTokenSha256) {
|
|
throw "Offline initializer write-freeze proof drift"
|
|
}
|
|
$manifestPath = Join-Path $resolvedPrivateStateDir "public-avatar-upload-$manifestSha256.json"
|
|
$quiescenceReceiptPath = Join-Path $resolvedPrivateStateDir "public-upload-quiescence-$quiescenceReceiptSha256.json"
|
|
$resolvedUploadRoot = Resolve-PublicRuntimeUploadRoot `
|
|
-SourceRoot $resolvedStableSourceRoot `
|
|
-UploadRoot $UserUploadDir `
|
|
-ProbeWritable
|
|
$manifestProof = Test-PublicRuntimeUploadManifest `
|
|
-PythonPath $resolvedPythonPath `
|
|
-ProbePath $manifestProbe `
|
|
-UploadRoot $resolvedUploadRoot `
|
|
-ManifestPath $manifestPath `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedFreezePath
|
|
if (-not $manifestProof.Ok) {
|
|
throw "Offline initializer manifest validation failed"
|
|
}
|
|
if (
|
|
[string]$manifestProof.Payload.database_target_sha256 -cne
|
|
$priorDatabaseTargetSha256
|
|
) {
|
|
throw "Offline initializer database target drifted from the prior API"
|
|
}
|
|
if (
|
|
[int]$manifestProof.Payload.preserved_object_count -ne
|
|
$ExpectedPreservedObjectCount -or
|
|
[long]$manifestProof.Payload.preserved_total_size_bytes -ne
|
|
$ExpectedPreservedTotalSizeBytes -or
|
|
[string]$manifestProof.Payload.preserved_object_set_sha256 -cne
|
|
$ExpectedPreservedInventorySha256
|
|
) {
|
|
throw "Offline initializer preserved inventory proof drifted"
|
|
}
|
|
$manifestRequiredObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $manifestProof.Payload `
|
|
-Name "required_object_count" `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestRequiredReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $manifestProof.Payload `
|
|
-Name "required_reference_count" `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestCurrentObjectCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $manifestProof.Payload `
|
|
-Name "current_object_count" `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestCurrentReferenceCount = Get-RequiredPrivacySafeCount `
|
|
-Payload $manifestProof.Payload `
|
|
-Name "current_reference_count" `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestPreservedDecodeCounts = Get-PreservedDecodeCountProof `
|
|
-Payload $manifestProof.Payload `
|
|
-PreservedObjectCount $ExpectedPreservedObjectCount `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestRequiredDecodeCounts = Get-RequiredDecodeInvalidCountProof `
|
|
-Payload $manifestProof.Payload `
|
|
-RequiredObjectCount $manifestRequiredObjectCount `
|
|
-RequiredReferenceCount $manifestRequiredReferenceCount `
|
|
-Role "Offline initializer manifest validator"
|
|
$manifestCurrentDecodeCounts = Get-CurrentDecodeInvalidCountProof `
|
|
-Payload $manifestProof.Payload `
|
|
-CurrentObjectCount $manifestCurrentObjectCount `
|
|
-CurrentReferenceCount $manifestCurrentReferenceCount `
|
|
-Role "Offline initializer manifest validator"
|
|
if (
|
|
$manifestRequiredObjectCount -ne $initializerRequiredObjectCount -or
|
|
$manifestRequiredReferenceCount -ne $initializerReferenceCount -or
|
|
$manifestCurrentObjectCount -ne $initializerRequiredObjectCount -or
|
|
$manifestCurrentReferenceCount -ne $initializerReferenceCount -or
|
|
$manifestPreservedDecodeCounts.ValidCount -ne
|
|
$initializerPreservedDecodeCounts.ValidCount -or
|
|
$manifestPreservedDecodeCounts.InvalidCount -ne
|
|
$initializerPreservedDecodeCounts.InvalidCount -or
|
|
$manifestRequiredDecodeCounts.ObjectCount -ne
|
|
$initializerRequiredDecodeCounts.ObjectCount -or
|
|
$manifestRequiredDecodeCounts.ReferenceCount -ne
|
|
$initializerRequiredDecodeCounts.ReferenceCount -or
|
|
$manifestCurrentDecodeCounts.ObjectCount -ne
|
|
$initializerRequiredDecodeCounts.ObjectCount -or
|
|
$manifestCurrentDecodeCounts.ReferenceCount -ne
|
|
$initializerRequiredDecodeCounts.ReferenceCount
|
|
) {
|
|
throw "Offline initializer decode proof drifted across preflight, manifest, or current DB"
|
|
}
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:new_api_frozen
|
|
$failureStage = "new_api_frozen"
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $WhisperPort `
|
|
-Expected $whisperIdentity `
|
|
-Role "whisper sidecar"
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $MeloTtsPort `
|
|
-Expected $meloTtsIdentity `
|
|
-Role "MeloTTS sidecar"
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $priorEnvironmentFilePath `
|
|
-ExpectedSha256 $priorEnvironmentFileSha256 `
|
|
-Role "prior API"
|
|
Assert-EnvironmentFilePinned `
|
|
-Path $stableEnvironmentFilePath `
|
|
-ExpectedSha256 $stableEnvironmentFileSha256 `
|
|
-Role "new release"
|
|
Assert-OfflinedRuntimeRaceGate `
|
|
-TaskSnapshot $taskSnapshot `
|
|
-ConfigPath $resolvedCloudflaredConfigPath
|
|
$newApiLaunchAttempted = $true
|
|
$startOutput = @(
|
|
& $startScript `
|
|
-Workspace $resolvedStableSourceRoot `
|
|
-ApiPort $ApiPort `
|
|
-Python $resolvedPythonPath `
|
|
-UserUploadDir $resolvedUploadRoot `
|
|
-UserUploadManifestPath $manifestPath `
|
|
-ExpectedUserUploadManifestSha256 $manifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-ForceApiRestart `
|
|
-SkipEngineRestart `
|
|
-SkipWebRestart `
|
|
-SkipCloudflaredRestart `
|
|
-ExpectedSourceCommit $ExpectedSourceCommit `
|
|
-ExpectedSourceTree $ExpectedSourceTree `
|
|
-ExpectedPythonSha256 $ExpectedPythonSha256 `
|
|
-OfflineBootstrapQuiescenceReceiptPath $quiescenceReceiptPath `
|
|
-ExpectedOfflineBootstrapQuiescenceReceiptSha256 $quiescenceReceiptSha256 `
|
|
-ExpectedOfflineBootstrapLegacySourceCommit $ExpectedLegacySourceCommit `
|
|
-ExpectedOfflineBootstrapLegacySourceTree $ExpectedLegacySourceTree `
|
|
-RecoveryLockPath $bootstrapLock.Path `
|
|
-InheritedRecoveryLockReceiptPath $bootstrapLock.Path `
|
|
-ExpectedInheritedRecoveryLockReceiptSha256 $bootstrapLock.ReceiptSha256 `
|
|
-CoordinatedTaskNames $CoordinatedTaskNames
|
|
)
|
|
$null = $startOutput
|
|
$newApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort
|
|
$newApiIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $newApiPid `
|
|
-Role "new API" `
|
|
-ExpectedCwd (Join-Path $resolvedStableSourceRoot "apps\api") `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-ApiIdentityContract -Identity $newApiIdentity
|
|
if ($newApiIdentity.executable_sha256 -cne $ExpectedPythonSha256) {
|
|
throw "New API Python identity drift"
|
|
}
|
|
Assert-NewApiEnvironmentMatches `
|
|
-Expected $requiredApiEnvironment `
|
|
-ActualIdentity $newApiIdentity
|
|
$localUploadProof = Test-PublicRuntimeApiUploadRoot `
|
|
-PythonPath $resolvedPythonPath `
|
|
-ProbePath $uploadProbe `
|
|
-ExpectedUploadRoot $resolvedUploadRoot `
|
|
-ExpectedApiCwd (Join-Path $resolvedStableSourceRoot "apps\api") `
|
|
-ExpectedManifestPath $manifestPath `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedWriteFreezePath $resolvedFreezePath `
|
|
-ExpectedDatabaseTargetSha256 ([string]$manifestProof.Payload.database_target_sha256) `
|
|
-ApiPort $ApiPort
|
|
if (-not $localUploadProof.Ok) {
|
|
throw "New API upload-root identity validation failed"
|
|
}
|
|
$null = Wait-ApiHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec $HealthTimeoutSeconds `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedFreezeActive $true `
|
|
-ExpectedFreezeTokenSha256 $freezeTokenSha256
|
|
Wait-GoogleAuthContract `
|
|
-Uri "http://127.0.0.1:$ApiPort/auth/config" `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:new_tunnel_public_frozen
|
|
$failureStage = "new_tunnel_public_frozen"
|
|
Assert-TunnelAbsent `
|
|
-ConfigPath $resolvedCloudflaredConfigPath `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredConfigPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredConfigSha256 `
|
|
-Role "cloudflared config"
|
|
$tunnelLogSuffix = [Guid]::NewGuid().ToString("N")
|
|
$newTunnelProcess = Start-Process `
|
|
-WindowStyle Hidden `
|
|
-FilePath $resolvedCloudflaredPath `
|
|
-ArgumentList @("tunnel", "--config", $resolvedCloudflaredConfigPath, "run") `
|
|
-WorkingDirectory $resolvedStableSourceRoot `
|
|
-RedirectStandardOutput (Join-Path $resolvedPrivateStateDir "new-tunnel-$tunnelLogSuffix.out.log") `
|
|
-RedirectStandardError (Join-Path $resolvedPrivateStateDir "new-tunnel-$tunnelLogSuffix.err.log") `
|
|
-PassThru
|
|
$newTunnelIdentity = Wait-ProcessIdentity `
|
|
-ProcessId $newTunnelProcess.Id `
|
|
-Role "new tunnel" `
|
|
-ExpectedCwd $resolvedStableSourceRoot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if ($newTunnelIdentity.executable_sha256 -cne $ExpectedCloudflaredSha256) {
|
|
throw "New tunnel executable identity drift"
|
|
}
|
|
if (-not (Test-ArgumentPair `
|
|
-Arguments @($newTunnelIdentity.argument_list) `
|
|
-Name "--config" `
|
|
-Value $resolvedCloudflaredConfigPath
|
|
)) {
|
|
throw "New tunnel config identity drift"
|
|
}
|
|
$null = Wait-ApiHealth `
|
|
-Uri $PublicHealthUrl `
|
|
-TimeoutSec $HealthTimeoutSeconds `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedFreezeActive $true `
|
|
-ExpectedFreezeTokenSha256 $freezeTokenSha256
|
|
Wait-GoogleAuthContract `
|
|
-Uri $publicAuthConfigUrl `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
Assert-CurrentTunnelIdentity `
|
|
-ConfigPath $resolvedCloudflaredConfigPath `
|
|
-Expected $newTunnelIdentity
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $ApiPort `
|
|
-Expected $newApiIdentity `
|
|
-Role "new API"
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:no_rollback_boundary
|
|
$failureStage = "no_rollback_boundary"
|
|
$noRollback = $true
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:write_release
|
|
$failureStage = "write_release"
|
|
Remove-OwnedFreezeByHash `
|
|
-Path $resolvedFreezePath `
|
|
-ExpectedTokenSha256 $freezeTokenSha256
|
|
$null = Wait-ApiHealth `
|
|
-Uri "http://127.0.0.1:$ApiPort/health" `
|
|
-TimeoutSec $HealthTimeoutSeconds `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedFreezeActive $false
|
|
$null = Wait-ApiHealth `
|
|
-Uri $PublicHealthUrl `
|
|
-TimeoutSec $HealthTimeoutSeconds `
|
|
-ExpectedManifestSha256 $manifestSha256 `
|
|
-ExpectedFreezeActive $false
|
|
Wait-GoogleAuthContract `
|
|
-Uri "http://127.0.0.1:$ApiPort/auth/config" `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
Wait-GoogleAuthContract `
|
|
-Uri $publicAuthConfigUrl `
|
|
-TimeoutSec $HealthTimeoutSeconds
|
|
Assert-CurrentTunnelIdentity `
|
|
-ConfigPath $resolvedCloudflaredConfigPath `
|
|
-Expected $newTunnelIdentity
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $ApiPort `
|
|
-Expected $newApiIdentity `
|
|
-Role "new API"
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $WhisperPort `
|
|
-Expected $whisperIdentity `
|
|
-Role "whisper sidecar"
|
|
Assert-ListenerIdentityUnchanged `
|
|
-Port $MeloTtsPort `
|
|
-Expected $meloTtsIdentity `
|
|
-Role "MeloTTS sidecar"
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:cutover_receipt_publish
|
|
$failureStage = "cutover_receipt_publish"
|
|
Assert-PublicRuntimeTasksDisabledAndIdle `
|
|
-Snapshot $taskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $disabledOriginalTaskDefinitionSnapshot
|
|
$cutoverReceipt = [ordered]@{
|
|
schema_version = "vignette.legacy-public-upload-bootstrap.v1"
|
|
status = "passed"
|
|
operational_success = $false
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ")
|
|
source = [ordered]@{
|
|
git_commit = $ExpectedSourceCommit
|
|
git_tree = $ExpectedSourceTree
|
|
}
|
|
storage = [ordered]@{
|
|
migration_manifest_sha256 = $manifestSha256
|
|
quiescence_receipt_sha256 = $quiescenceReceiptSha256
|
|
database_target_sha256 = [string]$manifestProof.Payload.database_target_sha256
|
|
preserved_object_count = [int]$manifestProof.Payload.preserved_object_count
|
|
preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes
|
|
preserved_object_set_sha256 = [string]$manifestProof.Payload.preserved_object_set_sha256
|
|
preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount
|
|
preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount
|
|
required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount
|
|
required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount
|
|
reference_count = [int]$manifestProof.Payload.current_reference_count
|
|
unique_object_count = [int]$manifestProof.Payload.current_object_count
|
|
reference_set_sha256 = [string]$manifestProof.Payload.current_reference_set_sha256
|
|
current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount
|
|
current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount
|
|
write_freeze_released = $true
|
|
required_environment_sha256 = $requiredEnvironmentDigest
|
|
}
|
|
processes = [ordered]@{
|
|
api = ConvertTo-SafeProcessIdentity -Identity $newApiIdentity
|
|
tunnel = ConvertTo-SafeProcessIdentity -Identity $newTunnelIdentity
|
|
}
|
|
validation = [ordered]@{
|
|
local_frozen = $true
|
|
public_frozen = $true
|
|
local_unfrozen = $true
|
|
public_unfrozen = $true
|
|
no_rollback_boundary_crossed = $true
|
|
}
|
|
task_maintenance = [ordered]@{
|
|
disabled = $true
|
|
idle = $true
|
|
restored = $false
|
|
task_count = @($taskSnapshot).Count
|
|
original_definition_set_sha256 = [string]$originalTaskDefinitionSnapshot.set_sha256
|
|
disabled_pre_cutover_definition_set_sha256 = [string]$disabledOriginalTaskDefinitionSnapshot.set_sha256
|
|
}
|
|
privacy = [ordered]@{
|
|
raw_paths_recorded = $false
|
|
raw_urls_recorded = $false
|
|
raw_user_ids_recorded = $false
|
|
raw_filenames_recorded = $false
|
|
raw_secrets_recorded = $false
|
|
}
|
|
}
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $disabledOriginalTaskDefinitionSnapshot
|
|
$cutoverReceiptSha256 = Write-PrivacySafeReceiptCreateOnly `
|
|
-Path $resolvedCutoverReceiptPath `
|
|
-Payload $cutoverReceipt
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:task_maintenance_exit
|
|
$failureStage = "task_definition_cutover"
|
|
Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot
|
|
Assert-FileSha256 `
|
|
-Path $resolvedPythonPath `
|
|
-ExpectedSha256 $ExpectedPythonSha256 `
|
|
-Role "Python before task definition cutover"
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredSha256 `
|
|
-Role "cloudflared before task definition cutover"
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredConfigPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredConfigSha256 `
|
|
-Role "cloudflared config before task definition cutover"
|
|
$bootInstallAction = {
|
|
& $bootTaskInstaller `
|
|
-StableSourceRoot $resolvedStableSourceRoot `
|
|
-Python $resolvedPythonPath `
|
|
-Cloudflared $resolvedCloudflaredPath `
|
|
-CloudflaredConfig $resolvedCloudflaredConfigPath `
|
|
-UserUploadDir $resolvedUploadRoot `
|
|
-UserUploadManifestPath $manifestPath `
|
|
-ExpectedUserUploadManifestSha256 $manifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-TaskName "VignettePublicRuntime" `
|
|
-InitiallyDisabled
|
|
}.GetNewClosure()
|
|
$watchdogInstallAction = {
|
|
& $watchdogTaskInstaller `
|
|
-StableSourceRoot $resolvedStableSourceRoot `
|
|
-TaskName "VignettePublicRuntimeWatchdog" `
|
|
-Python $resolvedPythonPath `
|
|
-UserUploadDir $resolvedUploadRoot `
|
|
-UserUploadManifestPath $manifestPath `
|
|
-ExpectedUserUploadManifestSha256 $manifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-Cloudflared $resolvedCloudflaredPath `
|
|
-CloudflaredConfig $resolvedCloudflaredConfigPath `
|
|
-PublicHealthUrl $PublicHealthUrl `
|
|
-InitiallyDisabled
|
|
}.GetNewClosure()
|
|
Invoke-PublicRuntimeTaskDefinitionInstallerPairDisabled `
|
|
-BootInstaller $bootInstallAction `
|
|
-WatchdogInstaller $watchdogInstallAction `
|
|
-MaintenanceSnapshot $taskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-PublicRuntimeTasksDisabledAndIdle `
|
|
-Snapshot $taskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
$newDisabledTaskDefinitionSnapshot = Get-PublicRuntimeTaskDefinitionSnapshot
|
|
$null = Assert-NewPublicRuntimeTaskDefinitionsPinned `
|
|
-Snapshot $newDisabledTaskDefinitionSnapshot `
|
|
-StableSourceRoot $resolvedStableSourceRoot `
|
|
-ExpectedSourceCommit $ExpectedSourceCommit `
|
|
-ExpectedSourceTree $ExpectedSourceTree `
|
|
-PythonPath $resolvedPythonPath `
|
|
-UserUploadDir $resolvedUploadRoot `
|
|
-UserUploadManifestPath $manifestPath `
|
|
-ExpectedUserUploadManifestSha256 $manifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-CloudflaredPath $resolvedCloudflaredPath `
|
|
-CloudflaredConfigPath $resolvedCloudflaredConfigPath `
|
|
-PublicHealthUrl $PublicHealthUrl
|
|
Assert-BootstrapSourceProvenance -Root $resolvedStableSourceRoot
|
|
Assert-FileSha256 `
|
|
-Path $resolvedPythonPath `
|
|
-ExpectedSha256 $ExpectedPythonSha256 `
|
|
-Role "Python after task definition cutover"
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredSha256 `
|
|
-Role "cloudflared after task definition cutover"
|
|
Assert-FileSha256 `
|
|
-Path $resolvedCloudflaredConfigPath `
|
|
-ExpectedSha256 $ExpectedCloudflaredConfigSha256 `
|
|
-Role "cloudflared config after task definition cutover"
|
|
|
|
$failureStage = "task_maintenance_exit"
|
|
$operationalTaskDefinitionSnapshot = Enable-NewPublicRuntimeTaskDefinitions `
|
|
-DisabledSnapshot $newDisabledTaskDefinitionSnapshot
|
|
$null = Assert-NewPublicRuntimeTaskDefinitionsPinned `
|
|
-Snapshot $operationalTaskDefinitionSnapshot `
|
|
-StableSourceRoot $resolvedStableSourceRoot `
|
|
-ExpectedSourceCommit $ExpectedSourceCommit `
|
|
-ExpectedSourceTree $ExpectedSourceTree `
|
|
-PythonPath $resolvedPythonPath `
|
|
-UserUploadDir $resolvedUploadRoot `
|
|
-UserUploadManifestPath $manifestPath `
|
|
-ExpectedUserUploadManifestSha256 $manifestSha256 `
|
|
-UserUploadWriteFreezePath $resolvedFreezePath `
|
|
-CloudflaredPath $resolvedCloudflaredPath `
|
|
-CloudflaredConfigPath $resolvedCloudflaredConfigPath `
|
|
-PublicHealthUrl $PublicHealthUrl `
|
|
-AllowEnabled
|
|
$taskRestoreTruth = Get-PublicRuntimeTaskTruth `
|
|
-Snapshot $taskSnapshot `
|
|
-ExpectedRestored
|
|
if (-not [bool]$taskRestoreTruth.expected_state_restored) {
|
|
throw "Task restoration truth proof failed"
|
|
}
|
|
|
|
# LEGACY_BOOTSTRAP_STAGE:task_recovery_receipt_publish
|
|
$failureStage = "task_recovery_receipt_publish"
|
|
$taskRecoveryReceipt = [ordered]@{
|
|
schema_version = "vignette.legacy-public-upload-task-recovery.v1"
|
|
status = "passed"
|
|
operational_success = $true
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ")
|
|
cutover_receipt_sha256 = $cutoverReceiptSha256
|
|
preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes
|
|
preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount
|
|
preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount
|
|
required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount
|
|
required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount
|
|
current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount
|
|
current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount
|
|
tasks_restored = $true
|
|
task_count = @($taskSnapshot).Count
|
|
task_enabled_count = [int]$taskRestoreTruth.enabled_count
|
|
task_running_count = [int]$taskRestoreTruth.running_count
|
|
task_definitions = [ordered]@{
|
|
original_set_sha256 = [string]$originalTaskDefinitionSnapshot.set_sha256
|
|
installed_disabled_set_sha256 = [string]$newDisabledTaskDefinitionSnapshot.set_sha256
|
|
operational_set_sha256 = [string]$operationalTaskDefinitionSnapshot.set_sha256
|
|
}
|
|
privacy = [ordered]@{
|
|
raw_paths_recorded = $false
|
|
raw_urls_recorded = $false
|
|
raw_user_ids_recorded = $false
|
|
raw_filenames_recorded = $false
|
|
raw_secrets_recorded = $false
|
|
}
|
|
}
|
|
$null = Assert-PublicRuntimeTaskDefinitionSnapshotCurrent `
|
|
-ExpectedSnapshot $operationalTaskDefinitionSnapshot
|
|
$taskRecoveryReceiptSha256 = Write-PrivacySafeReceiptCreateOnly `
|
|
-Path $resolvedTaskRecoveryReceiptPath `
|
|
-Payload $taskRecoveryReceipt
|
|
$taskMaintenanceEntered = $false
|
|
} catch {
|
|
$rollbackSucceeded = $false
|
|
$tasksRestored = $false
|
|
if (
|
|
$freezeTokenSha256 -notmatch "^[0-9a-f]{64}$" -and
|
|
$freezeAbsentBeforeInitializer -and
|
|
[System.IO.File]::Exists($resolvedFreezePath)
|
|
) {
|
|
try {
|
|
$freezeTokenSha256 = Get-OwnedFreezeTokenSha256 -Path $resolvedFreezePath
|
|
} catch {
|
|
$freezeTokenSha256 = ""
|
|
}
|
|
}
|
|
if ($newApiLaunchAttempted -and $null -eq $newApiIdentity) {
|
|
try {
|
|
$candidateApiPid = Get-ExactLoopbackListenerPid -Port $ApiPort
|
|
$candidateApi = Wait-ProcessIdentity `
|
|
-ProcessId $candidateApiPid `
|
|
-Role "owned new API recovery" `
|
|
-ExpectedCwd (Join-Path $resolvedStableSourceRoot "apps\api") `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
Assert-ApiIdentityContract -Identity $candidateApi
|
|
if ($candidateApi.executable_sha256 -cne $ExpectedPythonSha256) {
|
|
throw "Owned new API recovery identity drift"
|
|
}
|
|
$newApiIdentity = $candidateApi
|
|
} catch {
|
|
$newApiIdentity = $null
|
|
}
|
|
}
|
|
if ($null -eq $newTunnelIdentity -and $null -ne $newTunnelProcess) {
|
|
try {
|
|
$candidateTunnel = Wait-ProcessIdentity `
|
|
-ProcessId $newTunnelProcess.Id `
|
|
-Role "owned new tunnel recovery" `
|
|
-ExpectedCwd $resolvedStableSourceRoot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
if (
|
|
$candidateTunnel.executable_sha256 -cne $ExpectedCloudflaredSha256 -or
|
|
-not (Test-ArgumentPair `
|
|
-Arguments @($candidateTunnel.argument_list) `
|
|
-Name "--config" `
|
|
-Value $resolvedCloudflaredConfigPath)
|
|
) {
|
|
throw "Owned new tunnel recovery identity drift"
|
|
}
|
|
$newTunnelIdentity = $candidateTunnel
|
|
} catch {
|
|
$newTunnelIdentity = $null
|
|
}
|
|
}
|
|
if (-not $noRollback) {
|
|
try {
|
|
if ($runtimeMutationStarted) {
|
|
Restore-LegacyRuntime `
|
|
-PriorApi $priorApiIdentity `
|
|
-PriorTunnel $priorTunnelIdentity `
|
|
-ConfigPath $resolvedCloudflaredConfigPath `
|
|
-FreezePath $resolvedFreezePath `
|
|
-FreezeTokenSha256 $freezeTokenSha256 `
|
|
-PrivateStateRoot $resolvedPrivateStateDir `
|
|
-PriorEnvironmentPath $priorEnvironmentFilePath `
|
|
-PriorEnvironmentSha256 $priorEnvironmentFileSha256 `
|
|
-OwnedNewApi $newApiIdentity `
|
|
-OwnedNewTunnel $newTunnelIdentity
|
|
} elseif ([System.IO.File]::Exists($resolvedFreezePath)) {
|
|
if ($freezeTokenSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
throw "Pre-mutation freeze ownership is unknown"
|
|
}
|
|
Remove-OwnedFreezeByHash `
|
|
-Path $resolvedFreezePath `
|
|
-ExpectedTokenSha256 $freezeTokenSha256
|
|
}
|
|
if ($taskMaintenanceEntered) {
|
|
Exit-PublicRuntimeTaskMaintenance -Snapshot $taskSnapshot
|
|
$restoredTruth = Get-PublicRuntimeTaskTruth `
|
|
-Snapshot $taskSnapshot `
|
|
-ExpectedRestored
|
|
$tasksRestored = [bool]$restoredTruth.expected_state_restored
|
|
if (-not $tasksRestored) {
|
|
throw "Task restoration truth proof failed"
|
|
}
|
|
$taskMaintenanceEntered = $false
|
|
}
|
|
$rollbackSucceeded = $true
|
|
} catch {
|
|
$rollbackSucceeded = $false
|
|
}
|
|
}
|
|
if ($taskMaintenanceEntered) {
|
|
try {
|
|
Suspend-PublicRuntimeTasks `
|
|
-Snapshot $taskSnapshot `
|
|
-TimeoutSec $ProcessStopTimeoutSeconds
|
|
} catch {
|
|
}
|
|
}
|
|
$taskTruth = [ordered]@{
|
|
enabled_count = 0
|
|
running_count = 0
|
|
all_disabled_and_idle = $false
|
|
expected_state_restored = $false
|
|
}
|
|
if (@($taskSnapshot).Count -gt 0) {
|
|
try {
|
|
$taskTruth = Get-PublicRuntimeTaskTruth -Snapshot $taskSnapshot
|
|
} catch {
|
|
}
|
|
}
|
|
try {
|
|
$failureReceipt = [ordered]@{
|
|
schema_version = "vignette.legacy-public-upload-bootstrap-failure.v1"
|
|
status = "failed"
|
|
captured_at_utc = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ")
|
|
failure_stage = $failureStage
|
|
no_rollback_boundary_crossed = $noRollback
|
|
rollback_attempted = $runtimeMutationStarted -and -not $noRollback
|
|
rollback_succeeded = $rollbackSucceeded
|
|
tasks_restored = $tasksRestored
|
|
tasks_remain_disabled = [bool]$taskTruth.all_disabled_and_idle
|
|
task_enabled_count = [int]$taskTruth.enabled_count
|
|
task_running_count = [int]$taskTruth.running_count
|
|
cutover_receipt_published = ($cutoverReceiptSha256 -match "^[0-9a-f]{64}$")
|
|
cutover_receipt_sha256 = $cutoverReceiptSha256
|
|
privacy = [ordered]@{
|
|
raw_paths_recorded = $false
|
|
raw_urls_recorded = $false
|
|
raw_user_ids_recorded = $false
|
|
raw_filenames_recorded = $false
|
|
raw_secrets_recorded = $false
|
|
}
|
|
}
|
|
$failureJson = ConvertTo-Json -InputObject $failureReceipt -Depth 6 -Compress
|
|
$failureDigest = Get-Utf8Sha256 -Value ($failureJson + [Environment]::NewLine)
|
|
$failureReceiptPath = Join-Path $resolvedPrivateStateDir "legacy-bootstrap-failure-$failureDigest.json"
|
|
$null = Write-PrivacySafeReceiptCreateOnly `
|
|
-Path $failureReceiptPath `
|
|
-Payload $failureReceipt
|
|
} catch {
|
|
}
|
|
if ($noRollback) {
|
|
throw "Legacy public upload bootstrap failed after the no-rollback boundary; tasks remain disabled"
|
|
}
|
|
if ($rollbackSucceeded) {
|
|
throw "Legacy public upload bootstrap failed before the boundary; prior runtime and tasks were restored"
|
|
}
|
|
throw "Legacy public upload bootstrap failed closed before the boundary; tasks remain disabled"
|
|
} finally {
|
|
if ($null -ne $callerRequiredEnvironmentSnapshot) {
|
|
Restore-SelectedProcessEnvironment `
|
|
-Snapshot $callerRequiredEnvironmentSnapshot
|
|
}
|
|
if ($null -ne $bootstrapLock -and $null -ne $bootstrapLock.Stream) {
|
|
$bootstrapLock.Stream.Dispose()
|
|
}
|
|
}
|
|
|
|
$result = [ordered]@{
|
|
status = "passed"
|
|
cutover_receipt_sha256 = $cutoverReceiptSha256
|
|
task_recovery_receipt_sha256 = $taskRecoveryReceiptSha256
|
|
preserved_total_size_bytes = [long]$manifestProof.Payload.preserved_total_size_bytes
|
|
preserved_decode_valid_count = [int]$manifestPreservedDecodeCounts.ValidCount
|
|
preserved_decode_invalid_count = [int]$manifestPreservedDecodeCounts.InvalidCount
|
|
required_decode_invalid_object_count = [int]$manifestRequiredDecodeCounts.ObjectCount
|
|
required_decode_invalid_reference_count = [int]$manifestRequiredDecodeCounts.ReferenceCount
|
|
current_decode_invalid_object_count = [int]$manifestCurrentDecodeCounts.ObjectCount
|
|
current_decode_invalid_reference_count = [int]$manifestCurrentDecodeCounts.ReferenceCount
|
|
}
|
|
Write-Output (ConvertTo-Json -InputObject $result -Compress)
|