param( [switch]$ConfirmCutover, [string]$CandidateContainer = 'vignette-recovered-prod-candidate', [string]$RecoveredVolume = 'vignette_recovered_prod_20260807', [string]$ReleaseWorkspace = 'D:\workspace\vignette-outcome-g7-release-20260807' ) $ErrorActionPreference = 'Stop' [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false) $OutputEncoding = [System.Text.UTF8Encoding]::new($false) if (-not $ConfirmCutover) { throw '공개 DB 전환은 -ConfirmCutover를 명시해야 한다.' } $watchdogTask = 'VignettePublicRuntimeWatchdog' $activeName = 'vignette-dev-db' $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $preservedName = "vignette-dev-db-pre-recovery-$stamp" $failedName = "vignette-recovered-cutover-failed-$stamp" $startScript = Join-Path $ReleaseWorkspace 'scripts\start-public-runtime.ps1' $initPath = Join-Path (Split-Path -Parent $PSScriptRoot) 'infra\db\init' $cutoverStarted = $false $watchdogDisabled = $false function Invoke-DockerChecked([string[]]$Arguments, [string]$Message) { & docker @Arguments if ($LASTEXITCODE -ne 0) { throw "$Message (exit=$LASTEXITCODE)" } } function Get-Container([string]$Name) { $raw = & docker inspect $Name if ($LASTEXITCODE -ne 0) { throw "container inspect 실패: $Name" } $items = $raw | ConvertFrom-Json if ($items.Count -ne 1) { throw "container inspect 결과 불일치: $Name" } return $items[0] } function Get-ContainerEnv([object]$Container) { $values = @{} foreach ($entry in $Container.Config.Env) { $parts = $entry -split '=', 2 $values[$parts[0]] = $parts[1] } return $values } function Test-ContainerNameExists([string]$Name) { $names = @(& docker ps -a --format '{{.Names}}') if ($LASTEXITCODE -ne 0) { throw 'docker container 목록 조회 실패' } return $names -contains $Name } function Wait-Healthy([string]$Name, [int]$Attempts = 60) { for ($index = 0; $index -lt $Attempts; $index++) { $state = & docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' $Name 2>$null if ($LASTEXITCODE -eq 0 -and "$state".Trim() -eq 'healthy') { return } Start-Sleep -Seconds 2 } throw "container health timeout: $Name" } function Restart-PublicApi { & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript ` -Workspace $ReleaseWorkspace ` -ForceApiRestart ` -SkipWebRestart ` -SkipCloudflaredRestart ` -SkipEngineRestart if ($LASTEXITCODE -ne 0) { throw "공개 API 재기동 실패: $LASTEXITCODE" } } function Clear-DatabaseEnvironment { foreach ($key in @( 'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DB', 'APP_DB_USER', 'APP_DB_PASSWORD', 'PGDATA' )) { [Environment]::SetEnvironmentVariable($key, $null, 'Process') } } try { if (Test-ContainerNameExists $preservedName) { throw "보존 이름 충돌: $preservedName" } if (Test-ContainerNameExists $failedName) { throw "실패 보존 이름 충돌: $failedName" } $current = Get-Container $activeName $candidate = Get-Container $CandidateContainer if ($current.State.Health.Status -ne 'healthy') { throw '현재 공개 DB가 healthy가 아님' } if ($candidate.State.Health.Status -ne 'healthy') { throw '복구 후보 DB가 healthy가 아님' } if ($current.Config.Image -ne $candidate.Config.Image) { throw '현재/후보 DB 이미지 불일치' } $candidateData = $candidate.Mounts | Where-Object { $_.Destination -eq '/var/lib/postgresql/data' } if ($candidateData.Name -ne $RecoveredVolume -or -not $candidateData.RW) { throw '복구 후보 PGDATA volume 불일치' } $envValues = Get-ContainerEnv $candidate foreach ($key in @( 'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DB', 'APP_DB_USER', 'APP_DB_PASSWORD', 'PGDATA' )) { if (-not $envValues.ContainsKey($key) -or [string]::IsNullOrWhiteSpace($envValues[$key])) { throw "복구 후보 env 누락: $key" } [Environment]::SetEnvironmentVariable($key, $envValues[$key], 'Process') } Disable-ScheduledTask -TaskName $watchdogTask | Out-Null $watchdogDisabled = $true if ((Get-ScheduledTask -TaskName $watchdogTask).Settings.Enabled) { throw 'watchdog 비활성화 실패' } Invoke-DockerChecked @('stop', $activeName) '현재 DB 정지 실패' | Out-Null Invoke-DockerChecked @('rename', $activeName, $preservedName) '현재 DB 보존 rename 실패' | Out-Null $cutoverStarted = $true Invoke-DockerChecked @('stop', $CandidateContainer) '복구 후보 정지 실패' | Out-Null $dockerArgs = @( 'run', '-d', '--name', $activeName, '--restart', 'unless-stopped', '-p', '55432:5432', '-e', 'POSTGRES_USER', '-e', 'POSTGRES_PASSWORD', '-e', 'POSTGRES_DB', '-e', 'APP_DB_USER', '-e', 'APP_DB_PASSWORD', '-e', 'PGDATA', '--mount', ("type=volume,source={0},target=/var/lib/postgresql/data" -f $RecoveredVolume), '--mount', ("type=bind,source={0},target=/docker-entrypoint-initdb.d,readonly" -f $initPath), '--health-cmd', ("pg_isready -U {0} -d {1}" -f $envValues.POSTGRES_USER, $envValues.POSTGRES_DB), '--health-interval', '5s', '--health-timeout', '5s', '--health-retries', '60', $candidate.Config.Image ) Invoke-DockerChecked $dockerArgs '복구 DB 표준 container 생성 실패' | Out-Null Wait-Healthy $activeName $countsSql = @' SELECT (SELECT count(*) FROM app.app_user)::text || '|' || (SELECT count(*) FROM app.sessions)::text || '|' || (SELECT count(*) FROM app.turns)::text || '|' || (SELECT count(*) FROM app.sessions s LEFT JOIN app.app_user u ON u.user_id=s.learner_id WHERE u.user_id IS NULL)::text || '|' || (SELECT count(*) FROM app.turns t LEFT JOIN app.sessions s ON s.id=t.session_id WHERE s.id IS NULL)::text '@ $counts = & docker exec $activeName psql ` -U $envValues.POSTGRES_USER ` -d $envValues.POSTGRES_DB ` -At -v ON_ERROR_STOP=1 ` -c $countsSql if ($LASTEXITCODE -ne 0 -or "$counts".Trim() -ne '84|30|705|0|0') { throw "복구 DB 불변식 불일치: $counts" } Clear-DatabaseEnvironment Restart-PublicApi $localHealth = $null $publicHealth = $null for ($index = 0; $index -lt 45; $index++) { try { $localHealth = Invoke-RestMethod 'http://127.0.0.1:8001/health' -TimeoutSec 5 $publicHealth = Invoke-RestMethod 'https://api-vignette.chanpaca.net/health' -TimeoutSec 8 } catch { $localHealth = $null $publicHealth = $null } $localReady = $null -ne $localHealth -and $localHealth.environment -eq 'prod' -and $localHealth.db -and $localHealth.engine $publicReady = $null -ne $publicHealth -and $publicHealth.environment -eq 'prod' -and $publicHealth.db -and $publicHealth.engine if ($localReady -and $publicReady) { break } Start-Sleep -Seconds 2 } if (-not $localReady -or -not $publicReady) { throw '공개 health가 제한 시간 안에 prod/db/engine ready가 되지 않음' } $openapi = Invoke-RestMethod 'https://api-vignette.chanpaca.net/openapi.json' -TimeoutSec 20 $pathCount = @($openapi.paths.PSObject.Properties).Count if ($pathCount -ne 119) { throw "공개 OpenAPI path 불일치: $pathCount" } Enable-ScheduledTask -TaskName $watchdogTask | Out-Null $watchdogDisabled = $false [PSCustomObject]@{ status = 'PASS' stamp = $stamp preserved_current_container = $preservedName active_container = $activeName active_volume = $RecoveredVolume counts = "$counts".Trim() local_health = $localHealth.status public_health = $publicHealth.status openapi_paths = $pathCount watchdog_enabled = (Get-ScheduledTask -TaskName $watchdogTask).Settings.Enabled } | ConvertTo-Json -Compress } catch { $reason = $_.Exception.Message Write-Output "CUTOVER_ERROR=$reason" Clear-DatabaseEnvironment if ($cutoverStarted) { if (Test-ContainerNameExists $activeName) { & docker stop $activeName 1>$null 2>$null Invoke-DockerChecked @('rename', $activeName, $failedName) '실패 복구 DB 보존 rename 실패' | Out-Null } if (Test-ContainerNameExists $preservedName) { Invoke-DockerChecked @('rename', $preservedName, $activeName) '원래 DB rollback rename 실패' | Out-Null Invoke-DockerChecked @('start', $activeName) '원래 DB rollback start 실패' | Out-Null Wait-Healthy $activeName Restart-PublicApi } } if ($watchdogDisabled) { Enable-ScheduledTask -TaskName $watchdogTask | Out-Null } throw "공개 DB 전환 실패, 원래 DB로 rollback: $reason" }