G0~G8 성과·동맹 측정 OS 작업 일괄 고정
8월 7일까지 워킹트리에만 남아 있던 미커밋 작업을 커밋한다. 여러 사본 폴더(worktree·clone)에 흩어져 있던 중간 스냅샷을 정리하기 전에 원본을 git 이력으로 고정하는 것이 목적이다. - contracts/routes/services: measurement, outcome_trajectory, rupture_repair, deliberate_practice, calibration_transfer, supervision_research, multimodal_alliance, continuous_improvement 계열 신규 모듈과 테스트 - infra/db/init: 07~16 마이그레이션(측정 기반~calibration transfer 실행) - apps/web: 세션 리뷰 카드·관리 화면·E2E 스펙 추가 - docs/ops: G0~G8 라이브 통합·배포·롤백 증거 문서와 evidence JSON/PNG - scripts: smoke·ledger·릴리스 에이전트·NAS 프리뷰 운영 스크립트 engine.public 로그 .bak과 apps/web/test-results 산출물은 커밋에서 제외했다.
This commit is contained in:
Yun Chan
parent
93dd8f82d7
commit
16e791e044
390 changed files with 243188 additions and 499 deletions
218
scripts/cutover-recovered-public-db.ps1
Normal file
218
scripts/cutover-recovered-public-db.ps1
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
param(
|
||||
[switch]$ConfirmCutover,
|
||||
[string]$CandidateContainer = 'vignette-recovered-prod-candidate',
|
||||
[string]$RecoveredVolume = 'vignette_recovered_prod_20260807',
|
||||
[string]$ReleaseWorkspace = 'D:\workspace\vignette-outcome-g7-release-20260807'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)
|
||||
$OutputEncoding = [System.Text.UTF8Encoding]::new($false)
|
||||
|
||||
if (-not $ConfirmCutover) {
|
||||
throw '공개 DB 전환은 -ConfirmCutover를 명시해야 한다.'
|
||||
}
|
||||
|
||||
$watchdogTask = 'VignettePublicRuntimeWatchdog'
|
||||
$activeName = 'vignette-dev-db'
|
||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$preservedName = "vignette-dev-db-pre-recovery-$stamp"
|
||||
$failedName = "vignette-recovered-cutover-failed-$stamp"
|
||||
$startScript = Join-Path $ReleaseWorkspace 'scripts\start-public-runtime.ps1'
|
||||
$initPath = Join-Path (Split-Path -Parent $PSScriptRoot) 'infra\db\init'
|
||||
$cutoverStarted = $false
|
||||
$watchdogDisabled = $false
|
||||
|
||||
function Invoke-DockerChecked([string[]]$Arguments, [string]$Message) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Message (exit=$LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
function Get-Container([string]$Name) {
|
||||
$raw = & docker inspect $Name
|
||||
if ($LASTEXITCODE -ne 0) { throw "container inspect 실패: $Name" }
|
||||
$items = $raw | ConvertFrom-Json
|
||||
if ($items.Count -ne 1) { throw "container inspect 결과 불일치: $Name" }
|
||||
return $items[0]
|
||||
}
|
||||
|
||||
function Get-ContainerEnv([object]$Container) {
|
||||
$values = @{}
|
||||
foreach ($entry in $Container.Config.Env) {
|
||||
$parts = $entry -split '=', 2
|
||||
$values[$parts[0]] = $parts[1]
|
||||
}
|
||||
return $values
|
||||
}
|
||||
|
||||
function Test-ContainerNameExists([string]$Name) {
|
||||
$names = @(& docker ps -a --format '{{.Names}}')
|
||||
if ($LASTEXITCODE -ne 0) { throw 'docker container 목록 조회 실패' }
|
||||
return $names -contains $Name
|
||||
}
|
||||
|
||||
function Wait-Healthy([string]$Name, [int]$Attempts = 60) {
|
||||
for ($index = 0; $index -lt $Attempts; $index++) {
|
||||
$state = & docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' $Name 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and "$state".Trim() -eq 'healthy') { return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
throw "container health timeout: $Name"
|
||||
}
|
||||
|
||||
function Restart-PublicApi {
|
||||
& powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript `
|
||||
-Workspace $ReleaseWorkspace `
|
||||
-ForceApiRestart `
|
||||
-SkipWebRestart `
|
||||
-SkipCloudflaredRestart `
|
||||
-SkipEngineRestart
|
||||
if ($LASTEXITCODE -ne 0) { throw "공개 API 재기동 실패: $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Clear-DatabaseEnvironment {
|
||||
foreach ($key in @(
|
||||
'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DB',
|
||||
'APP_DB_USER', 'APP_DB_PASSWORD', 'PGDATA'
|
||||
)) {
|
||||
[Environment]::SetEnvironmentVariable($key, $null, 'Process')
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (Test-ContainerNameExists $preservedName) { throw "보존 이름 충돌: $preservedName" }
|
||||
if (Test-ContainerNameExists $failedName) { throw "실패 보존 이름 충돌: $failedName" }
|
||||
|
||||
$current = Get-Container $activeName
|
||||
$candidate = Get-Container $CandidateContainer
|
||||
if ($current.State.Health.Status -ne 'healthy') { throw '현재 공개 DB가 healthy가 아님' }
|
||||
if ($candidate.State.Health.Status -ne 'healthy') { throw '복구 후보 DB가 healthy가 아님' }
|
||||
if ($current.Config.Image -ne $candidate.Config.Image) { throw '현재/후보 DB 이미지 불일치' }
|
||||
|
||||
$candidateData = $candidate.Mounts | Where-Object { $_.Destination -eq '/var/lib/postgresql/data' }
|
||||
if ($candidateData.Name -ne $RecoveredVolume -or -not $candidateData.RW) {
|
||||
throw '복구 후보 PGDATA volume 불일치'
|
||||
}
|
||||
|
||||
$envValues = Get-ContainerEnv $candidate
|
||||
foreach ($key in @(
|
||||
'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DB',
|
||||
'APP_DB_USER', 'APP_DB_PASSWORD', 'PGDATA'
|
||||
)) {
|
||||
if (-not $envValues.ContainsKey($key) -or [string]::IsNullOrWhiteSpace($envValues[$key])) {
|
||||
throw "복구 후보 env 누락: $key"
|
||||
}
|
||||
[Environment]::SetEnvironmentVariable($key, $envValues[$key], 'Process')
|
||||
}
|
||||
|
||||
Disable-ScheduledTask -TaskName $watchdogTask | Out-Null
|
||||
$watchdogDisabled = $true
|
||||
if ((Get-ScheduledTask -TaskName $watchdogTask).Settings.Enabled) {
|
||||
throw 'watchdog 비활성화 실패'
|
||||
}
|
||||
|
||||
Invoke-DockerChecked @('stop', $activeName) '현재 DB 정지 실패' | Out-Null
|
||||
Invoke-DockerChecked @('rename', $activeName, $preservedName) '현재 DB 보존 rename 실패' | Out-Null
|
||||
$cutoverStarted = $true
|
||||
Invoke-DockerChecked @('stop', $CandidateContainer) '복구 후보 정지 실패' | Out-Null
|
||||
|
||||
$dockerArgs = @(
|
||||
'run', '-d',
|
||||
'--name', $activeName,
|
||||
'--restart', 'unless-stopped',
|
||||
'-p', '55432:5432',
|
||||
'-e', 'POSTGRES_USER',
|
||||
'-e', 'POSTGRES_PASSWORD',
|
||||
'-e', 'POSTGRES_DB',
|
||||
'-e', 'APP_DB_USER',
|
||||
'-e', 'APP_DB_PASSWORD',
|
||||
'-e', 'PGDATA',
|
||||
'--mount', ("type=volume,source={0},target=/var/lib/postgresql/data" -f $RecoveredVolume),
|
||||
'--mount', ("type=bind,source={0},target=/docker-entrypoint-initdb.d,readonly" -f $initPath),
|
||||
'--health-cmd', ("pg_isready -U {0} -d {1}" -f $envValues.POSTGRES_USER, $envValues.POSTGRES_DB),
|
||||
'--health-interval', '5s',
|
||||
'--health-timeout', '5s',
|
||||
'--health-retries', '60',
|
||||
$candidate.Config.Image
|
||||
)
|
||||
Invoke-DockerChecked $dockerArgs '복구 DB 표준 container 생성 실패' | Out-Null
|
||||
Wait-Healthy $activeName
|
||||
|
||||
$countsSql = @'
|
||||
SELECT (SELECT count(*) FROM app.app_user)::text || '|' ||
|
||||
(SELECT count(*) FROM app.sessions)::text || '|' ||
|
||||
(SELECT count(*) FROM app.turns)::text || '|' ||
|
||||
(SELECT count(*) FROM app.sessions s LEFT JOIN app.app_user u ON u.user_id=s.learner_id WHERE u.user_id IS NULL)::text || '|' ||
|
||||
(SELECT count(*) FROM app.turns t LEFT JOIN app.sessions s ON s.id=t.session_id WHERE s.id IS NULL)::text
|
||||
'@
|
||||
$counts = & docker exec $activeName psql `
|
||||
-U $envValues.POSTGRES_USER `
|
||||
-d $envValues.POSTGRES_DB `
|
||||
-At -v ON_ERROR_STOP=1 `
|
||||
-c $countsSql
|
||||
if ($LASTEXITCODE -ne 0 -or "$counts".Trim() -ne '84|30|705|0|0') {
|
||||
throw "복구 DB 불변식 불일치: $counts"
|
||||
}
|
||||
|
||||
Clear-DatabaseEnvironment
|
||||
Restart-PublicApi
|
||||
|
||||
$localHealth = $null
|
||||
$publicHealth = $null
|
||||
for ($index = 0; $index -lt 45; $index++) {
|
||||
try {
|
||||
$localHealth = Invoke-RestMethod 'http://127.0.0.1:8001/health' -TimeoutSec 5
|
||||
$publicHealth = Invoke-RestMethod 'https://api-vignette.chanpaca.net/health' -TimeoutSec 8
|
||||
} catch {
|
||||
$localHealth = $null
|
||||
$publicHealth = $null
|
||||
}
|
||||
$localReady = $null -ne $localHealth -and $localHealth.environment -eq 'prod' -and $localHealth.db -and $localHealth.engine
|
||||
$publicReady = $null -ne $publicHealth -and $publicHealth.environment -eq 'prod' -and $publicHealth.db -and $publicHealth.engine
|
||||
if ($localReady -and $publicReady) { break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $localReady -or -not $publicReady) {
|
||||
throw '공개 health가 제한 시간 안에 prod/db/engine ready가 되지 않음'
|
||||
}
|
||||
$openapi = Invoke-RestMethod 'https://api-vignette.chanpaca.net/openapi.json' -TimeoutSec 20
|
||||
$pathCount = @($openapi.paths.PSObject.Properties).Count
|
||||
if ($pathCount -ne 119) { throw "공개 OpenAPI path 불일치: $pathCount" }
|
||||
|
||||
Enable-ScheduledTask -TaskName $watchdogTask | Out-Null
|
||||
$watchdogDisabled = $false
|
||||
|
||||
[PSCustomObject]@{
|
||||
status = 'PASS'
|
||||
stamp = $stamp
|
||||
preserved_current_container = $preservedName
|
||||
active_container = $activeName
|
||||
active_volume = $RecoveredVolume
|
||||
counts = "$counts".Trim()
|
||||
local_health = $localHealth.status
|
||||
public_health = $publicHealth.status
|
||||
openapi_paths = $pathCount
|
||||
watchdog_enabled = (Get-ScheduledTask -TaskName $watchdogTask).Settings.Enabled
|
||||
} | ConvertTo-Json -Compress
|
||||
} catch {
|
||||
$reason = $_.Exception.Message
|
||||
Write-Output "CUTOVER_ERROR=$reason"
|
||||
Clear-DatabaseEnvironment
|
||||
|
||||
if ($cutoverStarted) {
|
||||
if (Test-ContainerNameExists $activeName) {
|
||||
& docker stop $activeName 1>$null 2>$null
|
||||
Invoke-DockerChecked @('rename', $activeName, $failedName) '실패 복구 DB 보존 rename 실패' | Out-Null
|
||||
}
|
||||
if (Test-ContainerNameExists $preservedName) {
|
||||
Invoke-DockerChecked @('rename', $preservedName, $activeName) '원래 DB rollback rename 실패' | Out-Null
|
||||
Invoke-DockerChecked @('start', $activeName) '원래 DB rollback start 실패' | Out-Null
|
||||
Wait-Healthy $activeName
|
||||
Restart-PublicApi
|
||||
}
|
||||
}
|
||||
if ($watchdogDisabled) {
|
||||
Enable-ScheduledTask -TaskName $watchdogTask | Out-Null
|
||||
}
|
||||
throw "공개 DB 전환 실패, 원래 DB로 rollback: $reason"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue