운영 안정화와 화면 회귀 검증을 반영
Some checks failed
API contract / OpenAPI type drift (push) Failing after 6m2s

This commit is contained in:
Yun Chan 2026-10-20 16:33:48 +09:00
parent 6e3123091a
commit aeddaa6972
69 changed files with 3178 additions and 424 deletions

View file

@ -90,6 +90,30 @@ function Wait-Health([string]$url, [string]$label, [int]$timeoutSec = 40) {
Write-Host " WARN $label 미응답 ($url) — .devlogs 로그 확인"; return $false
}
function Wait-ApiDbHealth([string]$url, [int]$timeoutSec = 40) {
$deadline = (Get-Date).AddSeconds($timeoutSec)
while ((Get-Date) -lt $deadline) {
try {
$response = Invoke-WebRequest -Uri $url -TimeoutSec 4 -UseBasicParsing -ErrorAction Stop
if ($response.StatusCode -eq 200) {
$health = $response.Content | ConvertFrom-Json -ErrorAction Stop
if ($health.db -is [bool] -and $health.db -eq $true) {
Write-Host " OK api DB ($url)"
return $true
}
}
} catch { Start-Sleep -Milliseconds 600; continue }
Start-Sleep -Milliseconds 600
}
Write-Host " WARN api DB 미준비 ($url) — .devlogs 로그 확인"
return $false
}
function Confirm-ManagedApiDbHealth([bool]$managedDbReady, [string]$url) {
if (-not $managedDbReady) { return $true }
return Wait-ApiDbHealth $url
}
function Read-EnvFileValue([string]$path, [string]$key) {
if (!(Test-Path -LiteralPath $path)) { return $null }
$pattern = ('^\s*{0}\s*=' -f [regex]::Escape($key))
@ -170,9 +194,9 @@ function Get-DevDbContainerEnvValue([string]$key, [string]$fallback) {
return $fallback
}
function Wait-DevDbReady([string]$ownerUser, [int]$attempts = 24) {
function Wait-DevDbReady([string]$ownerUser, [string]$dbName, [int]$attempts = 24) {
for ($i = 0; $i -lt $attempts; $i++) {
Invoke-DockerProbe exec $DevDbContainerName pg_isready -U $ownerUser | Out-Null
Invoke-DockerProbe exec $DevDbContainerName pg_isready -h 127.0.0.1 -p 5432 -U $ownerUser -d $dbName | Out-Null
if ($LASTEXITCODE -eq 0) { return $true }
Start-Sleep -Seconds 2
}
@ -186,12 +210,7 @@ function Inspect-DevDbContainer {
}
$ownerUser = Get-DevDbContainerEnvValue 'POSTGRES_USER' 'vignette_owner'
$dbName = Get-DevDbContainerEnvValue 'POSTGRES_DB' 'vignette'
Invoke-DockerProbe exec $DevDbContainerName pg_isready -U $ownerUser | Out-Null
if ($LASTEXITCODE -eq 0) {
Write-Host ' OK db (vignette-dev-db 준비됨)'
} else {
Write-Host ' WARN db 컨테이너는 있으나 pg_isready 실패'
}
Write-Host ' OK db (vignette-dev-db 준비됨)'
Test-DevDbRoleSafety "$ownerUser" "$dbName"
}
@ -205,7 +224,8 @@ function Start-ExistingDevDbContainer([string]$state) {
throw '기존 vignette-dev-db 시작 실패. 데이터 교체 없이 중단함. pg_dump와 복구 검증을 갖춘 별도 유지보수 절차가 필요하다.'
}
$ownerUser = Get-DevDbContainerEnvValue 'POSTGRES_USER' 'vignette_owner'
if (-not (Wait-DevDbReady $ownerUser)) {
$dbName = Get-DevDbContainerEnvValue 'POSTGRES_DB' 'vignette'
if (-not (Wait-DevDbReady $ownerUser $dbName)) {
throw '기존 vignette-dev-db가 시작됐지만 pg_isready가 실패함. 컨테이너/볼륨을 교체하지 않고 중단함.'
}
Inspect-DevDbContainer
@ -229,33 +249,38 @@ function Ensure-DevDbDataVolume {
function Ensure-DevDb {
if (!(Get-Command docker -ErrorAction SilentlyContinue)) {
Write-Host ' WARN docker CLI 없음 - DB 없이 degraded(UI 세션 시작 제한).'
return
return $false
}
if (!(Test-Path -LiteralPath '\\.\pipe\dockerDesktopLinuxEngine')) {
Write-Host ' WARN Docker Desktop Linux engine 미실행 - DB 없이 degraded(UI 세션 시작 제한).'
return
return $false
}
Invoke-DockerProbe ps | Out-Null
if ($LASTEXITCODE -ne 0) {
Write-Host ' WARN Docker 데몬 미응답 - DB 없이 degraded(UI 세션 시작 제한). Docker Desktop 실행 필요.'
return
return $false
}
$containerState = Get-DevDbContainerState
if (-not [string]::IsNullOrWhiteSpace($containerState)) {
if ($containerState -eq 'running') {
$ownerUser = Get-DevDbContainerEnvValue 'POSTGRES_USER' 'vignette_owner'
$dbName = Get-DevDbContainerEnvValue 'POSTGRES_DB' 'vignette'
if (-not (Wait-DevDbReady $ownerUser $dbName)) {
throw '기존 실행 중 vignette-dev-db가 실제 TCP DB 준비 상태에 도달하지 못함. 컨테이너/볼륨을 교체하지 않고 중단함.'
}
Inspect-DevDbContainer
return
return $true
}
Start-ExistingDevDbContainer $containerState
return
return $true
}
$apiEnv = Join-Path $api '.env'
$dbUrl = Read-EnvFileValue $apiEnv 'DATABASE_URL'
if ($dbUrl -notmatch 'postgresql://([^:]+):([^@]+)@[^:]+:([0-9]+)/(\S+)') {
Write-Host ' WARN DATABASE_URL 파싱 실패 - DB 스킵'
return
return $false
}
$u = $Matches[1]
@ -293,11 +318,10 @@ function Ensure-DevDb {
throw '새 vignette-dev-db 컨테이너 생성 실패. named PGDATA volume은 보존함.'
}
if (Wait-DevDbReady $ownerUser) {
if (Wait-DevDbReady $ownerUser $db) {
Write-Host ' OK db (Postgres 준비됨, named PGDATA + init 스키마 적용)'
Test-DevDbRoleSafety $ownerUser $db
Start-Sleep -Seconds 1
return
return $true
}
throw '새 vignette-dev-db 준비 타임아웃. 컨테이너와 named PGDATA volume은 조사/복구를 위해 보존함.'
}
@ -315,10 +339,11 @@ if (-not $NoWeb) {
}
Start-Sleep -Seconds 2
$managedDbReady = $false
if (-not $NoDb) {
Write-Host '[DB] Postgres(pgvector) 보장...'
Warn-ComposeEnvReadiness
Ensure-DevDb
$managedDbReady = Ensure-DevDb
}
if ($UseHiggsVoice) {
@ -358,7 +383,14 @@ Write-Host '헬스 체크:'
if (-not $NoGateway) {
Wait-Health 'http://127.0.0.1:9099/health' 'gateway' | Out-Null
}
Wait-Health "http://127.0.0.1:$ApiPort/health" 'api' | Out-Null
if ($managedDbReady) {
if (-not (Confirm-ManagedApiDbHealth $managedDbReady "http://127.0.0.1:$ApiPort/health")) {
Write-Host 'ERROR: 관리 DB는 준비됐지만 API health의 db=true를 확인하지 못함. API를 자동 재시작하지 않고 중단함.'
exit 1
}
} else {
Wait-Health "http://127.0.0.1:$ApiPort/health" 'api' | Out-Null
}
if (-not $NoWeb) {
Wait-Health 'http://localhost:5173/' 'web' | Out-Null
}

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Roll up and optionally prune persisted admin health samples."""
"""저장된 관리자 헬스 표본을 집계하고 보존 기간이 지난 행을 정리한다."""
from __future__ import annotations
@ -24,12 +24,18 @@ if hasattr(sys.stderr, "reconfigure"):
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Roll up app.admin_health_event into app.admin_health_daily_rollup "
"and optionally prune old raw samples."
"app.admin_health_event를 app.admin_health_daily_rollup으로 집계하고 "
"보존 기간이 지난 raw 표본을 필요할 때 정리한다."
)
)
parser.add_argument("--rollup-days", type=int, required=True, help="Roll up raw samples older than this many days.")
parser.add_argument("--retention-days", type=int, required=True, help="Prune raw samples older than this many days.")
parser.add_argument("--rollup-days", type=int, required=True, help="이 일수보다 오래된 raw 표본을 집계한다.")
parser.add_argument("--retention-days", type=int, default=90, help="이 일수보다 오래된 raw 표본을 삭제한다. 기본값은 90일이다.")
parser.add_argument(
"--rollup-retention-days",
type=int,
default=365,
help="이 일수보다 오래된 일별 rollup을 삭제한다. 기본값은 365일이다.",
)
parser.add_argument("--apply", action="store_true", help="Mutate the database. Default is dry-run.")
parser.add_argument(
"--allow-non-dev-apply",
@ -63,6 +69,7 @@ async def _run(args: argparse.Namespace) -> dict[str, object]:
result = await maintain_admin_health_events(
rollup_days=args.rollup_days,
retention_days=args.retention_days,
rollup_retention_days=args.rollup_retention_days,
apply=args.apply,
)
finally:
@ -78,7 +85,10 @@ def _print_result(result: dict[str, object], *, as_json: bool) -> None:
print(
"{mode}: rollup_events={rollup_event_count}, rollup_buckets={rollup_bucket_count}, "
"upserted_rollups={upserted_rollups}, prunable_events={prunable_event_count}, "
"deleted_events={deleted_events}".format(mode=mode, **result)
"deleted_events={deleted_events}, rollup_retention_days={rollup_retention_days}, "
"prunable_rollups={prunable_rollup_count}, deleted_rollups={deleted_rollups}".format(
mode=mode, **result
)
)

View file

@ -0,0 +1,266 @@
#!/bin/sh
# 용도: NAS 호스트 프로세스로 떠 있는 cloudflared 터널을
# docker compose 컨테이너(profiles: public-cutover-gate)로 이관한다.
# 실행: NAS에서 root 권한으로 다음과 같이 실행한다.
# sudo sh scripts/nas-migrate-tunnel-to-compose.sh
# 안전성: 4단계 검증(컨테이너 running + 터널 연결 로그 + 공개 health 200)이 실패하면
# 호스트 터널을 절대 건드리지 않고 방금 만든 컨테이너만 정리한다.
# 호스트 터널을 이미 끈 뒤에 문제가 드러나면 호스트 터널을 자동 재기동해
# 서비스를 복구한다. 그래도 복구가 안 되면 exit 2로 끝내고 수동 조치를 안내한다.
# 즉 이 스크립트는 어떤 경로로 끝나도 서비스가 살아 있는 상태를 목표로 한다.
# 주의: docker compose 명령 중 설정을 병합해 평문으로 출력하는 하위 명령(config)은
# env-file에 담긴 시크릿을 그대로 표준출력에 뿌리므로 여기서 절대 쓰지 않는다.
set -eu
PROJECT_DIR="/volume1/docker/vignette-prod"
COMPOSE_FILE="docker-compose.nas.yml"
ENV_FILE="private/env.nas"
CLOUDFLARED_CONFIG="cloudflared/config.yml"
CREDENTIAL_SRC="$PROJECT_DIR/private/cloudflared/vignette-tunnel.json"
CREDENTIAL_FILENAME="vignette-tunnel.json"
VOLUME_NAME="vignette-prod-cloudflared-secrets-20260830"
IMAGE="cloudflare/cloudflared@sha256:d9ff527dc19799e24d3689058161fb01e752a24be1823577bae54c9c2489275f"
HEALTH_URL="https://api-vignette.chanpaca.net/health"
HOST_PID_FILE="$PROJECT_DIR/private/cloudflared-host.pid"
SERVICE_NAME="cloudflared"
COMPOSE_PROJECT="vignette-prod"
POLL_INTERVAL=5
MAX_WAIT=60
echo "[0/5] 사전 점검"
if [ "$(id -u)" -ne 0 ]; then
echo "오류: root 권한이 필요하다. 다음과 같이 실행하라: sudo sh scripts/nas-migrate-tunnel-to-compose.sh" >&2
exit 1
fi
if ! cd "$PROJECT_DIR"; then
echo "오류: 프로젝트 디렉터리로 이동할 수 없다: $PROJECT_DIR" >&2
exit 1
fi
DOCKER_BIN=""
if [ -x /usr/local/bin/docker ]; then
DOCKER_BIN="/usr/local/bin/docker"
elif command -v docker >/dev/null 2>&1; then
DOCKER_BIN="$(command -v docker)"
fi
if [ -z "$DOCKER_BIN" ]; then
echo "오류: docker 명령을 찾을 수 없다." >&2
exit 1
fi
echo " docker 명령: $DOCKER_BIN"
if [ ! -f "$CREDENTIAL_SRC" ]; then
echo "오류: 자격증명 원본 파일이 없다: $CREDENTIAL_SRC" >&2
exit 1
fi
for f in "$COMPOSE_FILE" "$ENV_FILE" "$CLOUDFLARED_CONFIG"; do
if [ ! -f "$f" ]; then
echo "오류: 필수 파일이 없다: $PROJECT_DIR/$f" >&2
exit 1
fi
done
BEFORE_PUBLIC=$(curl -s -o /dev/null -w "%{http_code}" --max-time 15 "$HEALTH_URL" || echo "000")
echo " 이관 전 공개 health 상태: $BEFORE_PUBLIC"
if [ "$BEFORE_PUBLIC" != "200" ]; then
echo " 경고: 이관 전 공개 health가 200이 아니다. 그래도 이관을 계속 진행한다."
fi
HOST_PID=$(ps -ef | grep "[c]loudflared" | grep "cloudflared-host.yml" | awk '{print $2}' | head -1)
if [ -n "$HOST_PID" ]; then
echo " 탐지된 호스트 터널 PID: $HOST_PID"
else
echo " 호스트 터널 프로세스를 찾지 못했다(이미 종료된 것으로 보고 계속 진행한다)."
fi
echo "[1/5] 볼륨 준비"
if ! "$DOCKER_BIN" volume inspect "$VOLUME_NAME" >/dev/null 2>&1; then
echo " 볼륨이 없다. 생성한다: $VOLUME_NAME"
"$DOCKER_BIN" volume create "$VOLUME_NAME" >/dev/null
else
echo " 볼륨이 이미 존재한다: $VOLUME_NAME"
fi
VOLUME_MOUNT=$("$DOCKER_BIN" volume inspect -f '{{ .Mountpoint }}' "$VOLUME_NAME")
echo " 볼륨 마운트 경로: $VOLUME_MOUNT"
CRED_PATH="$VOLUME_MOUNT/$CREDENTIAL_FILENAME"
if [ -f "$CRED_PATH" ]; then
echo " 자격증명 파일이 이미 존재한다. 복사를 건너뛴다."
else
echo " 자격증명 파일을 복사한다."
cp "$CREDENTIAL_SRC" "$CRED_PATH"
fi
echo " 자격증명 파일 정보(내용은 출력하지 않는다):"
ls -la "$CRED_PATH"
echo "[2/5] 권한 맞추기"
if ! "$DOCKER_BIN" image inspect "$IMAGE" >/dev/null 2>&1; then
echo " 이미지를 로컬에서 찾을 수 없다. pull 한다: $IMAGE"
"$DOCKER_BIN" pull "$IMAGE"
fi
IMG_USER=$("$DOCKER_BIN" image inspect "$IMAGE" --format '{{.Config.User}}')
echo " 이미지 지정 사용자: '$IMG_USER'"
case "$IMG_USER" in
''|root|0)
echo " root 사용자로 판단한다 -> chown root:root, chmod 600"
chown root:root "$CRED_PATH"
chmod 600 "$CRED_PATH"
;;
*[!0-9:]*)
echo " 경고: 이미지 사용자가 이름 형태('$IMG_USER')라 UID를 자동으로 추측하지 않는다."
echo " 경고: root 소유 600 권한을 유지한 채 다음 단계로 진행한다."
echo " 경고: 4단계 검증이 실패하면 이 사용자 불일치를 원인으로 먼저 의심하라."
chown root:root "$CRED_PATH"
chmod 600 "$CRED_PATH"
;;
*)
UID_PART=${IMG_USER%%:*}
echo " 숫자 UID 사용자로 판단한다 -> chown $UID_PART:$UID_PART, chmod 600"
chown "$UID_PART:$UID_PART" "$CRED_PATH"
chmod 600 "$CRED_PATH"
;;
esac
echo "[3/5] 컨테이너 기동"
EXISTING_STATUS=$("$DOCKER_BIN" ps --filter "label=com.docker.compose.project=$COMPOSE_PROJECT" --filter "label=com.docker.compose.service=$SERVICE_NAME" --format '{{.Status}}' | head -1)
UP_OK=true
if [ -n "$EXISTING_STATUS" ]; then
echo " 컨테이너가 이미 실행 중이다: $EXISTING_STATUS (중복 기동하지 않는다)"
else
if ! "$DOCKER_BIN" compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --profile public-cutover-gate up -d --no-deps "$SERVICE_NAME"; then
UP_OK=false
echo " 오류: 컨테이너 기동 명령이 실패했다."
fi
fi
echo "[4/5] 검증 (최대 ${MAX_WAIT}초, ${POLL_INTERVAL}초 간격 폴링)"
ELAPSED=0
VERIFY_OK=false
if [ "$UP_OK" = "true" ]; then
while [ "$ELAPSED" -lt "$MAX_WAIT" ]; do
CONTAINER_STATUS=$("$DOCKER_BIN" ps --filter "label=com.docker.compose.project=$COMPOSE_PROJECT" --filter "label=com.docker.compose.service=$SERVICE_NAME" --format '{{.Names}} {{.Status}}' | head -1)
echo " 컨테이너 상태: ${CONTAINER_STATUS:-(없음, 아직 running이 아니다)}"
TUNNEL_LOG_HITS=$("$DOCKER_BIN" compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --profile public-cutover-gate logs "$SERVICE_NAME" 2>/dev/null | grep -c "Registered tunnel connection" || true)
TUNNEL_LOG_HITS=${TUNNEL_LOG_HITS:-0}
echo " 터널 연결 로그(Registered tunnel connection) 건수: $TUNNEL_LOG_HITS"
CURRENT_PUBLIC=$(curl -s -o /dev/null -w "%{http_code}" --max-time 15 "$HEALTH_URL" || echo "000")
echo " 공개 health 상태: $CURRENT_PUBLIC"
if [ -n "$CONTAINER_STATUS" ] && [ "$TUNNEL_LOG_HITS" -ge 1 ] && [ "$CURRENT_PUBLIC" = "200" ]; then
VERIFY_OK=true
break
fi
sleep "$POLL_INTERVAL"
ELAPSED=$((ELAPSED + POLL_INTERVAL))
done
else
echo " 컨테이너 기동이 실패해 폴링을 건너뛴다."
fi
if [ "$VERIFY_OK" = "true" ]; then
echo " 검증 성공: 컨테이너 running + 터널 연결 로그 확인 + 공개 health 200"
else
echo " 검증 실패: 조건을 만족하지 못했다."
fi
echo " 컨테이너 로그 마지막 20줄(연결 로그라 토큰이 없는 것이 통상이지만 그대로 출력한다):"
"$DOCKER_BIN" compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --profile public-cutover-gate logs --tail 20 "$SERVICE_NAME" || true
if [ "$VERIFY_OK" = "true" ]; then
echo "[5/5] 호스트 터널 종료 (검증 성공 경로)"
if [ -n "$HOST_PID" ]; then
echo " 호스트 터널 프로세스 종료 시도: PID $HOST_PID"
kill "$HOST_PID" 2>/dev/null || true
sleep 5
if kill -0 "$HOST_PID" 2>/dev/null; then
echo " 아직 살아있다. SIGKILL로 강제 종료한다."
kill -9 "$HOST_PID" 2>/dev/null || true
fi
else
echo " 종료할 호스트 터널 PID가 없다(이미 종료된 상태)."
fi
if [ -f "$HOST_PID_FILE" ]; then
rm -f "$HOST_PID_FILE"
echo " PID 파일을 삭제했다: $HOST_PID_FILE"
fi
AFTER_PUBLIC=$(curl -s -o /dev/null -w "%{http_code}" --max-time 15 "$HEALTH_URL" || echo "000")
echo " 호스트 터널 종료 후 공개 health 상태: $AFTER_PUBLIC"
if [ "$AFTER_PUBLIC" != "200" ]; then
echo " 경고: 호스트 터널 종료 후 공개 health가 200이 아니다. 서비스 복구를 위해 호스트 터널을 재기동한다."
echo " 1) 방금 만든 컨테이너를 정리한다."
"$DOCKER_BIN" compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --profile public-cutover-gate rm -sf "$SERVICE_NAME" || true
echo " 2) 호스트 터널을 재기동한다."
cd "$PROJECT_DIR"
nohup ./bin/cloudflared tunnel --no-autoupdate --config private/cloudflared-host.yml run >> private/cloudflared-host.log 2>&1 </dev/null &
echo $! > private/cloudflared-host.pid
echo " 3) 25초 대기 후 공개 health를 재확인한다."
sleep 25
RESTORE_PUBLIC=$(curl -s -o /dev/null -w "%{http_code}" --max-time 15 "$HEALTH_URL" || echo "000")
echo " 호스트 터널 재기동 후 공개 health 상태: $RESTORE_PUBLIC"
if [ "$RESTORE_PUBLIC" = "200" ]; then
echo "MIGRATE_FAILED_HOST_RESTORED"
exit 1
fi
echo "MIGRATE_FAILED_SERVICE_DOWN"
echo " 경고: 서비스가 다운된 상태다. 사람이 즉시 확인하고 조치해야 한다."
echo " 수동 복구 명령:"
echo " cd $PROJECT_DIR"
echo " sh scripts/nas-recover-prod.sh"
echo " 위 스크립트로도 복구되지 않으면 cloudflared 대시보드와 컨테이너·호스트 로그를 직접 확인하라."
exit 2
fi
echo "MIGRATE_DONE"
else
echo "[5/5] 롤백 (검증 실패 경로) — 호스트 터널은 건드리지 않는다"
if [ -n "$EXISTING_STATUS" ]; then
echo " 주의: 이 컨테이너는 스크립트 실행 전부터 떠 있던 것이라 제거하지 않는다."
echo " 주의: 이미 이관이 끝난 상태일 수 있고, 제거하면 되돌릴 호스트 터널이 없어 서비스가 끊긴다."
else
"$DOCKER_BIN" compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --profile public-cutover-gate rm -sf "$SERVICE_NAME" || true
fi
ROLLBACK_PUBLIC=$(curl -s -o /dev/null -w "%{http_code}" --max-time 15 "$HEALTH_URL" || echo "000")
echo " 롤백 후 공개 health 상태(호스트 터널 기준, 200이어야 정상): $ROLLBACK_PUBLIC"
if [ "$ROLLBACK_PUBLIC" != "200" ]; then
echo " 경고: 롤백 뒤에도 공개 health가 200이 아니다. 서비스가 끊겼을 수 있으니 즉시 확인하라."
echo " 수동 복구: cd $PROJECT_DIR && sh scripts/nas-recover-prod.sh"
fi
echo " 실패 원인 후보:"
echo " (a) 자격증명 권한 불일치 - 2단계에서 이름 형태 사용자 경고가 있었다면 이것을 의심하라."
echo " (b) 볼륨 마운트 문제 - $VOLUME_MOUNT 아래 자격증명 파일 경로를 확인하라."
echo " (c) 컨테이너 기동 자체 실패 - 3단계 오류 메시지를 확인하라(UP_OK=$UP_OK)."
echo " (d) 터널 미연결 - 위에 출력된 컨테이너 로그에서 Registered tunnel connection 여부를 확인하라."
echo "MIGRATE_FAILED"
exit 1
fi

View file

@ -0,0 +1,45 @@
#!/bin/sh
# NAS(vignette-prod)에서 실행하는 재기동 복구 스크립트.
# 재부팅 뒤 공개 API(https://api-vignette.chanpaca.net)가 530이면 실행한다.
# sudo·docker 권한 불필요.
# 이 스크립트는 호스트 상주 터널용이며 compose 터널과 동시에 쓰지 않는다.
set -eu
cd /volume1/docker/vignette-prod
echo "[1/5] cloudflared 중복 실행 확인"
if ps -ef | grep -v grep | grep -q "[c]loudflared tunnel"; then
echo "cloudflared가 이미 실행 중이다. 터널 기동을 건너뛴다."
else
echo "[2/5] 오리진(api) 사전 확인"
origin_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:18080/health || echo "000")
if [ "$origin_code" != "200" ]; then
echo "경고: 로컬 오리진 http://127.0.0.1:18080/health 상태 코드 $origin_code (200 아님). 터널만 띄워도 502가 될 수 있다. 계속 진행한다."
fi
echo "[3/5] 호스트 터널 기동"
nohup ./bin/cloudflared tunnel --no-autoupdate --config private/cloudflared-host.yml run >> private/cloudflared-host.log 2>&1 </dev/null &
echo $! > private/cloudflared-host.pid
echo "[4/5] 25초 대기 후 프로세스 생존 확인"
sleep 25
pid=$(cat private/cloudflared-host.pid)
if kill -0 "$pid" 2>/dev/null; then
echo "cloudflared(pid $pid) 생존 확인"
else
echo "경고: cloudflared(pid $pid) 생존 확인 실패"
fi
echo "로그 마지막 10줄:"
tail -n 10 private/cloudflared-host.log
fi
echo "[5/5] 완료 판정"
local_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:18080/health || echo "000")
public_code=$(curl -s -o /dev/null -w "%{http_code}" https://api-vignette.chanpaca.net/health || echo "000")
echo "로컬 health: $local_code"
echo "공개 health: $public_code"
if [ "$public_code" != "200" ]; then
echo "RECOVER_FAILED"
exit 1
fi
echo "RECOVER_DONE"

View file

@ -127,6 +127,9 @@ RELEASE_DB_MIGRATIONS = (
"20_public_bootstrap_ticket_events.sql",
"21_single_active_session.sql",
"22_case_profile_multi_case.sql",
"23_client_affect_trace.sql",
"24_client_inner_reaction.sql",
"25_admin_health_rollup_retention.sql",
)
RELEASE_DB_ONLINE_MIGRATIONS = frozenset(
{
@ -150,6 +153,9 @@ REQUIRED_RELEASE_PAYLOAD_PATHS = (
"infra/db/init/20_public_bootstrap_ticket_events.sql",
"infra/db/init/21_single_active_session.sql",
"infra/db/init/22_case_profile_multi_case.sql",
"infra/db/init/23_client_affect_trace.sql",
"infra/db/init/24_client_inner_reaction.sql",
"infra/db/init/25_admin_health_rollup_retention.sql",
)
REQUIRED_OPENAPI_PATHS = {

View file

@ -211,8 +211,12 @@ class LocalController:
time.monotonic() - started,
)
if check and result.returncode != 0:
tail = (result.stderr or result.stdout).strip()[-1200:]
raise GateError(f"{stage}: exit {result.returncode}: {tail}")
stdout_tail = result.stdout.strip()[-600:]
stderr_tail = result.stderr.strip()[-600:]
raise GateError(
f"{stage}: exit {result.returncode}: "
f"stdout tail:\n{stdout_tail}\nstderr tail:\n{stderr_tail}"
)
return result
def start(

View file

@ -0,0 +1,179 @@
from __future__ import annotations
import re
import shutil
import subprocess
import tempfile
import unittest
from pathlib import Path
DEV_UP_PATH = Path(__file__).with_name("dev-up.ps1")
class DevUpDatabaseReadinessTest(unittest.TestCase):
"""Docker 없이 PowerShell 테스트 더블로 polling 동작을 검증한다."""
@classmethod
def setUpClass(cls) -> None:
cls.source = DEV_UP_PATH.read_text(encoding="utf-8")
cls.powershell = shutil.which("powershell.exe")
assert cls.powershell is not None, (
"이 검증은 Windows PowerShell 5.1을 필수 사전조건으로 한다."
)
def function_source(self, name: str) -> str:
match = re.search(rf"^function {re.escape(name)}\b", self.source, re.MULTILINE)
self.assertIsNotNone(match, name)
assert match is not None
body_start = self.source.index("{", match.start())
depth = 0
for index in range(body_start, len(self.source)):
if self.source[index] == "{":
depth += 1
elif self.source[index] == "}":
depth -= 1
if depth == 0:
return self.source[match.start() : index + 1]
self.fail(f"unterminated PowerShell function: {name}")
def run_powershell(self, script: str) -> subprocess.CompletedProcess[str]:
assert self.powershell is not None
with tempfile.TemporaryDirectory() as directory:
harness = Path(directory) / "dev-up-readiness-harness.ps1"
prelude = (
"[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false)\n"
"$OutputEncoding = [Text.UTF8Encoding]::new($false)\n"
)
harness.write_text(prelude + script, encoding="utf-8-sig")
return subprocess.run(
[self.powershell, "-NoProfile", "-File", str(harness)],
check=False,
capture_output=True,
text=True,
encoding="utf-8",
)
def test_tcp_readiness_retries_then_succeeds_with_real_database_name(self) -> None:
wait = self.function_source("Wait-DevDbReady")
completed = self.run_powershell(
"\n".join(
(
"$DevDbContainerName = 'vignette-dev-db'",
"$script:probeCalls = 0",
"$script:probeArguments = ''",
"function Invoke-DockerProbe {",
" $script:probeCalls += 1",
" $script:probeArguments = $args -join ' '",
" if ($script:probeCalls -lt 3) { $global:LASTEXITCODE = 1 } else { $global:LASTEXITCODE = 0 }",
"}",
"function Start-Sleep { param([int]$Seconds, [int]$Milliseconds) }",
wait,
"if (-not (Wait-DevDbReady 'owner' 'vignette' 3)) { throw 'expected readiness success' }",
"if ($script:probeCalls -ne 3) { throw \"probe calls: $script:probeCalls\" }",
"if ($script:probeArguments -ne 'exec vignette-dev-db pg_isready -h 127.0.0.1 -p 5432 -U owner -d vignette') { throw \"unexpected probe: $script:probeArguments\" }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
def test_tcp_readiness_times_out_without_false_success(self) -> None:
wait = self.function_source("Wait-DevDbReady")
completed = self.run_powershell(
"\n".join(
(
"$DevDbContainerName = 'vignette-dev-db'",
"$script:probeCalls = 0",
"function Invoke-DockerProbe { $script:probeCalls += 1; $global:LASTEXITCODE = 1 }",
"function Start-Sleep { param([int]$Seconds, [int]$Milliseconds) }",
wait,
"if (Wait-DevDbReady 'owner' 'vignette' 2) { throw 'timeout was reported as success' }",
"if ($script:probeCalls -ne 2) { throw \"probe calls: $script:probeCalls\" }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
def test_http_200_with_db_false_is_rejected(self) -> None:
wait = self.function_source("Wait-ApiDbHealth")
completed = self.run_powershell(
"\n".join(
(
"$script:clock = [System.Collections.Generic.Queue[datetime]]::new()",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:01:00Z')",
"function Get-Date { return $script:clock.Dequeue() }",
"function Invoke-WebRequest { return [pscustomobject]@{ StatusCode = 200; Content = '{\"db\":false}' } }",
"function Start-Sleep { param([int]$Seconds, [int]$Milliseconds) }",
wait,
"if (Wait-ApiDbHealth 'http://127.0.0.1:8000/health' 40) { throw 'db=false was accepted' }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
def test_http_200_with_string_true_is_rejected(self) -> None:
wait = self.function_source("Wait-ApiDbHealth")
completed = self.run_powershell(
"\n".join(
(
"$script:clock = [System.Collections.Generic.Queue[datetime]]::new()",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:01:00Z')",
"function Get-Date { return $script:clock.Dequeue() }",
"function Invoke-WebRequest { return [pscustomobject]@{ StatusCode = 200; Content = '{\"db\":\"true\"}' } }",
"function Start-Sleep { param([int]$Seconds, [int]$Milliseconds) }",
wait,
"if (Wait-ApiDbHealth 'http://127.0.0.1:8000/health' 40) { throw 'string db=true was accepted' }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
def test_http_200_with_db_false_retries_until_boolean_true(self) -> None:
wait = self.function_source("Wait-ApiDbHealth")
completed = self.run_powershell(
"\n".join(
(
"$script:clock = [System.Collections.Generic.Queue[datetime]]::new()",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:00Z')",
"$script:clock.Enqueue([datetime]'2026-01-01T00:00:01Z')",
"$script:requestCalls = 0",
"function Get-Date { return $script:clock.Dequeue() }",
"function Invoke-WebRequest {",
" $script:requestCalls += 1",
" if ($script:requestCalls -eq 1) { return [pscustomobject]@{ StatusCode = 200; Content = '{\"db\":false}' } }",
" return [pscustomobject]@{ StatusCode = 200; Content = '{\"db\":true}' }",
"}",
"function Start-Sleep { param([int]$Seconds, [int]$Milliseconds) }",
wait,
"if (-not (Wait-ApiDbHealth 'http://127.0.0.1:8000/health' 40)) { throw 'db=false to db=true retry did not succeed' }",
"if ($script:requestCalls -ne 2) { throw \"request calls: $script:requestCalls\" }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
def test_degraded_mode_skips_managed_database_health_requirement(self) -> None:
confirm = self.function_source("Confirm-ManagedApiDbHealth")
completed = self.run_powershell(
"\n".join(
(
"$script:apiDbHealthCalls = 0",
"function Wait-ApiDbHealth { $script:apiDbHealthCalls += 1; return $false }",
confirm,
"if (-not (Confirm-ManagedApiDbHealth $false 'http://127.0.0.1:8000/health')) { throw 'degraded mode was rejected' }",
"if ($script:apiDbHealthCalls -ne 0) { throw 'degraded mode called managed DB health check' }",
"if (Confirm-ManagedApiDbHealth $true 'http://127.0.0.1:8000/health') { throw 'managed DB health failure was accepted' }",
"if ($script:apiDbHealthCalls -ne 1) { throw \"managed DB health calls: $script:apiDbHealthCalls\" }",
)
)
)
self.assertEqual(0, completed.returncode, completed.stderr)
if __name__ == "__main__":
unittest.main()

View file

@ -0,0 +1,76 @@
"""CLI contracts for admin health retention maintenance."""
from __future__ import annotations
import contextlib
import importlib.util
import io
import unittest
from pathlib import Path
SCRIPT_PATH = Path(__file__).with_name("maintain-admin-health-events.py")
SPEC = importlib.util.spec_from_file_location("maintain_admin_health_events", SCRIPT_PATH)
if SPEC is None or SPEC.loader is None:
raise RuntimeError(f"cannot load maintenance CLI: {SCRIPT_PATH}")
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class MaintainAdminHealthEventsCliTest(unittest.TestCase):
def test_retention_arguments_keep_raw_requirements_and_rollup_default(self) -> None:
args = MODULE.build_parser().parse_args(["--rollup-days", "90"])
self.assertEqual(args.rollup_days, 90)
self.assertEqual(args.retention_days, 90)
self.assertEqual(args.rollup_retention_days, 365)
self.assertFalse(args.apply)
self.assertFalse(args.allow_non_dev_apply)
def test_apply_guard_flags_remain_explicit(self) -> None:
args = MODULE.build_parser().parse_args(
[
"--rollup-days",
"90",
"--retention-days",
"90",
"--rollup-retention-days",
"400",
"--apply",
"--allow-non-dev-apply",
]
)
self.assertTrue(args.apply)
self.assertTrue(args.allow_non_dev_apply)
self.assertEqual(args.rollup_retention_days, 400)
def test_human_output_includes_raw_and_rollup_prune_results(self) -> None:
result = {
"applied": False,
"rollup_days": 90,
"retention_days": 90,
"rollup_retention_days": 365,
"rollup_event_count": 8,
"rollup_bucket_count": 2,
"upserted_rollups": 0,
"prunable_event_count": 1,
"deleted_events": 0,
"prunable_rollup_count": 3,
"deleted_rollups": 0,
}
stdout = io.StringIO()
with contextlib.redirect_stdout(stdout):
MODULE._print_result(result, as_json=False)
rendered = stdout.getvalue()
self.assertIn("dry-run", rendered)
self.assertIn("prunable_events=1", rendered)
self.assertIn("rollup_retention_days=365", rendered)
self.assertIn("prunable_rollups=3", rendered)
self.assertIn("deleted_rollups=0", rendered)
if __name__ == "__main__":
unittest.main()

View file

@ -367,26 +367,74 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "isolated NAS preview"):
self.agent_module.validate_target(wrong)
def test_mobile_shell_and_late_migrations_are_required_release_payloads(self) -> None:
manifest = {
"classifications": {
"related": [
{
"disposition": "candidate-whole-file",
"paths": [
path
for path in self.agent_module.REQUIRED_RELEASE_PAYLOAD_PATHS
if path != "apps/web/src/components/shell/shell.css"
],
}
]
}
}
with self.assertRaisesRegex(
self.agent_module.StageFailure,
def test_runtime_critical_payloads_fail_closed_when_missing_from_manifest(
self,
) -> None:
for missing_path in (
"apps/web/src/components/shell/shell.css",
"infra/db/init/23_client_affect_trace.sql",
"infra/db/init/24_client_inner_reaction.sql",
"infra/db/init/25_admin_health_rollup_retention.sql",
):
self.agent_module.validate_required_release_payload(manifest)
with self.subTest(missing_path=missing_path):
manifest = {
"classifications": {
"related": [
{
"disposition": "candidate-whole-file",
"paths": [
path
for path in self.agent_module.REQUIRED_RELEASE_PAYLOAD_PATHS
if path != missing_path
],
}
]
}
}
with self.assertRaisesRegex(
self.agent_module.StageFailure,
missing_path,
):
self.agent_module.validate_required_release_payload(manifest)
def test_client_affect_migrations_are_transactional_release_payloads(
self,
) -> None:
expected_migrations = (
"23_client_affect_trace.sql",
"24_client_inner_reaction.sql",
)
for migration_name in expected_migrations:
with self.subTest(migration_name=migration_name):
self.assertIn(
migration_name,
self.agent_module.RELEASE_DB_MIGRATIONS,
)
self.assertNotIn(
migration_name,
self.agent_module.RELEASE_DB_ONLINE_MIGRATIONS,
)
self.assertIn(
f"infra/db/init/{migration_name}",
self.agent_module.REQUIRED_RELEASE_PAYLOAD_PATHS,
)
def test_admin_health_rollup_retention_migration_is_transactional_release_payload(
self,
) -> None:
migration_name = "25_admin_health_rollup_retention.sql"
migration_path = SCRIPT_PATH.parent.parent / "infra" / "db" / "init" / migration_name
migration = migration_path.read_text(encoding="utf-8")
self.assertIn("CREATE POLICY p_admin_health_daily_rollup_delete", migration)
self.assertIn("FOR DELETE", migration)
self.assertIn("app.current_role_name() = 'admin'", migration)
self.assertIn(migration_name, self.agent_module.RELEASE_DB_MIGRATIONS)
self.assertNotIn(migration_name, self.agent_module.RELEASE_DB_ONLINE_MIGRATIONS)
self.assertIn(
f"infra/db/init/{migration_name}",
self.agent_module.REQUIRED_RELEASE_PAYLOAD_PATHS,
)
def test_admin_usage_index_migration_is_idempotent_and_release_required(self) -> None:
migration_path = (
@ -892,6 +940,9 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
"20_public_bootstrap_ticket_events.sql",
"21_single_active_session.sql",
"22_case_profile_multi_case.sql",
"23_client_affect_trace.sql",
"24_client_inner_reaction.sql",
"25_admin_health_rollup_retention.sql",
),
self.agent_module.RELEASE_DB_MIGRATIONS,
)
@ -1099,6 +1150,24 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
if "22_case_profile_multi_case.sql" in line
and "psql -v ON_ERROR_STOP=1" in line
)
affect_trace_migration_command = next(
line
for line in promote_command.splitlines()
if "23_client_affect_trace.sql" in line
and "psql -v ON_ERROR_STOP=1" in line
)
inner_reaction_migration_command = next(
line
for line in promote_command.splitlines()
if "24_client_inner_reaction.sql" in line
and "psql -v ON_ERROR_STOP=1" in line
)
rollup_retention_migration_command = next(
line
for line in promote_command.splitlines()
if "25_admin_health_rollup_retention.sql" in line
and "psql -v ON_ERROR_STOP=1" in line
)
self.assertIn("--single-transaction", transactional_migration_command)
self.assertNotIn("--single-transaction", online_migration_command)
self.assertNotIn(
@ -1107,6 +1176,17 @@ class OutcomeReleaseAgentTests(unittest.TestCase):
self.assertNotIn(
"--single-transaction", case_profile_online_migration_command
)
self.assertIn("--single-transaction", affect_trace_migration_command)
self.assertIn("--single-transaction", inner_reaction_migration_command)
self.assertIn("--single-transaction", rollup_retention_migration_command)
self.assertLess(
promote_command.index("23_client_affect_trace.sql"),
promote_command.index("24_client_inner_reaction.sql"),
)
self.assertLess(
promote_command.index("24_client_inner_reaction.sql"),
promote_command.index("up -d --no-deps --force-recreate api web proxy"),
)
self.assertEqual(f"{root}/release/infra/.env", state["env_file"])
def test_adopted_legacy_state_infers_compose_adjacent_env_file(self) -> None:

View file

@ -27,6 +27,36 @@ runner = load_module()
class PeriodicRunnerStaticContractTests(unittest.TestCase):
def test_local_controller_failure_keeps_both_bounded_stream_tails(self) -> None:
stdout = "x" * 1300 + "PLAYWRIGHT_ASSERTION"
stderr = "y" * 1300 + "NODE_WARNING"
completed = runner.subprocess.CompletedProcess(
args=["node.exe"],
returncode=17,
stdout=stdout,
stderr=stderr,
)
with patch.object(runner.subprocess, "run", return_value=completed):
with self.assertRaises(runner.GateError) as caught:
runner.LocalController().run(
"periodic_browser",
["node.exe", "playwright"],
cwd=Path("."),
timeout=1,
)
message = runner._safe_error(caught.exception)
self.assertIn("stdout tail:\n", message)
self.assertIn("stderr tail:\n", message)
self.assertIn("PLAYWRIGHT_ASSERTION", message)
self.assertIn("NODE_WARNING", message)
stdout_tail, stderr_tail = message.split("stdout tail:\n", 1)[1].split(
"\nstderr tail:\n", 1
)
self.assertLessEqual(len(stdout_tail), 600)
self.assertLessEqual(len(stderr_tail), 600)
def test_browser_failures_are_not_retried_automatically(self) -> None:
self.assertEqual(runner.PLAYWRIGHT_RETRIES, 0)