From 5cb530e153d8b000f8f62b8cf3567f9383f355ce Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Wed, 12 Aug 2026 21:17:04 +0900 Subject: [PATCH] =?UTF-8?q?=EA=B3=B5=EA=B0=9C=20=EB=9F=B0=ED=83=80?= =?UTF-8?q?=EC=9E=84=20=EC=9B=8C=EC=B9=98=EB=8F=85=20=EC=B0=BD=C2=B7?= =?UTF-8?q?=EB=B3=B5=EA=B5=AC=20=EA=B2=B0=ED=95=A8=203=EA=B1=B4=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 재부팅 뒤 워치독이 5분마다 콘솔 창에 실패만 뿌리던 문제의 원인 세 가지를 고친다. - 워치독이 DB 다운을 감지하고 직접 복구한다. postgres(vignette-dev-db) 기동은 boot 담당이라 start-public-runtime.ps1 재호출로는 절대 복구되지 않았고, 그 결과 워치독은 고칠 수 없는 대상에 start를 무한 재시도하며 실패만 기록했다. db를 health check 항목에 넣고, 재시작 전에 컨테이너를 되살리며, 복구 실패 시에는 runtime 재시작을 시도하지 않고 종료한다. - 작업 액션을 wscript 런처(watch-public-runtime-task.vbs) 경유로 등록한다. powershell.exe를 직접 등록하면 -WindowStyle Hidden이어도 conhost 창이 매 실행 번쩍이고, 5분 주기에서는 그것이 곧 화면을 가리는 창이 된다. 런처는 pin 인자를 해석하지 않고 전달만 하며 provenance 검증은 기존대로 watchdog이 수행한다. - boot이 web preview 상태를 보고 -SkipWebRestart를 조건부로 붙인다. 무조건 스킵하면 재부팅 직후처럼 vite가 죽은 상태에서 boot 경로로는 web이 영영 복구되지 않았다. hidden trigger는 액션이 wscript 런처를 거치는지 함께 검증하도록 맞췄다. --- docs/ops/public-runtime-watchdog.md | 15 +++-- scripts/boot-public-runtime.ps1 | 50 ++++++++++++---- scripts/install-public-runtime-task.ps1 | 22 ++++--- ...test_public_runtime_watchdog_provenance.py | 16 ++++- scripts/watch-public-runtime-hidden.vbs | 4 +- scripts/watch-public-runtime-task.vbs | 45 ++++++++++++++ scripts/watch-public-runtime.ps1 | 60 ++++++++++++++++++- 7 files changed, 184 insertions(+), 28 deletions(-) create mode 100644 scripts/watch-public-runtime-task.vbs diff --git a/docs/ops/public-runtime-watchdog.md b/docs/ops/public-runtime-watchdog.md index fc8e58a..65abcdb 100644 --- a/docs/ops/public-runtime-watchdog.md +++ b/docs/ops/public-runtime-watchdog.md @@ -129,13 +129,20 @@ LastTaskResult=0과 stable release root의 public-runtime-watchdog.failcount=0 ## 숨김 수동 Trigger -watch-public-runtime-hidden.vbs는 source script를 직접 실행하지 않는다. 등록된 watchdog task action에 -StableSourceRoot, commit, tree, watchdog SHA, start SHA marker가 모두 있고 legacy Workspace action이 아님을 -검사한 뒤 Start-ScheduledTask만 호출한다. +watch-public-runtime-hidden.vbs는 source script를 직접 실행하지 않는다. 등록된 watchdog task action이 +wscript 런처(watch-public-runtime-task.vbs)를 거치고 StableSourceRoot, commit, tree, watchdog SHA, +start SHA marker가 모두 있으며 legacy Workspace action이 아님을 검사한 뒤 Start-ScheduledTask만 호출한다. cscript.exe //nologo scripts\watch-public-runtime-hidden.vbs -task가 아직 legacy shared-worktree action이면 VBS도 fail-closed한다. +task가 아직 legacy shared-worktree action이거나 powershell.exe를 직접 실행하도록 등록돼 있으면 VBS도 fail-closed한다. + +## Task Action이 wscript 런처를 거치는 이유 + +install-public-runtime-task.ps1은 액션을 `wscript.exe "\scripts\watch-public-runtime-task.vbs" -File ...`로 +등록한다. powershell.exe를 액션으로 직접 등록하면 `-WindowStyle Hidden`을 붙여도 conhost 창이 실행 순간 +번쩍이고, 5분 주기 watchdog에서는 그것이 곧 "5분마다 화면에 뜨는 콘솔 창"이 된다(2026-08-08/09/12 세 번 재발). +런처는 pin 인자를 해석하지 않고 그대로 전달만 하며, provenance 검증은 watch-public-runtime.ps1이 수행한다. ## Manual Source Recovery diff --git a/scripts/boot-public-runtime.ps1 b/scripts/boot-public-runtime.ps1 index 8704cd8..44c3ed0 100644 --- a/scripts/boot-public-runtime.ps1 +++ b/scripts/boot-public-runtime.ps1 @@ -28,6 +28,7 @@ param( [int]$DbTimeoutSec = 90, [int]$DbPort = 55432, [int]$ApiPort = 8001, + [int]$WebPort = 5174, [int]$EnginePort = 9099, [int]$WhisperPort = 9882, [int]$MeloTtsPort = 9883, @@ -225,6 +226,22 @@ function Test-VoiceApiHealthy { return $false } +function Test-WebPreviewHealthy { + # 재부팅 직후 web preview(vite)는 항상 죽어 있다. 이 검사 없이 무조건 -SkipWebRestart를 + # 넘기면 boot 경로로는 web이 영원히 복구되지 않는다(2026-08-12 확인). + try { + $req = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$WebPort/") + $req.Timeout = 5000 + $req.ReadWriteTimeout = 5000 + $req.Proxy = $null + $resp = $req.GetResponse() + $resp.Close() + return $true + } catch { + return $false + } +} + function Test-VoiceSidecarStack { $probeArgs = @( "-X", "utf8", "-B", $voiceSidecarProbe, @@ -288,18 +305,29 @@ if (-not (Test-Tcp -Host_ "127.0.0.1" -Port $DbPort)) { } Write-BootLog "postgres 127.0.0.1:$DbPort up" -# 3) 엔진/API/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지) -if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack)) { - Write-BootLog "control plane, engine, exact local voice API, and sidecars already healthy; skipping runtime restart" +# 3) 엔진/API/web/cloudflared — 이미 healthy 면 스킵(불필요한 재시작/다운타임 방지) +# web preview는 살아 있을 때만 -SkipWebRestart 한다. 무조건 스킵하면 재부팅 직후처럼 +# vite가 죽은 상태에서 boot 경로로는 web이 영영 복구되지 않는다. +$webHealthy = Test-WebPreviewHealthy +if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiHealthy) -and (Test-VoiceSidecarStack) -and $webHealthy) { + Write-BootLog "control plane, engine, exact local voice API, sidecars, and web preview already healthy; skipping runtime restart" } else { - Write-BootLog "running start-public-runtime.ps1 -SkipWebRestart" - $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript ` - -Workspace $resolvedSourceRoot ` - -Python $Python ` - -EnginePort $EnginePort ` - -WhisperPort $WhisperPort ` - -MeloTtsPort $MeloTtsPort ` - -SkipWebRestart 2>&1 + $startArgs = @( + "-Workspace", $resolvedSourceRoot, + "-Python", $Python, + "-ApiPort", $ApiPort, + "-WebPort", $WebPort, + "-EnginePort", $EnginePort, + "-WhisperPort", $WhisperPort, + "-MeloTtsPort", $MeloTtsPort + ) + if ($webHealthy) { + $startArgs += "-SkipWebRestart" + } else { + Write-BootLog "web preview down on 127.0.0.1:$WebPort; including web in restart" + } + Write-BootLog ("running start-public-runtime.ps1 " + ($startArgs -join " ")) + $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript @startArgs 2>&1 $out | ForEach-Object { Write-BootLog (" pub> " + $_) } if ($LASTEXITCODE -ne 0) { Write-BootLog "ERROR: start-public-runtime.ps1 exit $LASTEXITCODE" diff --git a/scripts/install-public-runtime-task.ps1 b/scripts/install-public-runtime-task.ps1 index ec0be69..f3a3082 100644 --- a/scripts/install-public-runtime-task.ps1 +++ b/scripts/install-public-runtime-task.ps1 @@ -20,6 +20,7 @@ $installerScript = Join-Path $resolvedSourceRoot "scripts\install-public-runtime $watchScript = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime.ps1" $startScript = Join-Path $resolvedSourceRoot "scripts\start-public-runtime.ps1" $voiceSidecarProbe = Join-Path $resolvedSourceRoot "scripts\probe-public-voice-sidecars.py" +$taskLauncher = Join-Path $resolvedSourceRoot "scripts\watch-public-runtime-task.vbs" function Invoke-GitText { param([string[]]$Arguments) @@ -31,7 +32,7 @@ function Invoke-GitText { return (@($value) -join [Environment]::NewLine).Trim() } -foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe)) { +foreach ($requiredScript in @($installerScript, $watchScript, $startScript, $voiceSidecarProbe, $taskLauncher)) { if (!(Test-Path -LiteralPath $requiredScript -PathType Leaf)) { throw "Public runtime script not found at $requiredScript" } @@ -73,7 +74,8 @@ foreach ($relativePath in @( "scripts/install-public-runtime-task.ps1", "scripts/watch-public-runtime.ps1", "scripts/start-public-runtime.ps1", - "scripts/probe-public-voice-sidecars.py" + "scripts/probe-public-voice-sidecars.py", + "scripts/watch-public-runtime-task.vbs" )) { Invoke-GitText -Arguments @("ls-files", "--error-unmatch", "--", $relativePath) | Out-Null } @@ -83,12 +85,16 @@ $sourceTree = Invoke-GitText -Arguments @("rev-parse", "--verify", "HEAD^{tree}" $watchdogSha256 = (Get-FileHash -LiteralPath $watchScript -Algorithm SHA256).Hash.ToLowerInvariant() $startScriptSha256 = (Get-FileHash -LiteralPath $startScript -Algorithm SHA256).Hash.ToLowerInvariant() -$powershell = (Get-Command powershell.exe).Source +$wscript = Join-Path $env:SystemRoot "System32\wscript.exe" +if (!(Test-Path -LiteralPath $wscript -PathType Leaf)) { + throw "wscript.exe not found at $wscript" +} $userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name +# powershell.exe 를 액션으로 직접 등록하면 -WindowStyle Hidden 이어도 conhost 창이 +# 매 실행 번쩍인다. 5분 주기 워치독에서는 그게 곧 "5분마다 뜨는 창"이 된다. +# wscript 런처를 거쳐 Run(cmd, 0) 으로 띄우면 창이 생성되지 않는다. $actionArguments = @( - "-NoProfile", - "-ExecutionPolicy Bypass", "-File `"$watchScript`"", "-StableSourceRoot `"$resolvedSourceRoot`"", "-ExpectedSourceCommit $sourceCommit", @@ -108,8 +114,8 @@ if ($AdditionalPublicHealthUrls.Count -gt 0) { } $action = New-ScheduledTaskAction ` - -Execute $powershell ` - -Argument ($actionArguments -join " ") ` + -Execute $wscript ` + -Argument ("`"$taskLauncher`" " + ($actionArguments -join " ")) ` -WorkingDirectory $resolvedSourceRoot $logonTrigger = New-ScheduledTaskTrigger -AtLogOn -User $userId @@ -144,7 +150,7 @@ $task = New-ScheduledTask ` Register-ScheduledTask -TaskName $TaskName -InputObject $task -Force | Out-Null Write-Output "Installed scheduled task '$TaskName' for $userId" -Write-Output "Action: $powershell $($actionArguments -join ' ')" +Write-Output "Action: $wscript `"$taskLauncher`" $($actionArguments -join ' ')" Write-Output "Pinned source: root=$resolvedSourceRoot commit=$sourceCommit tree=$sourceTree" Write-Output "Pinned scripts: watchdog_sha256=$watchdogSha256 start_sha256=$startScriptSha256" Write-Output "Interval: every $IntervalMinutes minute(s), plus at user logon" diff --git a/scripts/test_public_runtime_watchdog_provenance.py b/scripts/test_public_runtime_watchdog_provenance.py index 4807e16..bffe07a 100644 --- a/scripts/test_public_runtime_watchdog_provenance.py +++ b/scripts/test_public_runtime_watchdog_provenance.py @@ -14,6 +14,7 @@ INSTALLER = SCRIPTS / "install-public-runtime-task.ps1" BOOT = SCRIPTS / "boot-public-runtime.ps1" BOOT_REGISTER = SCRIPTS / "register-boot-task.ps1" HIDDEN_TRIGGER = SCRIPTS / "watch-public-runtime-hidden.vbs" +TASK_LAUNCHER = SCRIPTS / "watch-public-runtime-task.vbs" START = SCRIPTS / "start-public-runtime.ps1" VOICE_PROBE = SCRIPTS / "probe-public-voice-sidecars.py" REPO_ROOT = SCRIPTS.parent @@ -142,8 +143,12 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): BOOT_SOURCE, ) self.assertGreaterEqual(BOOT_SOURCE.count("Test-VoiceApiHealthy"), 3) - self.assertIn("-WhisperPort $WhisperPort", BOOT_SOURCE) - self.assertIn("-MeloTtsPort $MeloTtsPort", BOOT_SOURCE) + self.assertIn('"-WhisperPort", $WhisperPort', BOOT_SOURCE) + self.assertIn('"-MeloTtsPort", $MeloTtsPort', BOOT_SOURCE) + # web preview는 살아 있을 때만 스킵한다. 무조건 -SkipWebRestart면 재부팅 후 + # boot 경로로 web이 복구되지 않는다. + self.assertIn("$webHealthy = Test-WebPreviewHealthy", BOOT_SOURCE) + self.assertIn('if ($webHealthy) {\n $startArgs += "-SkipWebRestart"', BOOT_SOURCE) self.assertIn("scripts/probe-public-voice-sidecars.py", INSTALLER_SOURCE) self.assertIn("scripts/probe-public-voice-sidecars.py", BOOT_REGISTER_SOURCE) @@ -400,12 +405,14 @@ if ($result.Detail -ne 'exact readiness probe failed') { copied_boot_register = scripts / BOOT_REGISTER.name copied_start = scripts / START.name copied_voice_probe = scripts / VOICE_PROBE.name + copied_task_launcher = scripts / TASK_LAUNCHER.name shutil.copy2(WATCHDOG, copied_watchdog) shutil.copy2(INSTALLER, copied_installer) shutil.copy2(BOOT, copied_boot) shutil.copy2(BOOT_REGISTER, copied_boot_register) shutil.copy2(START, copied_start) shutil.copy2(VOICE_PROBE, copied_voice_probe) + shutil.copy2(TASK_LAUNCHER, copied_task_launcher) self._git(root, "init") self._git(root, "config", "user.name", "Watchdog Contract Test") @@ -419,6 +426,7 @@ if ($result.Detail -ne 'exact readiness probe failed') { "scripts/register-boot-task.ps1", "scripts/start-public-runtime.ps1", "scripts/probe-public-voice-sidecars.py", + "scripts/watch-public-runtime-task.vbs", ) self._git(root, "commit", "-m", "watchdog fixture") self._git(root, "checkout", "--detach") @@ -585,7 +593,9 @@ if ($result.Detail -ne 'exact readiness probe failed') { command = wrapped[len(prefix) : -1].replace('""', '"') root = r"C:\Pinned Vignette Release" + # 액션은 wscript 런처를 거친다 — powershell.exe 직접 실행은 conhost 창이 번쩍인다. arguments = ( + f'"{root}\\scripts\\watch-public-runtime-task.vbs" ' f'-File "{root}\\scripts\\watch-public-runtime.ps1" ' f'-StableSourceRoot "{root}" ' f"-ExpectedSourceCommit {'a' * 40} " @@ -597,7 +607,7 @@ if ($result.Detail -ne 'exact readiness probe failed') { "$script:watchdogTriggered=$false;" "function Get-ScheduledTask { param($TaskName,$ErrorAction) " "$action=[pscustomobject]@{" - "Execute='C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe';" + "Execute='C:\\Windows\\System32\\wscript.exe';" f"WorkingDirectory='{root}';Arguments='{arguments}'" "};[pscustomobject]@{Actions=@($action)}};" "function Start-ScheduledTask { param($TaskName) " diff --git a/scripts/watch-public-runtime-hidden.vbs b/scripts/watch-public-runtime-hidden.vbs index eecca18..bbac35d 100644 --- a/scripts/watch-public-runtime-hidden.vbs +++ b/scripts/watch-public-runtime-hidden.vbs @@ -14,9 +14,11 @@ Set objShell = CreateObject("WScript.Shell") command = "$ErrorActionPreference='Stop';" & _ "$task=Get-ScheduledTask -TaskName 'VignettePublicRuntimeWatchdog' -ErrorAction Stop;" & _ "$action=@($task.Actions)[0];" & _ - "if([IO.Path]::GetFileName($action.Execute) -ne 'powershell.exe'){throw 'Pinned watchdog task must execute powershell.exe'};" & _ + "if([IO.Path]::GetFileName($action.Execute) -ne 'wscript.exe'){throw 'Pinned watchdog task must execute wscript.exe'};" & _ "$root=$action.WorkingDirectory;if([string]::IsNullOrWhiteSpace($root)){throw 'Pinned watchdog working directory is missing'};" & _ "$quote=[char]34;$expectedScript=Join-Path $root 'scripts\watch-public-runtime.ps1';" & _ + "$expectedLauncher=Join-Path $root 'scripts\watch-public-runtime-task.vbs';" & _ + "if($action.Arguments.IndexOf($quote+$expectedLauncher+$quote,[StringComparison]::OrdinalIgnoreCase) -lt 0){throw 'Pinned watchdog launcher does not match its working directory'};" & _ "$expectedFileArg='-File '+$quote+$expectedScript+$quote;$expectedRootArg='-StableSourceRoot '+$quote+$root+$quote;" & _ "if($action.Arguments.IndexOf($expectedFileArg,[StringComparison]::OrdinalIgnoreCase) -lt 0){throw 'Pinned watchdog script path does not match its working directory'};" & _ "if($action.Arguments.IndexOf($expectedRootArg,[StringComparison]::OrdinalIgnoreCase) -lt 0){throw 'Pinned watchdog source root does not match its working directory'};" & _ diff --git a/scripts/watch-public-runtime-task.vbs b/scripts/watch-public-runtime-task.vbs new file mode 100644 index 0000000..9c59687 --- /dev/null +++ b/scripts/watch-public-runtime-task.vbs @@ -0,0 +1,45 @@ +' VignettePublicRuntimeWatchdog 작업 스케줄러 액션용 숨김 런처. +' +' 작업 스케줄러가 powershell.exe 를 직접 실행하면 -WindowStyle Hidden 이어도 conhost +' 콘솔 창이 실행 순간 번쩍인다. 워치독은 5분 주기라 그 번쩍임이 계속 화면을 가린다 +' (2026-08-08/09/12 세 번 반복된 증상). wscript 로 Run(cmd, 0) 하면 창이 아예 생성되지 +' 않는다. GlassDeck scripts\sync-ai-auth-hidden.vbs 와 동일 패턴이다. +' +' 이 래퍼는 pin 인자를 해석하지 않는다. 등록된 인자를 그대로 전달만 하고, provenance +' 검증(detached HEAD / clean tree / SHA pin)은 watch-public-runtime.ps1 이 수행한다. +' 경로를 하드코딩하지 않으므로 stable source root 가 바뀌어도 이 파일은 그대로 쓴다. +Option Explicit + +Dim objShell, args, i, arg, cmd, exitCode, dryRun + +Set args = WScript.Arguments +If args.Count = 0 Then + WScript.Echo "usage: watch-public-runtime-task.vbs -File -StableSourceRoot ..." + WScript.Quit 2 +End If + +If LCase(args(0)) = "syntax-only" Then + WScript.Quit 0 +End If + +dryRun = False +cmd = "powershell.exe -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden" +For i = 0 To args.Count - 1 + arg = args(i) + If i = 0 And LCase(arg) = "print-command" Then + dryRun = True + ElseIf InStr(arg, " ") > 0 Then + cmd = cmd & " """ & arg & """" + Else + cmd = cmd & " " & arg + End If +Next + +If dryRun Then + WScript.Echo cmd + WScript.Quit 0 +End If + +Set objShell = CreateObject("WScript.Shell") +exitCode = objShell.Run(cmd, 0, True) +WScript.Quit exitCode diff --git a/scripts/watch-public-runtime.ps1 b/scripts/watch-public-runtime.ps1 index 6513125..5ee9cdb 100644 --- a/scripts/watch-public-runtime.ps1 +++ b/scripts/watch-public-runtime.ps1 @@ -18,6 +18,8 @@ [int]$EnginePort = 9099, [int]$WhisperPort = 9882, [int]$MeloTtsPort = 9883, + [int]$DbPort = 55432, + [string]$DbContainer = "vignette-dev-db", [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", @@ -285,6 +287,48 @@ function Test-CloudflaredProcess { } } +function Test-DatabaseListening { + $ok = $false + try { + $client = New-Object System.Net.Sockets.TcpClient + $async = $client.BeginConnect("127.0.0.1", $DbPort, $null, $null) + $ok = $async.AsyncWaitHandle.WaitOne(3000) -and $client.Connected + $client.Close() + } catch { + $ok = $false + } + + [pscustomobject]@{ + Name = "db" + Ok = $ok + Detail = if ($ok) { "127.0.0.1:$DbPort" } else { "not listening on 127.0.0.1:$DbPort" } + } +} + +# postgres(docker: vignette-dev-db)는 start-public-runtime.ps1의 복구 범위 밖이다(boot 담당). +# DB가 내려간 채로 start만 반복 호출하면 API가 매번 startup에서 죽어 워치독은 영원히 +# 실패만 기록한다 — 2026-08-12 PC 비정상 종료 후 5분마다 실패 창이 뜬 사건의 구조다. +# 재시작 전에 컨테이너를 직접 되살린다. +function Restore-DatabaseContainer { + try { + $output = & docker.exe start $DbContainer 2>&1 + $output | ForEach-Object { Write-WatchdogLog " docker> $_" } + if ($LASTEXITCODE -ne 0) { + Write-WatchdogLog "db restore: docker start exit=$LASTEXITCODE" + return $false + } + } catch { + Write-WatchdogLog "db restore failed: $($_.Exception.Message)" + return $false + } + + for ($i = 1; $i -le 30; $i++) { + if ((Test-DatabaseListening).Ok) { return $true } + Start-Sleep -Seconds 2 + } + return $false +} + # engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. # /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은" # 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0). @@ -312,7 +356,8 @@ $checks = @( -Uri "http://127.0.0.1:$WebPort/" ` -IsHealthy { param($body) $true }), (Test-VoiceSidecarStack), - (Test-CloudflaredProcess) + (Test-CloudflaredProcess), + (Test-DatabaseListening) ) if (!$SkipPublicHealth) { @@ -352,6 +397,19 @@ if ($failCount -lt $FailuresBeforeRestart) { exit 1 } +# DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당). +# 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에 +# start를 반복 호출하는 것이 5분 주기 실패 로그의 정체였다. +if (-not (@($checks | Where-Object { $_.Name -eq "db" })[0]).Ok) { + Write-WatchdogLog "db down; restoring container '$DbContainer' before runtime restart" + if (Restore-DatabaseContainer) { + Write-WatchdogLog "db restored: 127.0.0.1:$DbPort" + } else { + Write-WatchdogLog "ERROR: db restore failed; skipping runtime restart (start script cannot fix a missing DB)" + exit 1 + } +} + $startArgs = @{ Workspace = $resolvedSourceRoot ApiPort = $ApiPort