+
{/* D1: 섹션 라벨 — dot 제거 */}
내담자와 쌓은 라포
회기를 건너 누적됩니다.
diff --git a/scripts/boot-public-runtime.ps1 b/scripts/boot-public-runtime.ps1
index 44c3ed0..c5776d2 100644
--- a/scripts/boot-public-runtime.ps1
+++ b/scripts/boot-public-runtime.ps1
@@ -327,10 +327,15 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiH
Write-BootLog "web preview down on 127.0.0.1:$WebPort; including web in restart"
}
Write-BootLog ("running start-public-runtime.ps1 " + ($startArgs -join " "))
- $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript @startArgs 2>&1
- $out | ForEach-Object { Write-BootLog (" pub> " + $_) }
- if ($LASTEXITCODE -ne 0) {
- Write-BootLog "ERROR: start-public-runtime.ps1 exit $LASTEXITCODE"
+ $startFailed = $false
+ try {
+ & $startScript @startArgs |
+ ForEach-Object { Write-BootLog (" pub> " + $_) }
+ } catch {
+ $startFailed = $true
+ Write-BootLog "ERROR: start-public-runtime.ps1 failed: $($_.Exception.Message)"
+ }
+ if ($startFailed) {
exit 1
}
}
diff --git a/scripts/install-public-runtime-task.ps1 b/scripts/install-public-runtime-task.ps1
index f3a3082..5dea5a6 100644
--- a/scripts/install-public-runtime-task.ps1
+++ b/scripts/install-public-runtime-task.ps1
@@ -127,10 +127,8 @@ $repeatTrigger = New-ScheduledTaskTrigger `
$settings = New-ScheduledTaskSettingsSet `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries `
- -ExecutionTimeLimit (New-TimeSpan -Minutes 10) `
+ -ExecutionTimeLimit (New-TimeSpan -Minutes 60) `
-MultipleInstances IgnoreNew `
- -RestartCount 3 `
- -RestartInterval (New-TimeSpan -Minutes 1) `
-StartWhenAvailable `
-WakeToRun
diff --git a/scripts/register-boot-task.ps1 b/scripts/register-boot-task.ps1
index c13e584..ed9168f 100644
--- a/scripts/register-boot-task.ps1
+++ b/scripts/register-boot-task.ps1
@@ -113,8 +113,9 @@ $principal = New-ScheduledTaskPrincipal `
$settings = New-ScheduledTaskSettingsSet `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries `
+ -MultipleInstances IgnoreNew `
-StartWhenAvailable `
- -ExecutionTimeLimit (New-TimeSpan -Minutes 15)
+ -ExecutionTimeLimit (New-TimeSpan -Minutes 60)
Register-ScheduledTask `
-TaskName $TaskName `
diff --git a/scripts/start-public-runtime.ps1 b/scripts/start-public-runtime.ps1
index c3ec44f..eb029af 100644
--- a/scripts/start-public-runtime.ps1
+++ b/scripts/start-public-runtime.ps1
@@ -6,6 +6,15 @@
[int]$WhisperPort = 9882,
[int]$MeloTtsPort = 9883,
[int]$VoiceSidecarReadySeconds = 300,
+ [ValidateRange(30, 600)]
+ [int]$ApiReadySeconds = 180,
+ [ValidateRange(30, 300)]
+ [int]$VoiceApiReadySeconds = 90,
+ [ValidateRange(60, 1800)]
+ [int]$WebBuildTimeoutSeconds = 600,
+ [string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock",
+ [ValidateRange(0, 300)]
+ [int]$RecoveryLockWaitSeconds = 0,
[string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe",
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
@@ -29,6 +38,46 @@
$ErrorActionPreference = "Stop"
+function Enter-RecoveryLock {
+ param(
+ [string]$LockPath,
+ [int]$WaitSeconds
+ )
+
+ if ([string]::IsNullOrWhiteSpace($LockPath)) {
+ throw "Public runtime recovery lock path is empty"
+ }
+ $resolvedLockPath = [System.IO.Path]::GetFullPath($LockPath)
+ $lockDirectory = [System.IO.Path]::GetDirectoryName($resolvedLockPath)
+ if ([string]::IsNullOrWhiteSpace($lockDirectory)) {
+ throw "Public runtime recovery lock path has no parent directory"
+ }
+ [System.IO.Directory]::CreateDirectory($lockDirectory) | Out-Null
+ $deadline = (Get-Date).AddSeconds($WaitSeconds)
+ do {
+ try {
+ return [System.IO.File]::Open(
+ $resolvedLockPath,
+ [System.IO.FileMode]::OpenOrCreate,
+ [System.IO.FileAccess]::ReadWrite,
+ [System.IO.FileShare]::None
+ )
+ } catch [System.IO.IOException] {
+ if ((Get-Date) -ge $deadline) {
+ throw "Another public runtime recovery is already in progress"
+ }
+ Start-Sleep -Seconds 1
+ }
+ } while ($true)
+}
+
+# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다.
+# lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다.
+$recoveryLock = Enter-RecoveryLock `
+ -LockPath $RecoveryLockPath `
+ -WaitSeconds $RecoveryLockWaitSeconds
+
+try {
# 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다
# 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려
# /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다.
@@ -344,6 +393,37 @@ function Stop-ProcessesBounded {
throw "Timed out stopping $Role process IDs: $($remaining -join ',')"
}
+function Stop-ProcessTreeBounded {
+ param(
+ [int]$RootProcessId,
+ [int]$TimeoutSec,
+ [string]$Role
+ )
+
+ $taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe"
+ $previousErrorActionPreference = $ErrorActionPreference
+ try {
+ $ErrorActionPreference = "Continue"
+ & $taskkill @("/PID", "$RootProcessId", "/T", "/F") | Out-Null
+ $taskkillExit = $LASTEXITCODE
+ } finally {
+ $ErrorActionPreference = $previousErrorActionPreference
+ }
+ if ($taskkillExit -ne 0) {
+ throw "Failed to stop $Role process tree rooted at PID $RootProcessId (taskkill exit=$taskkillExit)"
+ }
+
+ $deadline = (Get-Date).AddSeconds($TimeoutSec)
+ do {
+ if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) {
+ return @($RootProcessId)
+ }
+ Start-Sleep -Milliseconds 200
+ } while ((Get-Date) -lt $deadline)
+
+ throw "Timed out stopping $Role process tree rooted at PID $RootProcessId"
+}
+
function Get-UvicornProcessesByPort {
param(
[string]$AppImport,
@@ -1470,11 +1550,11 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) {
$health.db -eq $true -and
$health.engine -eq $true
} `
- -TimeoutSec 30
+ -TimeoutSec $ApiReadySeconds
$voiceHealth = Wait-JsonHealth `
-Uri "http://127.0.0.1:$ApiPort/voice/health" `
-IsHealthy { param($health) Test-VoiceApiReady -Health $health } `
- -TimeoutSec 30
+ -TimeoutSec $VoiceApiReadySeconds
}
if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) {
throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)"
@@ -1493,8 +1573,17 @@ if (!$SkipWebRestart) {
-ArgumentList @("/c", "npm run build") `
-WorkingDirectory $WebDir `
-NoNewWindow `
- -Wait `
-PassThru
+ if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) {
+ $null = @(
+ Stop-ProcessTreeBounded `
+ -RootProcessId $build.Id `
+ -TimeoutSec $ProcessStopTimeoutSeconds `
+ -Role "web build"
+ )
+ throw "Web build timed out after $WebBuildTimeoutSeconds seconds"
+ }
+ $build.Refresh()
if ($build.ExitCode -ne 0) {
throw "Web build failed with exit code $($build.ExitCode)"
}
@@ -1734,3 +1823,12 @@ if (!$SkipWebRestart) {
}
Write-Output "Health: $($health | ConvertTo-Json -Compress)"
Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)"
+} finally {
+ if ($null -ne $recoveryLock) {
+ try {
+ $recoveryLock.Dispose()
+ } catch {
+ Write-Warning "Public runtime recovery lock release failed: $($_.Exception.Message)"
+ }
+ }
+}
diff --git a/scripts/test_public_runtime_watchdog_provenance.py b/scripts/test_public_runtime_watchdog_provenance.py
index bffe07a..db8b877 100644
--- a/scripts/test_public_runtime_watchdog_provenance.py
+++ b/scripts/test_public_runtime_watchdog_provenance.py
@@ -98,7 +98,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
)
gate = BOOT_SOURCE.index("Assert-StableSourceProvenance", gate_comment)
docker_mutation = BOOT_SOURCE.index("docker.exe update")
- runtime_recovery = BOOT_SOURCE.index("-File $startScript")
+ runtime_recovery = BOOT_SOURCE.index("& $startScript @startArgs |")
self.assertLess(gate, docker_mutation)
self.assertLess(gate, runtime_recovery)
@@ -120,6 +120,115 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask")
self.assertLess(dirty_gate, task_registration)
+ def test_task_limits_cover_a_full_cold_start_without_overlap(self) -> None:
+ self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", BOOT_REGISTER_SOURCE)
+ self.assertIn("-MultipleInstances IgnoreNew", BOOT_REGISTER_SOURCE)
+ self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", INSTALLER_SOURCE)
+ self.assertIn("-MultipleInstances IgnoreNew", INSTALLER_SOURCE)
+ self.assertNotIn("-RestartCount", INSTALLER_SOURCE)
+ self.assertNotIn("-RestartInterval", INSTALLER_SOURCE)
+
+ # 최초 health 진단 + DB 복구 + engine/voice/API/web의 각 bounded wait와
+ # process-stop 여유를 모두 합쳐도 작업 스케줄러 60분보다 5분 이상 짧다.
+ initial_health_seconds = 210
+ recovery_seconds = 60 + 90 + (300 * 2) + 180 + 90 + 600 + 30 + (15 * 4)
+ safety_margin_seconds = 300
+ task_limit_seconds = 60 * 60
+ self.assertLess(
+ initial_health_seconds + recovery_seconds + safety_margin_seconds,
+ task_limit_seconds,
+ )
+ boot_prerequisite_seconds = 360 + 90
+ self.assertLess(
+ boot_prerequisite_seconds + recovery_seconds + safety_margin_seconds,
+ task_limit_seconds,
+ )
+
+ def test_hard_down_bypasses_transient_failure_debounce(self) -> None:
+ hard_down = WATCHDOG_SOURCE.index("$hardDown =")
+ debounce = WATCHDOG_SOURCE.index(
+ "if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown)"
+ )
+ restart = WATCHDOG_SOURCE.index("& $startScript @startArgs")
+ self.assertLess(hard_down, debounce)
+ self.assertLess(debounce, restart)
+ self.assertIn('($FailedNames -contains "cloudflared")', WATCHDOG_SOURCE)
+ self.assertIn('($FailedNames -contains "api")', WATCHDOG_SOURCE)
+ self.assertIn('($FailedNames -contains "web-preview")', WATCHDOG_SOURCE)
+ self.assertIn('($FailedNames -contains "voice-sidecars")', WATCHDOG_SOURCE)
+ self.assertIn(
+ "immediate restart: public runtime hard-down detected",
+ WATCHDOG_SOURCE,
+ )
+
+ def test_hard_down_classifier_distinguishes_total_and_transient_failures(self) -> None:
+ powershell = shutil.which("powershell.exe")
+ if powershell is None:
+ self.skipTest("Windows PowerShell 5.1 is not available")
+
+ classifier = WATCHDOG_SOURCE[
+ WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown") :
+ WATCHDOG_SOURCE.index("# engine 판정은", WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown"))
+ ].strip()
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ harness = Path(temporary_directory) / "hard-down.ps1"
+ harness.write_text(
+ classifier
+ + r'''
+if (-not (Test-PublicRuntimeHardDown -FailedNames @('cloudflared'))) { exit 2 }
+if (-not (Test-PublicRuntimeHardDown -FailedNames @('api','web-preview','voice-sidecars'))) { exit 3 }
+if (Test-PublicRuntimeHardDown -FailedNames @('engine')) { exit 4 }
+exit 0
+''',
+ encoding="utf-8-sig",
+ )
+ completed = subprocess.run(
+ [
+ powershell,
+ "-NoLogo",
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-File",
+ str(harness),
+ ],
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ errors="replace",
+ timeout=30,
+ check=False,
+ )
+ self.assertEqual(
+ completed.returncode,
+ 0,
+ msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
+ )
+
+ def test_check_only_exits_before_failcount_log_or_recovery_mutation(self) -> None:
+ failed = WATCHDOG_SOURCE.index("$failed = @(")
+ check_only = WATCHDOG_SOURCE.index("if ($CheckOnly)", failed)
+ inner_health = WATCHDOG_SOURCE.index("if ($failed.Count -eq 0)", check_only)
+ normal_health = WATCHDOG_SOURCE.index(
+ "if ($failed.Count -eq 0)", inner_health + 1
+ )
+ fail_count = WATCHDOG_SOURCE.index("$failCount =", check_only)
+ restart = WATCHDOG_SOURCE.index("& $startScript @startArgs", fail_count)
+ self.assertLess(check_only, fail_count)
+ self.assertLess(check_only, restart)
+ check_only_contract = WATCHDOG_SOURCE[check_only:normal_health]
+ self.assertIn("Write-Output", check_only_contract)
+ self.assertNotIn("Set-FailCount", check_only_contract)
+ self.assertNotIn("Write-WatchdogLog", check_only_contract)
+ self.assertNotIn("Restore-DatabaseContainer", check_only_contract)
+
+ def test_boot_streams_start_script_progress_without_nested_powershell_buffering(self) -> None:
+ self.assertIn("& $startScript @startArgs |", BOOT_SOURCE)
+ self.assertIn('Write-BootLog (" pub> " + $_)', BOOT_SOURCE)
+ self.assertIn("start-public-runtime.ps1 failed", BOOT_SOURCE)
+ self.assertNotIn("$out = & powershell.exe", BOOT_SOURCE)
+
def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None:
for source in (WATCHDOG_SOURCE, BOOT_SOURCE):
for expected in (
diff --git a/scripts/test_start_public_runtime_contract.py b/scripts/test_start_public_runtime_contract.py
index b55442e..81fef78 100644
--- a/scripts/test_start_public_runtime_contract.py
+++ b/scripts/test_start_public_runtime_contract.py
@@ -16,6 +16,160 @@ WHISPER_START = (SCRIPTS / "start-local-whisper-stt.ps1").read_text(
class PublicRuntimeVoiceContractTest(unittest.TestCase):
+ def test_recovery_is_serialized_and_lock_is_always_released(self) -> None:
+ lock = PUBLIC_RUNTIME.index("$recoveryLock = Enter-RecoveryLock")
+ main_try = PUBLIC_RUNTIME.index("try {", lock)
+ first_runtime_mutation = PUBLIC_RUNTIME.index("Stop-UvicornByPort", main_try)
+ finalizer = PUBLIC_RUNTIME.rindex("} finally {")
+ dispose = PUBLIC_RUNTIME.index("$recoveryLock.Dispose()", finalizer)
+ self.assertLess(lock, main_try)
+ self.assertLess(main_try, first_runtime_mutation)
+ self.assertLess(finalizer, dispose)
+ self.assertIn(
+ '"$env:LOCALAPPDATA\\Vignette\\public-runtime-start.lock"',
+ PUBLIC_RUNTIME,
+ )
+
+ def test_recovery_lock_is_exclusive_and_reusable_in_windows_powershell(self) -> None:
+ powershell = shutil.which("powershell.exe")
+ if powershell is None:
+ self.skipTest("Windows PowerShell 5.1 is not available")
+
+ lock_function = PUBLIC_RUNTIME[
+ PUBLIC_RUNTIME.index("function Enter-RecoveryLock") :
+ PUBLIC_RUNTIME.index(
+ "# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다."
+ )
+ ].strip()
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ lock_path = str(Path(temporary_directory) / "runtime.lock").replace("'", "''")
+ harness = Path(temporary_directory) / "recovery-lock.ps1"
+ harness.write_text(
+ lock_function
+ + f"""
+$first = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0
+try {{
+ try {{
+ $second = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0
+ exit 2
+ }} catch {{
+ if ($_.Exception.Message -notlike '*already in progress*') {{ exit 3 }}
+ }}
+}} finally {{
+ $first.Dispose()
+}}
+$third = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0
+$third.Dispose()
+exit 0
+""",
+ encoding="utf-8-sig",
+ )
+ completed = subprocess.run(
+ [
+ powershell,
+ "-NoLogo",
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-File",
+ str(harness),
+ ],
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ errors="replace",
+ timeout=30,
+ check=False,
+ )
+ self.assertEqual(
+ completed.returncode,
+ 0,
+ msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
+ )
+
+ def test_cold_start_waits_are_bounded_and_configurable(self) -> None:
+ for expected in (
+ "[int]$ApiReadySeconds = 180",
+ "[int]$VoiceApiReadySeconds = 90",
+ "[int]$WebBuildTimeoutSeconds = 600",
+ "-TimeoutSec $ApiReadySeconds",
+ "-TimeoutSec $VoiceApiReadySeconds",
+ "$build.WaitForExit($WebBuildTimeoutSeconds * 1000)",
+ "Web build timed out after $WebBuildTimeoutSeconds seconds",
+ ):
+ with self.subTest(expected=expected):
+ self.assertIn(expected, PUBLIC_RUNTIME)
+
+ def test_web_build_timeout_stops_the_owned_process_tree(self) -> None:
+ powershell = shutil.which("powershell.exe")
+ if powershell is None:
+ self.skipTest("Windows PowerShell 5.1 is not available")
+
+ tree_functions = PUBLIC_RUNTIME[
+ PUBLIC_RUNTIME.index("function Stop-ProcessTreeBounded") :
+ PUBLIC_RUNTIME.index("function Get-UvicornProcessesByPort")
+ ].strip()
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ harness = Path(temporary_directory) / "process-tree.ps1"
+ child = Path(temporary_directory) / "child-sleeper.ps1"
+ child_pid = Path(temporary_directory) / "child.pid"
+ quoted_child_pid = str(child_pid).replace("'", "''")
+ child.write_text(
+ f"$PID | Set-Content -LiteralPath '{quoted_child_pid}' -Encoding ascii\n"
+ "Start-Sleep -Seconds 300\n",
+ encoding="utf-8-sig",
+ )
+ quoted_child = str(child).replace("'", "''")
+ quoted_child_pid_for_harness = str(child_pid).replace("'", "''")
+ harness.write_text(
+ tree_functions
+ + f'''
+$root = Start-Process -FilePath 'cmd.exe' `
+ -ArgumentList @('/c', 'powershell.exe -NoProfile -ExecutionPolicy Bypass -File "{quoted_child}"') `
+ -WindowStyle Hidden `
+ -PassThru
+try {{
+ $deadline = (Get-Date).AddSeconds(10)
+ while (-not (Test-Path -LiteralPath '{quoted_child_pid_for_harness}') -and (Get-Date) -lt $deadline) {{
+ Start-Sleep -Milliseconds 100
+ }}
+ if (-not (Test-Path -LiteralPath '{quoted_child_pid_for_harness}')) {{ exit 2 }}
+ $childProcessId = [int](Get-Content -LiteralPath '{quoted_child_pid_for_harness}' -Raw)
+ $stopped = @(Stop-ProcessTreeBounded -RootProcessId $root.Id -TimeoutSec 10 -Role 'test tree')
+ if ($null -ne (Get-Process -Id $root.Id -ErrorAction SilentlyContinue)) {{ exit 3 }}
+ if ($null -ne (Get-Process -Id $childProcessId -ErrorAction SilentlyContinue)) {{ exit 4 }}
+ exit 0
+}} finally {{
+ Stop-Process -Id $root.Id -Force -ErrorAction SilentlyContinue
+}}
+''',
+ encoding="utf-8-sig",
+ )
+ completed = subprocess.run(
+ [
+ powershell,
+ "-NoLogo",
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-File",
+ str(harness),
+ ],
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ errors="replace",
+ timeout=30,
+ check=False,
+ )
+ self.assertEqual(
+ completed.returncode,
+ 0,
+ msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
+ )
+
def test_public_provider_model_and_loopback_environment_are_explicit(self) -> None:
for expected in (
'$WhisperModel = "small"',
diff --git a/scripts/watch-public-runtime.ps1 b/scripts/watch-public-runtime.ps1
index 5ee9cdb..6563cc7 100644
--- a/scripts/watch-public-runtime.ps1
+++ b/scripts/watch-public-runtime.ps1
@@ -311,7 +311,7 @@ function Test-DatabaseListening {
# 재시작 전에 컨테이너를 직접 되살린다.
function Restore-DatabaseContainer {
try {
- $output = & docker.exe start $DbContainer 2>&1
+ $output = & docker.exe start $DbContainer 2>$null
$output | ForEach-Object { Write-WatchdogLog " docker> $_" }
if ($LASTEXITCODE -ne 0) {
Write-WatchdogLog "db restore: docker start exit=$LASTEXITCODE"
@@ -329,6 +329,17 @@ function Restore-DatabaseContainer {
return $false
}
+function Test-PublicRuntimeHardDown {
+ param([string[]]$FailedNames)
+
+ $allLocalSurfacesDown = (
+ ($FailedNames -contains "api") -and
+ ($FailedNames -contains "web-preview") -and
+ ($FailedNames -contains "voice-sidecars")
+ )
+ return ($FailedNames -contains "cloudflared") -or $allLocalSurfacesDown
+}
+
# engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다.
# /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은"
# 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0).
@@ -376,6 +387,14 @@ if (!$SkipPublicHealth) {
}
$failed = @($checks | Where-Object { -not $_.Ok })
+if ($CheckOnly) {
+ if ($failed.Count -eq 0) {
+ Write-Output "healthy: $($checks.Name -join ', ')"
+ exit 0
+ }
+ Write-Output "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
+ exit 1
+}
if ($failed.Count -eq 0) {
Set-FailCount 0
Write-WatchdogLog "healthy: $($checks.Name -join ', ')"
@@ -385,17 +404,20 @@ if ($failed.Count -eq 0) {
$failCount = (Get-FailCount) + 1
Set-FailCount $failCount
Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))"
-if ($CheckOnly) {
- exit 1
-}
# 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라
# 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다.
-# 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다.
-if ($failCount -lt $FailuresBeforeRestart) {
+# 다만 cloudflared가 사라졌거나 API·web·voice sidecar가 함께 죽은 경우는 transient
+# readiness가 아니라 공개 서비스 전체 소실이므로 첫 감지에서 즉시 복구한다.
+$failedNames = @($failed | ForEach-Object { $_.Name })
+$hardDown = Test-PublicRuntimeHardDown -FailedNames $failedNames
+if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown) {
Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)"
exit 1
}
+if ($hardDown -and $failCount -lt $FailuresBeforeRestart) {
+ Write-WatchdogLog "immediate restart: public runtime hard-down detected"
+}
# DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당).
# 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에
@@ -429,7 +451,7 @@ if ($SkipCloudflaredRestart) {
}
try {
- & $startScript @startArgs 2>&1 | ForEach-Object {
+ & $startScript @startArgs | ForEach-Object {
Write-WatchdogLog "$_"
}
} catch {