From 99779a6ab28e687cdba6533f2a104a68ac89fbc8 Mon Sep 17 00:00:00 2001 From: Yun Chan Date: Fri, 28 Aug 2026 18:07:28 +0900 Subject: [PATCH] =?UTF-8?q?=EC=9E=AC=EB=B6=80=ED=8C=85=20=EB=B3=B5?= =?UTF-8?q?=EA=B5=AC=EC=99=80=20=ED=95=99=EC=8A=B5=EC=9E=90=20=ED=8C=A8?= =?UTF-8?q?=EB=84=90=EC=9D=84=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/web/design.md | 2 + .../web/e2e/uc-learner-home-dashboard.spec.ts | 33 ++++ apps/web/src/pages/LearnerHome.tsx | 7 +- scripts/boot-public-runtime.ps1 | 13 +- scripts/install-public-runtime-task.ps1 | 4 +- scripts/register-boot-task.ps1 | 3 +- scripts/start-public-runtime.ps1 | 104 +++++++++++- ...test_public_runtime_watchdog_provenance.py | 111 ++++++++++++- scripts/test_start_public_runtime_contract.py | 154 ++++++++++++++++++ scripts/watch-public-runtime.ps1 | 36 +++- 10 files changed, 442 insertions(+), 25 deletions(-) diff --git a/apps/web/design.md b/apps/web/design.md index 1fb6db3..dff5998 100644 --- a/apps/web/design.md +++ b/apps/web/design.md @@ -12,3 +12,5 @@ - 320px과 모바일 프로젝트에서도 문서 가로 넘침이 없어야 하며, 오류·재시도 상태는 기존 관리자/리뷰 알림 패턴을 따른다. 국소 구현 소유자는 `src/pages/Admin.tsx`, `src/pages/admin/admin-console.css`, `src/pages/SessionReview.tsx`, `src/pages/session-review/`다. 공통 토큰과 앱 셸은 각각 `src/styles/tokens.css`, `src/components/shell/`의 소유권을 유지한다. + +- 2026-08-28 · `/learn` 성장 지표의 라포 헤더에서 중첩 inset surface를 제거해 같은 패널의 일반 헤더 위계와 정렬했다. 토큰 변경 없음. diff --git a/apps/web/e2e/uc-learner-home-dashboard.spec.ts b/apps/web/e2e/uc-learner-home-dashboard.spec.ts index b5e5599..4776599 100644 --- a/apps/web/e2e/uc-learner-home-dashboard.spec.ts +++ b/apps/web/e2e/uc-learner-home-dashboard.spec.ts @@ -700,6 +700,39 @@ test.describe("uc learner home dashboard", () => { await expect(page).toHaveURL(/\/learn\/practice$/); }); + test("성장 지표 라포 헤더는 중첩 inset 표면을 사용하지 않는다", async ({ + page, + }) => { + await installFixtures(page); + await page.goto("/learn"); + + await page.getByRole("tab", { name: "성장 지표" }).click(); + const rapportHeading = page + .locator("#lh-panel-growth") + .getByRole("heading", { name: "회기를 건너 누적됩니다." }); + const headingContent = rapportHeading.locator(".."); + await expect(rapportHeading).toBeVisible(); + await expect(headingContent).not.toHaveClass(/vg-surface/); + await expect + .poll(() => + headingContent.evaluate((element) => { + const style = getComputedStyle(element); + return { + backgroundColor: style.backgroundColor, + borderTopWidth: style.borderTopWidth, + borderRadius: style.borderRadius, + padding: style.padding, + }; + }), + ) + .toEqual({ + backgroundColor: "rgba(0, 0, 0, 0)", + borderTopWidth: "0px", + borderRadius: "0px", + padding: "0px", + }); + }); + // usecase: 첫 방문 학습자가 기록·성장 탭에서 빈 상태 안내 문구를 확인한다 test("신규 학습자의 기록·성장 탭이 빈 상태 안내 문구를 보여준다", async ({ page, diff --git a/apps/web/src/pages/LearnerHome.tsx b/apps/web/src/pages/LearnerHome.tsx index f1d1785..40512d7 100644 --- a/apps/web/src/pages/LearnerHome.tsx +++ b/apps/web/src/pages/LearnerHome.tsx @@ -1554,12 +1554,7 @@ export default function LearnerHome({ view = "dashboard" }: LearnerHomeProps) { className={surfaceClassName("lh-panel lh-panel--growth")} >
-
+
{/* D1: 섹션 라벨 — dot 제거 */} 내담자와 쌓은 라포

회기를 건너 누적됩니다.

diff --git a/scripts/boot-public-runtime.ps1 b/scripts/boot-public-runtime.ps1 index 44c3ed0..c5776d2 100644 --- a/scripts/boot-public-runtime.ps1 +++ b/scripts/boot-public-runtime.ps1 @@ -327,10 +327,15 @@ if ((Test-ApiControlPlaneHealthy) -and (Test-EngineHealthy) -and (Test-VoiceApiH Write-BootLog "web preview down on 127.0.0.1:$WebPort; including web in restart" } Write-BootLog ("running start-public-runtime.ps1 " + ($startArgs -join " ")) - $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startScript @startArgs 2>&1 - $out | ForEach-Object { Write-BootLog (" pub> " + $_) } - if ($LASTEXITCODE -ne 0) { - Write-BootLog "ERROR: start-public-runtime.ps1 exit $LASTEXITCODE" + $startFailed = $false + try { + & $startScript @startArgs | + ForEach-Object { Write-BootLog (" pub> " + $_) } + } catch { + $startFailed = $true + Write-BootLog "ERROR: start-public-runtime.ps1 failed: $($_.Exception.Message)" + } + if ($startFailed) { exit 1 } } diff --git a/scripts/install-public-runtime-task.ps1 b/scripts/install-public-runtime-task.ps1 index f3a3082..5dea5a6 100644 --- a/scripts/install-public-runtime-task.ps1 +++ b/scripts/install-public-runtime-task.ps1 @@ -127,10 +127,8 @@ $repeatTrigger = New-ScheduledTaskTrigger ` $settings = New-ScheduledTaskSettingsSet ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` - -ExecutionTimeLimit (New-TimeSpan -Minutes 10) ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 60) ` -MultipleInstances IgnoreNew ` - -RestartCount 3 ` - -RestartInterval (New-TimeSpan -Minutes 1) ` -StartWhenAvailable ` -WakeToRun diff --git a/scripts/register-boot-task.ps1 b/scripts/register-boot-task.ps1 index c13e584..ed9168f 100644 --- a/scripts/register-boot-task.ps1 +++ b/scripts/register-boot-task.ps1 @@ -113,8 +113,9 @@ $principal = New-ScheduledTaskPrincipal ` $settings = New-ScheduledTaskSettingsSet ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` + -MultipleInstances IgnoreNew ` -StartWhenAvailable ` - -ExecutionTimeLimit (New-TimeSpan -Minutes 15) + -ExecutionTimeLimit (New-TimeSpan -Minutes 60) Register-ScheduledTask ` -TaskName $TaskName ` diff --git a/scripts/start-public-runtime.ps1 b/scripts/start-public-runtime.ps1 index c3ec44f..eb029af 100644 --- a/scripts/start-public-runtime.ps1 +++ b/scripts/start-public-runtime.ps1 @@ -6,6 +6,15 @@ [int]$WhisperPort = 9882, [int]$MeloTtsPort = 9883, [int]$VoiceSidecarReadySeconds = 300, + [ValidateRange(30, 600)] + [int]$ApiReadySeconds = 180, + [ValidateRange(30, 300)] + [int]$VoiceApiReadySeconds = 90, + [ValidateRange(60, 1800)] + [int]$WebBuildTimeoutSeconds = 600, + [string]$RecoveryLockPath = "$env:LOCALAPPDATA\Vignette\public-runtime-start.lock", + [ValidateRange(0, 300)] + [int]$RecoveryLockWaitSeconds = 0, [string]$Python = "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", [string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe", [string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml", @@ -29,6 +38,46 @@ $ErrorActionPreference = "Stop" +function Enter-RecoveryLock { + param( + [string]$LockPath, + [int]$WaitSeconds + ) + + if ([string]::IsNullOrWhiteSpace($LockPath)) { + throw "Public runtime recovery lock path is empty" + } + $resolvedLockPath = [System.IO.Path]::GetFullPath($LockPath) + $lockDirectory = [System.IO.Path]::GetDirectoryName($resolvedLockPath) + if ([string]::IsNullOrWhiteSpace($lockDirectory)) { + throw "Public runtime recovery lock path has no parent directory" + } + [System.IO.Directory]::CreateDirectory($lockDirectory) | Out-Null + $deadline = (Get-Date).AddSeconds($WaitSeconds) + do { + try { + return [System.IO.File]::Open( + $resolvedLockPath, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) + } catch [System.IO.IOException] { + if ((Get-Date) -ge $deadline) { + throw "Another public runtime recovery is already in progress" + } + Start-Sleep -Seconds 1 + } + } while ($true) +} + +# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다. +# lock 파일은 stable Git root 밖에 두고 FileShare.None 핸들 수명으로만 소유권을 가진다. +$recoveryLock = Enter-RecoveryLock ` + -LockPath $RecoveryLockPath ` + -WaitSeconds $RecoveryLockWaitSeconds + +try { # 엔진 readiness 캐시 TTL. 기본 30초는 워치독 주기(5분)보다 짧아 매 헬스체크마다 # 실제 claude -p 생성을 새로 돌리게 만든다(재시작 폭풍의 근본 원인). 크게 늘려 # /ready 가 거의 항상 캐시를 반환하게 한다 → 헬스체크가 LLM 호출에 묶이지 않는다. @@ -344,6 +393,37 @@ function Stop-ProcessesBounded { throw "Timed out stopping $Role process IDs: $($remaining -join ',')" } +function Stop-ProcessTreeBounded { + param( + [int]$RootProcessId, + [int]$TimeoutSec, + [string]$Role + ) + + $taskkill = Join-Path $env:SystemRoot "System32\taskkill.exe" + $previousErrorActionPreference = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + & $taskkill @("/PID", "$RootProcessId", "/T", "/F") | Out-Null + $taskkillExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + if ($taskkillExit -ne 0) { + throw "Failed to stop $Role process tree rooted at PID $RootProcessId (taskkill exit=$taskkillExit)" + } + + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + if ($null -eq (Get-Process -Id $RootProcessId -ErrorAction SilentlyContinue)) { + return @($RootProcessId) + } + Start-Sleep -Milliseconds 200 + } while ((Get-Date) -lt $deadline) + + throw "Timed out stopping $Role process tree rooted at PID $RootProcessId" +} + function Get-UvicornProcessesByPort { param( [string]$AppImport, @@ -1470,11 +1550,11 @@ if ($apiControlPlaneReady -and -not $ForceApiRestart) { $health.db -eq $true -and $health.engine -eq $true } ` - -TimeoutSec 30 + -TimeoutSec $ApiReadySeconds $voiceHealth = Wait-JsonHealth ` -Uri "http://127.0.0.1:$ApiPort/voice/health" ` -IsHealthy { param($health) Test-VoiceApiReady -Health $health } ` - -TimeoutSec 30 + -TimeoutSec $VoiceApiReadySeconds } if ($health.environment -ne "prod" -or -not $health.db -or -not $health.engine) { throw "Admin/auth control plane is not production-safe: $($health | ConvertTo-Json -Compress)" @@ -1493,8 +1573,17 @@ if (!$SkipWebRestart) { -ArgumentList @("/c", "npm run build") ` -WorkingDirectory $WebDir ` -NoNewWindow ` - -Wait ` -PassThru + if (-not $build.WaitForExit($WebBuildTimeoutSeconds * 1000)) { + $null = @( + Stop-ProcessTreeBounded ` + -RootProcessId $build.Id ` + -TimeoutSec $ProcessStopTimeoutSeconds ` + -Role "web build" + ) + throw "Web build timed out after $WebBuildTimeoutSeconds seconds" + } + $build.Refresh() if ($build.ExitCode -ne 0) { throw "Web build failed with exit code $($build.ExitCode)" } @@ -1734,3 +1823,12 @@ if (!$SkipWebRestart) { } Write-Output "Health: $($health | ConvertTo-Json -Compress)" Write-Output "Voice health: $($voiceHealth | ConvertTo-Json -Compress)" +} finally { + if ($null -ne $recoveryLock) { + try { + $recoveryLock.Dispose() + } catch { + Write-Warning "Public runtime recovery lock release failed: $($_.Exception.Message)" + } + } +} diff --git a/scripts/test_public_runtime_watchdog_provenance.py b/scripts/test_public_runtime_watchdog_provenance.py index bffe07a..db8b877 100644 --- a/scripts/test_public_runtime_watchdog_provenance.py +++ b/scripts/test_public_runtime_watchdog_provenance.py @@ -98,7 +98,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): ) gate = BOOT_SOURCE.index("Assert-StableSourceProvenance", gate_comment) docker_mutation = BOOT_SOURCE.index("docker.exe update") - runtime_recovery = BOOT_SOURCE.index("-File $startScript") + runtime_recovery = BOOT_SOURCE.index("& $startScript @startArgs |") self.assertLess(gate, docker_mutation) self.assertLess(gate, runtime_recovery) @@ -120,6 +120,115 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase): task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask") self.assertLess(dirty_gate, task_registration) + def test_task_limits_cover_a_full_cold_start_without_overlap(self) -> None: + self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", BOOT_REGISTER_SOURCE) + self.assertIn("-MultipleInstances IgnoreNew", BOOT_REGISTER_SOURCE) + self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", INSTALLER_SOURCE) + self.assertIn("-MultipleInstances IgnoreNew", INSTALLER_SOURCE) + self.assertNotIn("-RestartCount", INSTALLER_SOURCE) + self.assertNotIn("-RestartInterval", INSTALLER_SOURCE) + + # 최초 health 진단 + DB 복구 + engine/voice/API/web의 각 bounded wait와 + # process-stop 여유를 모두 합쳐도 작업 스케줄러 60분보다 5분 이상 짧다. + initial_health_seconds = 210 + recovery_seconds = 60 + 90 + (300 * 2) + 180 + 90 + 600 + 30 + (15 * 4) + safety_margin_seconds = 300 + task_limit_seconds = 60 * 60 + self.assertLess( + initial_health_seconds + recovery_seconds + safety_margin_seconds, + task_limit_seconds, + ) + boot_prerequisite_seconds = 360 + 90 + self.assertLess( + boot_prerequisite_seconds + recovery_seconds + safety_margin_seconds, + task_limit_seconds, + ) + + def test_hard_down_bypasses_transient_failure_debounce(self) -> None: + hard_down = WATCHDOG_SOURCE.index("$hardDown =") + debounce = WATCHDOG_SOURCE.index( + "if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown)" + ) + restart = WATCHDOG_SOURCE.index("& $startScript @startArgs") + self.assertLess(hard_down, debounce) + self.assertLess(debounce, restart) + self.assertIn('($FailedNames -contains "cloudflared")', WATCHDOG_SOURCE) + self.assertIn('($FailedNames -contains "api")', WATCHDOG_SOURCE) + self.assertIn('($FailedNames -contains "web-preview")', WATCHDOG_SOURCE) + self.assertIn('($FailedNames -contains "voice-sidecars")', WATCHDOG_SOURCE) + self.assertIn( + "immediate restart: public runtime hard-down detected", + WATCHDOG_SOURCE, + ) + + def test_hard_down_classifier_distinguishes_total_and_transient_failures(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + classifier = WATCHDOG_SOURCE[ + WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown") : + WATCHDOG_SOURCE.index("# engine 판정은", WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown")) + ].strip() + with tempfile.TemporaryDirectory() as temporary_directory: + harness = Path(temporary_directory) / "hard-down.ps1" + harness.write_text( + classifier + + r''' +if (-not (Test-PublicRuntimeHardDown -FailedNames @('cloudflared'))) { exit 2 } +if (-not (Test-PublicRuntimeHardDown -FailedNames @('api','web-preview','voice-sidecars'))) { exit 3 } +if (Test-PublicRuntimeHardDown -FailedNames @('engine')) { exit 4 } +exit 0 +''', + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_check_only_exits_before_failcount_log_or_recovery_mutation(self) -> None: + failed = WATCHDOG_SOURCE.index("$failed = @(") + check_only = WATCHDOG_SOURCE.index("if ($CheckOnly)", failed) + inner_health = WATCHDOG_SOURCE.index("if ($failed.Count -eq 0)", check_only) + normal_health = WATCHDOG_SOURCE.index( + "if ($failed.Count -eq 0)", inner_health + 1 + ) + fail_count = WATCHDOG_SOURCE.index("$failCount =", check_only) + restart = WATCHDOG_SOURCE.index("& $startScript @startArgs", fail_count) + self.assertLess(check_only, fail_count) + self.assertLess(check_only, restart) + check_only_contract = WATCHDOG_SOURCE[check_only:normal_health] + self.assertIn("Write-Output", check_only_contract) + self.assertNotIn("Set-FailCount", check_only_contract) + self.assertNotIn("Write-WatchdogLog", check_only_contract) + self.assertNotIn("Restore-DatabaseContainer", check_only_contract) + + def test_boot_streams_start_script_progress_without_nested_powershell_buffering(self) -> None: + self.assertIn("& $startScript @startArgs |", BOOT_SOURCE) + self.assertIn('Write-BootLog (" pub> " + $_)', BOOT_SOURCE) + self.assertIn("start-public-runtime.ps1 failed", BOOT_SOURCE) + self.assertNotIn("$out = & powershell.exe", BOOT_SOURCE) + def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None: for source in (WATCHDOG_SOURCE, BOOT_SOURCE): for expected in ( diff --git a/scripts/test_start_public_runtime_contract.py b/scripts/test_start_public_runtime_contract.py index b55442e..81fef78 100644 --- a/scripts/test_start_public_runtime_contract.py +++ b/scripts/test_start_public_runtime_contract.py @@ -16,6 +16,160 @@ WHISPER_START = (SCRIPTS / "start-local-whisper-stt.ps1").read_text( class PublicRuntimeVoiceContractTest(unittest.TestCase): + def test_recovery_is_serialized_and_lock_is_always_released(self) -> None: + lock = PUBLIC_RUNTIME.index("$recoveryLock = Enter-RecoveryLock") + main_try = PUBLIC_RUNTIME.index("try {", lock) + first_runtime_mutation = PUBLIC_RUNTIME.index("Stop-UvicornByPort", main_try) + finalizer = PUBLIC_RUNTIME.rindex("} finally {") + dispose = PUBLIC_RUNTIME.index("$recoveryLock.Dispose()", finalizer) + self.assertLess(lock, main_try) + self.assertLess(main_try, first_runtime_mutation) + self.assertLess(finalizer, dispose) + self.assertIn( + '"$env:LOCALAPPDATA\\Vignette\\public-runtime-start.lock"', + PUBLIC_RUNTIME, + ) + + def test_recovery_lock_is_exclusive_and_reusable_in_windows_powershell(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + lock_function = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Enter-RecoveryLock") : + PUBLIC_RUNTIME.index( + "# boot task, watchdog, 수동 승격이 같은 포트와 프로세스를 동시에 교체하지 못하게 한다." + ) + ].strip() + with tempfile.TemporaryDirectory() as temporary_directory: + lock_path = str(Path(temporary_directory) / "runtime.lock").replace("'", "''") + harness = Path(temporary_directory) / "recovery-lock.ps1" + harness.write_text( + lock_function + + f""" +$first = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0 +try {{ + try {{ + $second = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0 + exit 2 + }} catch {{ + if ($_.Exception.Message -notlike '*already in progress*') {{ exit 3 }} + }} +}} finally {{ + $first.Dispose() +}} +$third = Enter-RecoveryLock -LockPath '{lock_path}' -WaitSeconds 0 +$third.Dispose() +exit 0 +""", + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + + def test_cold_start_waits_are_bounded_and_configurable(self) -> None: + for expected in ( + "[int]$ApiReadySeconds = 180", + "[int]$VoiceApiReadySeconds = 90", + "[int]$WebBuildTimeoutSeconds = 600", + "-TimeoutSec $ApiReadySeconds", + "-TimeoutSec $VoiceApiReadySeconds", + "$build.WaitForExit($WebBuildTimeoutSeconds * 1000)", + "Web build timed out after $WebBuildTimeoutSeconds seconds", + ): + with self.subTest(expected=expected): + self.assertIn(expected, PUBLIC_RUNTIME) + + def test_web_build_timeout_stops_the_owned_process_tree(self) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + self.skipTest("Windows PowerShell 5.1 is not available") + + tree_functions = PUBLIC_RUNTIME[ + PUBLIC_RUNTIME.index("function Stop-ProcessTreeBounded") : + PUBLIC_RUNTIME.index("function Get-UvicornProcessesByPort") + ].strip() + with tempfile.TemporaryDirectory() as temporary_directory: + harness = Path(temporary_directory) / "process-tree.ps1" + child = Path(temporary_directory) / "child-sleeper.ps1" + child_pid = Path(temporary_directory) / "child.pid" + quoted_child_pid = str(child_pid).replace("'", "''") + child.write_text( + f"$PID | Set-Content -LiteralPath '{quoted_child_pid}' -Encoding ascii\n" + "Start-Sleep -Seconds 300\n", + encoding="utf-8-sig", + ) + quoted_child = str(child).replace("'", "''") + quoted_child_pid_for_harness = str(child_pid).replace("'", "''") + harness.write_text( + tree_functions + + f''' +$root = Start-Process -FilePath 'cmd.exe' ` + -ArgumentList @('/c', 'powershell.exe -NoProfile -ExecutionPolicy Bypass -File "{quoted_child}"') ` + -WindowStyle Hidden ` + -PassThru +try {{ + $deadline = (Get-Date).AddSeconds(10) + while (-not (Test-Path -LiteralPath '{quoted_child_pid_for_harness}') -and (Get-Date) -lt $deadline) {{ + Start-Sleep -Milliseconds 100 + }} + if (-not (Test-Path -LiteralPath '{quoted_child_pid_for_harness}')) {{ exit 2 }} + $childProcessId = [int](Get-Content -LiteralPath '{quoted_child_pid_for_harness}' -Raw) + $stopped = @(Stop-ProcessTreeBounded -RootProcessId $root.Id -TimeoutSec 10 -Role 'test tree') + if ($null -ne (Get-Process -Id $root.Id -ErrorAction SilentlyContinue)) {{ exit 3 }} + if ($null -ne (Get-Process -Id $childProcessId -ErrorAction SilentlyContinue)) {{ exit 4 }} + exit 0 +}} finally {{ + Stop-Process -Id $root.Id -Force -ErrorAction SilentlyContinue +}} +''', + encoding="utf-8-sig", + ) + completed = subprocess.run( + [ + powershell, + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(harness), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + self.assertEqual( + completed.returncode, + 0, + msg=f"stdout={completed.stdout}\nstderr={completed.stderr}", + ) + def test_public_provider_model_and_loopback_environment_are_explicit(self) -> None: for expected in ( '$WhisperModel = "small"', diff --git a/scripts/watch-public-runtime.ps1 b/scripts/watch-public-runtime.ps1 index 5ee9cdb..6563cc7 100644 --- a/scripts/watch-public-runtime.ps1 +++ b/scripts/watch-public-runtime.ps1 @@ -311,7 +311,7 @@ function Test-DatabaseListening { # 재시작 전에 컨테이너를 직접 되살린다. function Restore-DatabaseContainer { try { - $output = & docker.exe start $DbContainer 2>&1 + $output = & docker.exe start $DbContainer 2>$null $output | ForEach-Object { Write-WatchdogLog " docker> $_" } if ($LASTEXITCODE -ne 0) { Write-WatchdogLog "db restore: docker start exit=$LASTEXITCODE" @@ -329,6 +329,17 @@ function Restore-DatabaseContainer { return $false } +function Test-PublicRuntimeHardDown { + param([string[]]$FailedNames) + + $allLocalSurfacesDown = ( + ($FailedNames -contains "api") -and + ($FailedNames -contains "web-preview") -and + ($FailedNames -contains "voice-sidecars") + ) + return ($FailedNames -contains "cloudflared") -or $allLocalSurfacesDown +} + # engine 판정은 /health(프로세스 liveness)가 아니라 /ready(실제 claude -p 생성)로 한다. # /health는 ok:true만 보므로 "프로세스는 살아 있고 그 프로세스의 claude 세션만 죽은" # 상태를 통과시킨다(2026-08-07 공개 런타임: engine=false인데 워치독 lastResult=0). @@ -376,6 +387,14 @@ if (!$SkipPublicHealth) { } $failed = @($checks | Where-Object { -not $_.Ok }) +if ($CheckOnly) { + if ($failed.Count -eq 0) { + Write-Output "healthy: $($checks.Name -join ', ')" + exit 0 + } + Write-Output "unhealthy: $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))" + exit 1 +} if ($failed.Count -eq 0) { Set-FailCount 0 Write-WatchdogLog "healthy: $($checks.Name -join ', ')" @@ -385,17 +404,20 @@ if ($failed.Count -eq 0) { $failCount = (Get-FailCount) + 1 Set-FailCount $failCount Write-WatchdogLog "unhealthy ($failCount/$FailuresBeforeRestart): $((($failed | ForEach-Object { "$($_.Name)=$($_.Detail)" }) -join '; '))" -if ($CheckOnly) { - exit 1 -} # 연속 실패 debounce: claude -p readiness probe 는 콜드 스폰 시 10~20초가 정상이라 # 단발 timeout 을 장애로 오판해 전체 재시작하던 것이 재시작 폭풍의 원인이었다. -# 연속 $FailuresBeforeRestart 회 실패해야 실제 재시작한다. -if ($failCount -lt $FailuresBeforeRestart) { +# 다만 cloudflared가 사라졌거나 API·web·voice sidecar가 함께 죽은 경우는 transient +# readiness가 아니라 공개 서비스 전체 소실이므로 첫 감지에서 즉시 복구한다. +$failedNames = @($failed | ForEach-Object { $_.Name }) +$hardDown = Test-PublicRuntimeHardDown -FailedNames $failedNames +if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown) { Write-WatchdogLog "defer restart: $FailuresBeforeRestart 연속 실패 전까지 대기 (현재 $failCount)" exit 1 } +if ($hardDown -and $failCount -lt $FailuresBeforeRestart) { + Write-WatchdogLog "immediate restart: public runtime hard-down detected" +} # DB가 죽어 있으면 start-public-runtime.ps1으로는 절대 복구되지 않는다(DB 기동은 boot 담당). # 먼저 되살리고, 그래도 안 되면 재시작을 아예 시도하지 않는다 — 고칠 수 없는 대상에 @@ -429,7 +451,7 @@ if ($SkipCloudflaredRestart) { } try { - & $startScript @startArgs 2>&1 | ForEach-Object { + & $startScript @startArgs | ForEach-Object { Write-WatchdogLog "$_" } } catch {