재부팅 복구와 학습자 패널을 정리

scripts/test_public_runtime_watchdog_provenance.py

@@ -98,7 +98,7 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
         )
         gate = BOOT_SOURCE.index("Assert-StableSourceProvenance", gate_comment)
         docker_mutation = BOOT_SOURCE.index("docker.exe update")
-        runtime_recovery = BOOT_SOURCE.index("-File $startScript")
+        runtime_recovery = BOOT_SOURCE.index("& $startScript @startArgs |")
         self.assertLess(gate, docker_mutation)
         self.assertLess(gate, runtime_recovery)
 
@@ -120,6 +120,115 @@ class PublicRuntimeWatchdogProvenanceTest(unittest.TestCase):
         task_registration = BOOT_REGISTER_SOURCE.index("Register-ScheduledTask")
         self.assertLess(dirty_gate, task_registration)
 
+    def test_task_limits_cover_a_full_cold_start_without_overlap(self) -> None:
+        self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", BOOT_REGISTER_SOURCE)
+        self.assertIn("-MultipleInstances IgnoreNew", BOOT_REGISTER_SOURCE)
+        self.assertIn("-ExecutionTimeLimit (New-TimeSpan -Minutes 60)", INSTALLER_SOURCE)
+        self.assertIn("-MultipleInstances IgnoreNew", INSTALLER_SOURCE)
+        self.assertNotIn("-RestartCount", INSTALLER_SOURCE)
+        self.assertNotIn("-RestartInterval", INSTALLER_SOURCE)
+
+        # 최초 health 진단 + DB 복구 + engine/voice/API/web의 각 bounded wait와
+        # process-stop 여유를 모두 합쳐도 작업 스케줄러 60분보다 5분 이상 짧다.
+        initial_health_seconds = 210
+        recovery_seconds = 60 + 90 + (300 * 2) + 180 + 90 + 600 + 30 + (15 * 4)
+        safety_margin_seconds = 300
+        task_limit_seconds = 60 * 60
+        self.assertLess(
+            initial_health_seconds + recovery_seconds + safety_margin_seconds,
+            task_limit_seconds,
+        )
+        boot_prerequisite_seconds = 360 + 90
+        self.assertLess(
+            boot_prerequisite_seconds + recovery_seconds + safety_margin_seconds,
+            task_limit_seconds,
+        )
+
+    def test_hard_down_bypasses_transient_failure_debounce(self) -> None:
+        hard_down = WATCHDOG_SOURCE.index("$hardDown =")
+        debounce = WATCHDOG_SOURCE.index(
+            "if ($failCount -lt $FailuresBeforeRestart -and -not $hardDown)"
+        )
+        restart = WATCHDOG_SOURCE.index("& $startScript @startArgs")
+        self.assertLess(hard_down, debounce)
+        self.assertLess(debounce, restart)
+        self.assertIn('($FailedNames -contains "cloudflared")', WATCHDOG_SOURCE)
+        self.assertIn('($FailedNames -contains "api")', WATCHDOG_SOURCE)
+        self.assertIn('($FailedNames -contains "web-preview")', WATCHDOG_SOURCE)
+        self.assertIn('($FailedNames -contains "voice-sidecars")', WATCHDOG_SOURCE)
+        self.assertIn(
+            "immediate restart: public runtime hard-down detected",
+            WATCHDOG_SOURCE,
+        )
+
+    def test_hard_down_classifier_distinguishes_total_and_transient_failures(self) -> None:
+        powershell = shutil.which("powershell.exe")
+        if powershell is None:
+            self.skipTest("Windows PowerShell 5.1 is not available")
+
+        classifier = WATCHDOG_SOURCE[
+            WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown") :
+            WATCHDOG_SOURCE.index("# engine 판정은", WATCHDOG_SOURCE.index("function Test-PublicRuntimeHardDown"))
+        ].strip()
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            harness = Path(temporary_directory) / "hard-down.ps1"
+            harness.write_text(
+                classifier
+                + r'''
+if (-not (Test-PublicRuntimeHardDown -FailedNames @('cloudflared'))) { exit 2 }
+if (-not (Test-PublicRuntimeHardDown -FailedNames @('api','web-preview','voice-sidecars'))) { exit 3 }
+if (Test-PublicRuntimeHardDown -FailedNames @('engine')) { exit 4 }
+exit 0
+''',
+                encoding="utf-8-sig",
+            )
+            completed = subprocess.run(
+                [
+                    powershell,
+                    "-NoLogo",
+                    "-NoProfile",
+                    "-NonInteractive",
+                    "-ExecutionPolicy",
+                    "Bypass",
+                    "-File",
+                    str(harness),
+                ],
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=30,
+                check=False,
+            )
+            self.assertEqual(
+                completed.returncode,
+                0,
+                msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
+            )
+
+    def test_check_only_exits_before_failcount_log_or_recovery_mutation(self) -> None:
+        failed = WATCHDOG_SOURCE.index("$failed = @(")
+        check_only = WATCHDOG_SOURCE.index("if ($CheckOnly)", failed)
+        inner_health = WATCHDOG_SOURCE.index("if ($failed.Count -eq 0)", check_only)
+        normal_health = WATCHDOG_SOURCE.index(
+            "if ($failed.Count -eq 0)", inner_health + 1
+        )
+        fail_count = WATCHDOG_SOURCE.index("$failCount =", check_only)
+        restart = WATCHDOG_SOURCE.index("& $startScript @startArgs", fail_count)
+        self.assertLess(check_only, fail_count)
+        self.assertLess(check_only, restart)
+        check_only_contract = WATCHDOG_SOURCE[check_only:normal_health]
+        self.assertIn("Write-Output", check_only_contract)
+        self.assertNotIn("Set-FailCount", check_only_contract)
+        self.assertNotIn("Write-WatchdogLog", check_only_contract)
+        self.assertNotIn("Restore-DatabaseContainer", check_only_contract)
+
+    def test_boot_streams_start_script_progress_without_nested_powershell_buffering(self) -> None:
+        self.assertIn("& $startScript @startArgs |", BOOT_SOURCE)
+        self.assertIn('Write-BootLog ("  pub> " + $_)', BOOT_SOURCE)
+        self.assertIn("start-public-runtime.ps1 failed", BOOT_SOURCE)
+        self.assertNotIn("$out = & powershell.exe", BOOT_SOURCE)
+
     def test_watchdog_and_boot_probe_exact_local_voice_stack(self) -> None:
         for source in (WATCHDOG_SOURCE, BOOT_SOURCE):
             for expected in (