워치독 사이드카 장애 복구 수정
This commit is contained in:
parent
190ae70d4d
commit
527027b93d
2 changed files with 66 additions and 2 deletions
|
|
@ -182,6 +182,61 @@ $local = [pscustomobject]@{
|
||||||
if (-not (Test-VoiceApiReady -Health $local)) { throw 'exact local API was rejected' }
|
if (-not (Test-VoiceApiReady -Health $local)) { throw 'exact local API was rejected' }
|
||||||
$local.limits.uvicorn_ws_max_queue = 5
|
$local.limits.uvicorn_ws_max_queue = 5
|
||||||
if (Test-VoiceApiReady -Health $local) { throw 'wrong websocket queue was accepted' }
|
if (Test-VoiceApiReady -Health $local) { throw 'wrong websocket queue was accepted' }
|
||||||
|
''',
|
||||||
|
encoding="utf-8-sig",
|
||||||
|
)
|
||||||
|
completed = subprocess.run(
|
||||||
|
[
|
||||||
|
powershell,
|
||||||
|
"-NoLogo",
|
||||||
|
"-NoProfile",
|
||||||
|
"-NonInteractive",
|
||||||
|
"-ExecutionPolicy",
|
||||||
|
"Bypass",
|
||||||
|
"-File",
|
||||||
|
str(harness),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
timeout=30,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
completed.returncode,
|
||||||
|
0,
|
||||||
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failed_native_sidecar_probe_returns_unhealthy_under_powershell_51(self) -> None:
|
||||||
|
powershell = shutil.which("powershell.exe")
|
||||||
|
python = shutil.which("python.exe")
|
||||||
|
if powershell is None or python is None:
|
||||||
|
self.skipTest("Windows PowerShell 5.1 and Python are required")
|
||||||
|
|
||||||
|
watchdog_contract = WATCHDOG_SOURCE[
|
||||||
|
WATCHDOG_SOURCE.index("function Test-VoiceSidecarStack") :
|
||||||
|
WATCHDOG_SOURCE.index("function Test-CloudflaredProcess")
|
||||||
|
].strip()
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
harness = Path(temporary_directory) / "voice-sidecar-failure-contract.ps1"
|
||||||
|
python_escaped = python.replace("'", "''")
|
||||||
|
probe_escaped = str(VOICE_PROBE).replace("'", "''")
|
||||||
|
harness.write_text(
|
||||||
|
"$ErrorActionPreference = 'Stop'\n"
|
||||||
|
+ f"$Python = '{python_escaped}'\n"
|
||||||
|
+ f"$voiceSidecarProbe = '{probe_escaped}'\n"
|
||||||
|
+ "$WhisperPort = 1\n"
|
||||||
|
+ "$MeloTtsPort = 2\n"
|
||||||
|
+ watchdog_contract
|
||||||
|
+ r'''
|
||||||
|
$result = Test-VoiceSidecarStack
|
||||||
|
if ($result.Ok) { throw 'unreachable sidecars were accepted' }
|
||||||
|
if ($result.Detail -ne 'exact readiness probe failed') {
|
||||||
|
throw "unexpected detail: $($result.Detail)"
|
||||||
|
}
|
||||||
''',
|
''',
|
||||||
encoding="utf-8-sig",
|
encoding="utf-8-sig",
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -242,8 +242,17 @@ function Test-VoiceSidecarStack {
|
||||||
"--tts-language", "KR",
|
"--tts-language", "KR",
|
||||||
"--timeout-seconds", "5"
|
"--timeout-seconds", "5"
|
||||||
)
|
)
|
||||||
$output = @(& $Python @probeArgs 2>$null)
|
# Windows PowerShell 5.1은 native stderr를 ErrorRecord로 승격한다. 스크립트
|
||||||
$probeExit = $LASTEXITCODE
|
# 전역의 Stop 정책을 그대로 두면 sidecar 장애를 복구 신호로 반환하기 전에
|
||||||
|
# NativeCommandError로 종료하므로, 이 단일 probe 경계에서만 Continue로 낮춘다.
|
||||||
|
$previousErrorActionPreference = $ErrorActionPreference
|
||||||
|
try {
|
||||||
|
$ErrorActionPreference = "Continue"
|
||||||
|
$output = @(& $Python @probeArgs 2>$null)
|
||||||
|
$probeExit = $LASTEXITCODE
|
||||||
|
} finally {
|
||||||
|
$ErrorActionPreference = $previousErrorActionPreference
|
||||||
|
}
|
||||||
return [pscustomobject]@{
|
return [pscustomobject]@{
|
||||||
Name = "voice-sidecars"
|
Name = "voice-sidecars"
|
||||||
Ok = $probeExit -eq 0
|
Ok = $probeExit -eq 0
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue