327 lines
12 KiB
Python
327 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
SCRIPTS = Path(__file__).resolve().parent
|
|
CONTRACT = SCRIPTS / "public-runtime-task-maintenance.ps1"
|
|
START = SCRIPTS / "start-public-runtime.ps1"
|
|
|
|
|
|
class PublicRuntimeTaskMaintenanceTest(unittest.TestCase):
|
|
def _run_powershell(self, body: str) -> subprocess.CompletedProcess[str]:
|
|
powershell = shutil.which("powershell.exe")
|
|
if powershell is None:
|
|
self.skipTest("Windows PowerShell 5.1 is not available")
|
|
with tempfile.TemporaryDirectory(prefix="vignette-task-maintenance-") as raw:
|
|
harness = Path(raw) / "task-maintenance.ps1"
|
|
harness.write_text(body, encoding="utf-8-sig")
|
|
return subprocess.run(
|
|
[
|
|
powershell,
|
|
"-NoLogo",
|
|
"-NoProfile",
|
|
"-NonInteractive",
|
|
"-ExecutionPolicy",
|
|
"Bypass",
|
|
"-File",
|
|
str(harness),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
|
|
def test_enter_disables_and_exit_restores_only_previously_enabled_tasks(
|
|
self,
|
|
) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{
|
|
Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }}
|
|
Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }}
|
|
}}
|
|
function Get-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
return $script:states[$TaskName]
|
|
}}
|
|
function Disable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $false
|
|
}}
|
|
function Enable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $true
|
|
}}
|
|
. '{contract}'
|
|
$snapshot = @(Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 1)
|
|
if ($script:states.Watchdog.Settings.Enabled) {{ exit 2 }}
|
|
if ($script:states.Boot.Settings.Enabled) {{ exit 3 }}
|
|
Exit-PublicRuntimeTaskMaintenance -Snapshot $snapshot
|
|
if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 4 }}
|
|
if ($script:states.Boot.Settings.Enabled) {{ exit 5 }}
|
|
exit 0
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_failed_reenable_returns_every_owned_task_to_disabled_idle(self) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{
|
|
Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }}
|
|
Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }}
|
|
}}
|
|
function Get-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
return $script:states[$TaskName]
|
|
}}
|
|
function Disable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $false
|
|
}}
|
|
function Enable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
if ($TaskName -eq 'Boot') {{ throw 'synthetic enable failure' }}
|
|
$script:states[$TaskName].Settings.Enabled = $true
|
|
}}
|
|
. '{contract}'
|
|
$snapshot = @(
|
|
[pscustomobject]@{{ task_name = 'Watchdog'; exists = $true; was_enabled = $true }},
|
|
[pscustomobject]@{{ task_name = 'Boot'; exists = $true; was_enabled = $true }}
|
|
)
|
|
try {{
|
|
Exit-PublicRuntimeTaskMaintenance -Snapshot $snapshot
|
|
exit 2
|
|
}} catch {{
|
|
if ($script:states.Watchdog.Settings.Enabled) {{ exit 3 }}
|
|
if ($script:states.Boot.Settings.Enabled) {{ exit 4 }}
|
|
exit 0
|
|
}}
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_state_probe_reports_actual_restored_and_disabled_truth(self) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{
|
|
Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }}
|
|
Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $false }}; State = 'Ready' }}
|
|
}}
|
|
function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }}
|
|
. '{contract}'
|
|
$snapshot = @(
|
|
[pscustomobject]@{{ task_name = 'Watchdog'; exists = $true; was_enabled = $true }},
|
|
[pscustomobject]@{{ task_name = 'Boot'; exists = $true; was_enabled = $false }}
|
|
)
|
|
$restored = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot
|
|
if (-not $restored.verified) {{ exit 2 }}
|
|
if (-not $restored.restored_to_snapshot) {{ exit 3 }}
|
|
if ($restored.all_disabled_and_idle) {{ exit 4 }}
|
|
$script:states.Watchdog.Settings.Enabled = $false
|
|
$disabled = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot
|
|
if (-not $disabled.verified) {{ exit 5 }}
|
|
if (-not $disabled.all_disabled_and_idle) {{ exit 6 }}
|
|
if ($disabled.restored_to_snapshot) {{ exit 7 }}
|
|
exit 0
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_second_disable_failure_restores_full_preflight_snapshot(self) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{
|
|
Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }}
|
|
Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }}
|
|
}}
|
|
function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }}
|
|
function Disable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $false
|
|
if ($TaskName -eq 'Watchdog') {{ throw 'synthetic second disable failure' }}
|
|
}}
|
|
function Enable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $true
|
|
}}
|
|
. '{contract}'
|
|
try {{
|
|
Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 1 | Out-Null
|
|
exit 2
|
|
}} catch {{
|
|
if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 3 }}
|
|
if (-not $script:states.Boot.Settings.Enabled) {{ exit 4 }}
|
|
exit 0
|
|
}}
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_idle_timeout_restores_full_preflight_snapshot(self) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{
|
|
Watchdog = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Running' }}
|
|
Boot = [pscustomobject]@{{ Settings = [pscustomobject]@{{ Enabled = $true }}; State = 'Ready' }}
|
|
}}
|
|
function Get-ScheduledTask {{ param($TaskName, $ErrorAction); return $script:states[$TaskName] }}
|
|
function Disable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $false
|
|
}}
|
|
function Enable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:states[$TaskName].Settings.Enabled = $true
|
|
}}
|
|
. '{contract}'
|
|
try {{
|
|
Enter-PublicRuntimeTaskMaintenance -TaskNames @('Watchdog', 'Boot') -TimeoutSec 0 | Out-Null
|
|
exit 2
|
|
}} catch {{
|
|
if (-not $script:states.Watchdog.Settings.Enabled) {{ exit 3 }}
|
|
if (-not $script:states.Boot.Settings.Enabled) {{ exit 4 }}
|
|
exit 0
|
|
}}
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_task_created_after_absent_snapshot_fails_truth_without_mutating_it(
|
|
self,
|
|
) -> None:
|
|
contract = str(CONTRACT).replace("'", "''")
|
|
completed = self._run_powershell(
|
|
f"""
|
|
$script:states = @{{}}
|
|
$script:disableCalls = @()
|
|
function Get-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
if ($script:states.ContainsKey($TaskName)) {{ return $script:states[$TaskName] }}
|
|
return $null
|
|
}}
|
|
function Disable-ScheduledTask {{
|
|
param($TaskName, $ErrorAction)
|
|
$script:disableCalls += $TaskName
|
|
$script:states[$TaskName].Settings.Enabled = $false
|
|
}}
|
|
function Enable-ScheduledTask {{ param($TaskName, $ErrorAction) }}
|
|
. '{contract}'
|
|
$snapshot = @(Enter-PublicRuntimeTaskMaintenance -TaskNames @('Boot') -TimeoutSec 1)
|
|
$script:states.Boot = [pscustomobject]@{{
|
|
Settings = [pscustomobject]@{{ Enabled = $true }}
|
|
State = 'Ready'
|
|
}}
|
|
$truth = Get-PublicRuntimeTaskMaintenanceState -Snapshot $snapshot
|
|
if (-not $truth.verified) {{ exit 2 }}
|
|
if ($truth.all_disabled_and_idle) {{ exit 3 }}
|
|
if ($truth.restored_to_snapshot) {{ exit 4 }}
|
|
try {{
|
|
Assert-PublicRuntimeTasksDisabledAndIdle -Snapshot $snapshot -TimeoutSec 1
|
|
exit 5
|
|
}} catch {{
|
|
if ($script:disableCalls -contains 'Boot') {{ exit 6 }}
|
|
exit 0
|
|
}}
|
|
"""
|
|
)
|
|
self.assertEqual(
|
|
0,
|
|
completed.returncode,
|
|
msg=f"stdout={completed.stdout}\nstderr={completed.stderr}",
|
|
)
|
|
|
|
def test_start_reenables_tasks_only_after_unfrozen_passed_receipt(self) -> None:
|
|
source = START.read_text(encoding="utf-8")
|
|
maintenance = source.index("Enter-PublicRuntimeTaskMaintenance `")
|
|
mutation = source.index('$freshFailureStage = "api_cutover"')
|
|
no_rollback = source.index("$freshNoRollback = $true")
|
|
release = source.index("Exit-PublicUploadWriteFreeze `", no_rollback)
|
|
receipt_stage = source.index('$freshFailureStage = "receipt_publish"')
|
|
receipt_write = source.index("Write-Utf8TextAtomically `", receipt_stage)
|
|
install_tasks = source.index('$freshFailureStage = "task_definition_cutover"', receipt_write)
|
|
verify_disabled = source.index(
|
|
"Assert-NewPublicRuntimeTaskDefinitionsPinned `",
|
|
install_tasks,
|
|
)
|
|
restore_tasks = source.index(
|
|
"Enable-NewPublicRuntimeTaskDefinitions `",
|
|
verify_disabled,
|
|
)
|
|
actual_task_probe = source.index(
|
|
"Get-PublicRuntimeTaskMaintenanceState `",
|
|
restore_tasks,
|
|
)
|
|
task_entered_clear = source.index(
|
|
"$freshTaskMaintenanceEntered = $false",
|
|
actual_task_probe,
|
|
)
|
|
task_receipt_stage = source.index(
|
|
'$freshFailureStage = "task_recovery_receipt_publish"',
|
|
restore_tasks,
|
|
)
|
|
task_receipt_write = source.index(
|
|
"Write-Utf8TextAtomically `",
|
|
task_receipt_stage,
|
|
)
|
|
fully_committed = source.index(
|
|
"$freshPromotionCommitted = $true",
|
|
task_receipt_write,
|
|
)
|
|
self.assertLess(maintenance, mutation)
|
|
self.assertLess(no_rollback, release)
|
|
self.assertLess(release, receipt_stage)
|
|
self.assertLess(receipt_stage, receipt_write)
|
|
self.assertLess(receipt_write, install_tasks)
|
|
self.assertLess(install_tasks, verify_disabled)
|
|
self.assertLess(verify_disabled, restore_tasks)
|
|
self.assertLess(restore_tasks, actual_task_probe)
|
|
self.assertLess(actual_task_probe, task_receipt_stage)
|
|
self.assertLess(restore_tasks, task_receipt_stage)
|
|
self.assertLess(task_receipt_stage, task_receipt_write)
|
|
self.assertLess(task_receipt_write, task_entered_clear)
|
|
self.assertLess(task_entered_clear, fully_committed)
|
|
self.assertIn('scope = "runtime_storage_commit_only"', source)
|
|
self.assertIn('operational_success = $false', source)
|
|
self.assertIn('schema_version = "vignette.public-runtime-task-recovery.v1"', source)
|
|
self.assertIn("$freshTaskMaintenanceWasEntered", source)
|
|
self.assertIn("tasks_restored_to_snapshot = $TasksRestored", source)
|
|
self.assertIn("task_state_verified = $TaskStateVerified", source)
|
|
self.assertIn("Suspend-PublicRuntimeTasks `", source)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|