feat(workflow): checkpoint GSD before five-hour quota exhaustion
This commit is contained in:
1 parent
73954e675d
commit
c9b9f2eba9
9 files changed
+758
-1
No files matched your search
@@ -0,0 +1,236 @@
|
||||
"""Quota policy and recovery tests; no model calls or quota consumption."""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from tools.workflow import codex_quota_guard as guard
|
||||
|
||||
|
||||
def limits(used=79, duration=300, reset=2000):
|
||||
return {'rateLimits': {'primary': {'usedPercent': used, 'windowDurationMins': duration,
|
||||
'resetsAt': reset}}}
|
||||
|
||||
|
||||
class QuotaTests(unittest.TestCase):
|
||||
def test_boundary_thresholds(self):
|
||||
for used, expected in [(0, 'ready'), (79, 'ready'), (80, 'drain'), (84, 'drain'),
|
||||
(85, 'pause'), (100, 'pause')]:
|
||||
with self.subTest(used=used):
|
||||
self.assertEqual(guard.stage(guard.quota(limits(used), 1000), 1000), expected)
|
||||
|
||||
def test_actual_window_secondary(self):
|
||||
data = limits(99, 10080)
|
||||
data['rateLimits']['secondary'] = limits(40)['rateLimits']['primary']
|
||||
self.assertEqual(guard.quota(data, 1000)['remaining'], 60)
|
||||
|
||||
def test_general_bucket_preferred(self):
|
||||
data = limits(99)
|
||||
data['rateLimitsByLimitId'] = {'codex': limits(10)['rateLimits'],
|
||||
'specialty': limits(100)['rateLimits']}
|
||||
self.assertEqual(guard.quota(data, 1000)['remaining'], 90)
|
||||
|
||||
def test_invalid_windows_never_look_healthy(self):
|
||||
for data in [limits(50, 10080), limits(50, reset=999), limits(float('nan')),
|
||||
limits(-1), limits(101), limits(True), {}, {'rateLimits': None}]:
|
||||
with self.subTest(data=data), self.assertRaises(ValueError):
|
||||
guard.quota(data, 1000)
|
||||
|
||||
def test_stale_and_expired_unknown(self):
|
||||
sample = guard.quota(limits(), 1000)
|
||||
self.assertEqual(guard.stage(sample, 1181), 'unknown')
|
||||
self.assertEqual(guard.stage(dict(sample, checkedAt=1999), 2000), 'unknown')
|
||||
self.assertEqual(guard.stage(None, 1000), 'unknown')
|
||||
|
||||
def test_reset_clears_pause_only_after_fresh_quota(self):
|
||||
sample = guard.quota(limits(99), 1000)
|
||||
self.assertEqual(guard.stage(sample, 2000), 'unknown')
|
||||
fresh = guard.quota(limits(1, reset=20000), 2001)
|
||||
self.assertEqual(guard.stage(fresh, 2001), 'ready')
|
||||
|
||||
def test_worker_denials_leave_checkpoint_tools_available(self):
|
||||
for used in (80, 85, 100):
|
||||
sample = guard.quota(limits(used), 1000)
|
||||
for tool in ('Agent', 'spawn_agent', 'followup_task', 'mcp__local__spawn_agent',
|
||||
'collaboration.spawn_agent'):
|
||||
answer = guard.hook_response('PreToolUse', tool, sample, 1000)
|
||||
self.assertEqual(answer['hookSpecificOutput']['permissionDecision'], 'deny')
|
||||
for tool in ('Bash', 'apply_patch', 'send_message', 'interrupt_agent'):
|
||||
answer = guard.hook_response('PreToolUse', tool, sample, 1000)
|
||||
self.assertNotIn('permissionDecision', answer.get('hookSpecificOutput', {}))
|
||||
|
||||
def test_unknown_denies_new_workers(self):
|
||||
answer = guard.hook_response('PreToolUse', 'spawn_agent', None, 1000)
|
||||
self.assertEqual(answer['hookSpecificOutput']['permissionDecision'], 'deny')
|
||||
|
||||
def test_ready_allows_dispatch(self):
|
||||
self.assertEqual(guard.hook_response('PreToolUse', 'spawn_agent',
|
||||
guard.quota(limits(), 1000), 1000), {})
|
||||
|
||||
def test_diagnostic_event_does_not_return_context(self):
|
||||
self.assertEqual(guard.hook_response('PreModelRequest', '',
|
||||
guard.quota(limits(90), 1000), 1000), {})
|
||||
|
||||
|
||||
class StateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.state = Path(self.temp.name) / 'state'
|
||||
self.patcher = patch.object(guard, 'STATE', self.state)
|
||||
self.patcher.start()
|
||||
self.addCleanup(self.patcher.stop)
|
||||
|
||||
def test_atomic_is_private_and_replaces(self):
|
||||
path = self.state / 'value.json'
|
||||
guard.atomic(path, {'one': 1})
|
||||
guard.atomic(path, {'two': 2})
|
||||
self.assertEqual(guard.read(path), {'two': 2})
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
|
||||
|
||||
def test_thread_registration_uses_agent_id(self):
|
||||
guard.register({'session_id': 'root', 'agent_id': 'child', 'turn_id': 'turn'})
|
||||
entries = list((self.state / 'threads').glob('*.json'))
|
||||
self.assertEqual(guard.read(entries[0])['threadId'], 'child')
|
||||
|
||||
def test_missing_turn_never_guessed(self):
|
||||
guard.register({'session_id': 'root'})
|
||||
self.assertFalse((self.state / 'threads').exists())
|
||||
|
||||
def test_context_only_once_per_turn(self):
|
||||
guard.atomic(self.state / 'quota.json', guard.quota(limits(90), 1000))
|
||||
payload = {'hook_event_name': 'PostToolUse', 'turn_id': 'turn'}
|
||||
with patch.object(guard.time, 'time', return_value=1000):
|
||||
self.assertIn('hookSpecificOutput', guard.handle_hook(payload))
|
||||
self.assertEqual(guard.handle_hook(payload), {})
|
||||
payload['turn_id'] = 'next'
|
||||
self.assertIn('hookSpecificOutput', guard.handle_hook(payload))
|
||||
|
||||
def test_only_unambiguous_events_register_thread(self):
|
||||
with patch.object(guard, 'register') as register:
|
||||
guard.handle_hook({'hook_event_name': 'PreToolUse', 'session_id': 'root', 'turn_id': 'child'})
|
||||
register.assert_not_called()
|
||||
guard.handle_hook({'hook_event_name': 'PreModelRequest', 'session_id': 'root', 'turn_id': 'root-turn'})
|
||||
register.assert_called_once()
|
||||
|
||||
def test_pause_saves_snapshot_and_steers_once(self):
|
||||
class FakeRPC:
|
||||
calls = []
|
||||
def call(self, method, params):
|
||||
self.calls.append((method, params))
|
||||
return limits(90)
|
||||
def close(self):
|
||||
pass
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', FakeRPC), patch.object(guard, 'snapshot') as snapshot, \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(snapshot.call_count, 2)
|
||||
steers = [p for m, p in FakeRPC.calls if m == 'turn/steer']
|
||||
self.assertEqual(len(steers), 1)
|
||||
self.assertEqual(steers[0]['expectedTurnId'], 'turn')
|
||||
self.assertEqual(steers[0]['threadId'], 'root')
|
||||
self.assertFalse(any(m == 'turn/start' for m, _ in FakeRPC.calls))
|
||||
|
||||
def test_inactive_turn_is_not_restarted(self):
|
||||
class FakeRPC:
|
||||
calls = []
|
||||
def call(self, method, params):
|
||||
self.calls.append(method)
|
||||
if method == 'turn/steer':
|
||||
raise RuntimeError('inactive')
|
||||
return limits(90)
|
||||
def close(self):
|
||||
pass
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', FakeRPC), patch.object(guard, 'snapshot'), \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(FakeRPC.calls.count('turn/steer'), 1)
|
||||
self.assertNotIn('turn/start', FakeRPC.calls)
|
||||
|
||||
def test_reset_timestamp_jitter_does_not_repeat_pause(self):
|
||||
class FakeRPC:
|
||||
steers = 0
|
||||
reads = 0
|
||||
def call(self, method, params):
|
||||
if method == 'turn/steer':
|
||||
self.steers += 1
|
||||
return {}
|
||||
self.reads += 1
|
||||
return limits(90, reset=2000 + self.reads)
|
||||
def close(self):
|
||||
pass
|
||||
rpc = FakeRPC()
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', return_value=rpc), patch.object(guard, 'snapshot'), \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(rpc.steers, 1)
|
||||
|
||||
def test_recovery_preserves_canonical_handoff(self):
|
||||
project = Path(self.temp.name) / 'project'
|
||||
(project / '.planning').mkdir(parents=True)
|
||||
handoff = project / '.planning/HANDOFF.json'
|
||||
handoff.write_text('{"precise": "continuation"}')
|
||||
process = unittest.mock.MagicMock()
|
||||
process.__enter__.return_value = process
|
||||
process.stdout.read.return_value = b'test git output'
|
||||
with patch.object(guard, 'ROOT', project), patch.object(guard.subprocess, 'Popen', return_value=process):
|
||||
guard.snapshot(guard.quota(limits(90), 1000))
|
||||
guard.snapshot(guard.quota(limits(90, reset=2001), 1000))
|
||||
self.assertEqual(handoff.read_text(), '{"precise": "continuation"}')
|
||||
self.assertFalse((self.state / 'recovery-2001').exists())
|
||||
record = guard.read(self.state / 'recovery-2000/record.json')
|
||||
self.assertIn('continuation', record['gsd']['.planning/HANDOFF.json'])
|
||||
|
||||
|
||||
class FrameTests(unittest.TestCase):
|
||||
def rpc(self, data=b''):
|
||||
rpc = guard.RPC.__new__(guard.RPC)
|
||||
rpc.sock = unittest.mock.Mock()
|
||||
rpc.buffer = data
|
||||
rpc.sequence = 0
|
||||
return rpc
|
||||
|
||||
def test_eof_terminates_instead_of_spinning(self):
|
||||
rpc = self.rpc()
|
||||
rpc.sock.recv.return_value = b''
|
||||
with self.assertRaises(EOFError):
|
||||
rpc.take(1)
|
||||
|
||||
def test_oversize_rejected_before_payload_read(self):
|
||||
rpc = self.rpc(b'\x81\x7f' + struct.pack('!Q', 2**21))
|
||||
with self.assertRaises(ValueError):
|
||||
rpc.message()
|
||||
rpc.sock.recv.assert_not_called()
|
||||
|
||||
def test_ping_and_fragmented_text(self):
|
||||
rpc = self.rpc(b'\x89\x01x' + b'\x01\x05{"a":' + b'\x80\x021}')
|
||||
self.assertEqual(rpc.message(), {'a': 1})
|
||||
rpc.sock.sendall.assert_called_once()
|
||||
|
||||
def test_client_masks_outgoing_frames(self):
|
||||
rpc = self.rpc()
|
||||
rpc.send({'test': 'value'})
|
||||
wire = rpc.sock.sendall.call_args.args[0]
|
||||
self.assertTrue(wire[1] & 128)
|
||||
mask = wire[2:6]
|
||||
raw = bytes(c ^ mask[i % 4] for i, c in enumerate(wire[6:]))
|
||||
self.assertEqual(json.loads(raw), {'test': 'value'})
|
||||
|
||||
def test_notifications_skipped_until_matching_response(self):
|
||||
rpc = self.rpc()
|
||||
with patch.object(rpc, 'message', side_effect=[{'method': 'notify'}, {'id': 1, 'result': {'ok': True}}]):
|
||||
self.assertEqual(rpc.call('test', {}), {'ok': True})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user