feat(workflow): checkpoint GSD before five-hour quota exhaustion
This commit is contained in:
1 parent
73954e675d
commit
c9b9f2eba9
9 files changed
+758
-1
No files matched your search
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"description": "GamePort quota guard: stop new workers at 20%; checkpoint and pause at 15%.",
|
||||
"hooks": {
|
||||
"SessionStart": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/usr/bin/python3 \"$(git rev-parse --show-toplevel)/tools/workflow/codex_quota_guard.py\" hook",
|
||||
"timeout": 5,
|
||||
"commandWindows": "cmd /c exit 0"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PreModelRequest": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/usr/bin/python3 \"$(git rev-parse --show-toplevel)/tools/workflow/codex_quota_guard.py\" hook",
|
||||
"timeout": 5,
|
||||
"commandWindows": "cmd /c exit 0"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"UserPromptSubmit": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/usr/bin/python3 \"$(git rev-parse --show-toplevel)/tools/workflow/codex_quota_guard.py\" hook",
|
||||
"timeout": 5,
|
||||
"commandWindows": "cmd /c exit 0"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PreToolUse": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/usr/bin/python3 \"$(git rev-parse --show-toplevel)/tools/workflow/codex_quota_guard.py\" hook",
|
||||
"timeout": 5,
|
||||
"commandWindows": "cmd /c exit 0"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PostToolUse": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/usr/bin/python3 \"$(git rev-parse --show-toplevel)/tools/workflow/codex_quota_guard.py\" hook",
|
||||
"timeout": 5,
|
||||
"commandWindows": "cmd /c exit 0"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -37,5 +37,5 @@
|
||||
"?? tools.yaml"
|
||||
],
|
||||
"next_action": "Afterquotareset reconcile01-05commits/diff;finish task2 withoutrepeatingtask1",
|
||||
"context_notes": "Quota interruption after server-restart recovery. Plans01-01..04 complete, primary-reviewed and pushed through de68796. Currentplan01-05 task1 RED446930d/GREEN6112339; task2 RED818191c. No01-05SUMMARY. Preserve partial uncommitted edits to scripts/collect-fidelity-windows.ps1 and tools/recon/fidelity_checkpoint.py; preserve unrelated tools.yaml. Do not restart task1 or treatREDcommit as completion. Inspect actual gitdiff before continuation.\nExisting generated packages artifacts/fidelity/01-05-packages/{linux-x64,win-x64} and tracerbundle artifacts/fidelity/01-05-tracer-checkpoint exist; inspect manifests/logs rather than rebuilding blindly. Task2 REDreproduced23tests with missing-session-materials assertion/statusAPI error. Cached local PowerShell executable .tools/powershell-check/root/opt/microsoft/powershell/7/pwsh and cacheddeb found; no successful execution/provenance was reported before quota. No third-party installers redistributed. Need finish task2Windowsagenda/selftests/complete truthfulresults, verify, writeSUMMARY andhandlerstate; then primaryreview/push. All120actualoriginal-DxWnd/nativeLinux/nativeWindowsacceptanceOPEN. Plans01-06Linuxcapture/01-07combinedhands-onreadiness+acceptance remain.\nSequential nativeexecutors because worktreebasecheckcannotresolveorigin/HEAD (repoGitea/main). Maincommits authorized;git.allow_default_branch_commits=true. Local sockettests/Node runner needescalation to actuallyexecute namedtests. Fullwave2scripts/test.sh authorizedpasses462Python+2HUD/allmanaged/36GPUzeroerrorsGTX1070/539hashes; observer23managed+7process/privateXvfb4onoffidentical. No newWindowsororiginalcomparison.\nUser asked howtoresumeandautomatelow5husagepause. Researched officialCodexaccount/rateLimits/read plusaccount/rateLimits/updated andPreToolUse/PostToolUsehooks; no guardconfigured. OnlySessionStarthook currentlyconfigured;statuslinealreadyfive-hour-limit. Do not claimautopauseinstalled. Need customwatcher/hook before it canbeautomatic. No activeworkersknown;recover_01_05 returnedquotaerror, do notimmediatelyredispatchwhilequotaexhausted. Resumewhenuserrequestsafterlimitreset; inspectactualremainingquota ratherthanstale10:28PMreset. No sourcefeatureworkwasdonebyprimaryforthisquestion.\n"
|
||||
"context_notes": "Quota interruption after server-restart recovery. Plans01-01..04 complete, primary-reviewed and pushed through de68796. Currentplan01-05 task1 RED446930d/GREEN6112339; task2 RED818191c. No01-05SUMMARY. Preserve partial uncommitted edits to scripts/collect-fidelity-windows.ps1 and tools/recon/fidelity_checkpoint.py; preserve unrelated tools.yaml. Do not restart task1 or treatREDcommit as completion. Inspect actual gitdiff before continuation.\nExisting generated packages artifacts/fidelity/01-05-packages/{linux-x64,win-x64} and tracerbundle artifacts/fidelity/01-05-tracer-checkpoint exist; inspect manifests/logs rather than rebuilding blindly. Task2 REDreproduced23tests with missing-session-materials assertion/statusAPI error. Cached local PowerShell executable .tools/powershell-check/root/opt/microsoft/powershell/7/pwsh and cacheddeb found; no successful execution/provenance was reported before quota. No third-party installers redistributed. Need finish task2Windowsagenda/selftests/complete truthfulresults, verify, writeSUMMARY andhandlerstate; then primaryreview/push. All120actualoriginal-DxWnd/nativeLinux/nativeWindowsacceptanceOPEN. Plans01-06Linuxcapture/01-07combinedhands-onreadiness+acceptance remain.\nSequential nativeexecutors because worktreebasecheckcannotresolveorigin/HEAD (repoGitea/main). Maincommits authorized;git.allow_default_branch_commits=true. Local sockettests/Node runner needescalation to actuallyexecute namedtests. Fullwave2scripts/test.sh authorizedpasses462Python+2HUD/allmanaged/36GPUzeroerrorsGTX1070/539hashes; observer23managed+7process/privateXvfb4onoffidentical. No newWindowsororiginalcomparison.\nUser asked howtoresumeandautomatelow5husagepause. Researched officialCodexaccount/rateLimits/read plusaccount/rateLimits/updated andPreToolUse/PostToolUsehooks; no guardconfigured. OnlySessionStarthook currentlyconfigured;statuslinealreadyfive-hour-limit. Do not claimautopauseinstalled. Need customwatcher/hook before it canbeautomatic. No activeworkersknown;recover_01_05 returnedquotaerror, do notimmediatelyredispatchwhilequotaexhausted. Resumewhenuserrequestsafterlimitreset; inspectactualremainingquota ratherthanstale10:28PMreset. No sourcefeatureworkwasdonebyprimaryforthisquestion.\n\n2026-11-13 quota-guard setup: tools/workflow/codex_quota_guard.py and .codex/hooks.json implement local Linux 60-second actual 300-minute quota polling; <=20% denies new worker dispatch, <=15% independent recovery snapshot plus cooperative GSD pause request. The watcher is running and live quota read works. Five project hooks are discovered but require user /hooks trust before registration/tool blocking activate. See docs/codex-quota-guard.md. No feature tasks resumed or partial collector edits committed during this setup. After reset/user resume, continue plan 01-05 task 2 from this original handoff.\n"
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
# Codex quota guard
|
||||
|
||||
This repository has a local Linux watcher and project hooks for the user's
|
||||
requested GSD quota policy. The watcher polls the existing Codex app-server every
|
||||
60 seconds; it does not start model turns or read credentials. It uses the
|
||||
300-minute account window and its actual reset timestamp, preferring the general
|
||||
`codex` bucket over specialty-model buckets.
|
||||
|
||||
- At **20% remaining or less**, `PreToolUse` denies new `Agent`, `spawn_agent`,
|
||||
and `followup_task` calls. Existing workers may finish their safe checkpoint.
|
||||
- At **15% remaining or less**, the watcher saves a separate recovery snapshot
|
||||
and sends each registered active turn one request to checkpoint with
|
||||
`$gsd-pause-work` and end. Tool hooks also supply that instruction once per turn.
|
||||
- Missing, expired, or older-than-180-second quota data blocks new worker dispatch
|
||||
until a fresh reading arrives. Checkpoint tools remain available.
|
||||
- The watcher never starts a turn, interrupts a process, commits unfinished work,
|
||||
pushes code, or automatically resumes after reset.
|
||||
|
||||
## One-time activation
|
||||
|
||||
The source is `tools/workflow/codex_quota_guard.py`; project configuration is
|
||||
`.codex/hooks.json`. Existing global GSD hooks are preserved and additive.
|
||||
Open **`/hooks`** in Codex, review and trust the five hooks from this project's
|
||||
`.codex/hooks.json`. New definitions remain skipped until reviewed; see the
|
||||
[official hook trust instructions](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks).
|
||||
The watcher has been started for this host, but thread registration and tool
|
||||
blocking require trusted hooks. A later session's `SessionStart` hook starts
|
||||
another watcher only if one is not already running.
|
||||
|
||||
The guard targets this Linux Codex host, not the Windows original-game test
|
||||
machine. Windows uses a no-op command override. It does not restart Codex or
|
||||
change provider/model/account settings.
|
||||
|
||||
## Commands
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
python3 tools/workflow/codex_quota_guard.py status
|
||||
python3 tools/workflow/codex_quota_guard.py refresh
|
||||
python3 tools/workflow/codex_quota_guard.py start
|
||||
python3 tools/workflow/codex_quota_guard.py stop
|
||||
```
|
||||
|
||||
`status` includes the lock-verified `running` flag, current reading, reset epoch,
|
||||
and any last poll error type. `refresh` uses the existing local daemon socket and
|
||||
may need execution outside the agent sandbox. It neither starts a daemon nor a
|
||||
model turn. `stop` requests termination within a second after any in-flight RPC
|
||||
finishes; hooks start the watcher again, so disable the project hooks in `/hooks`
|
||||
when turning the guard off permanently. `start` clears the stop marker when its
|
||||
singleton watcher starts. No service or cron entry is installed.
|
||||
|
||||
After quota resets, send **`$gsd-resume-work`**. The current saved continuation is
|
||||
phase 01, plan 01-05, task 2; preserve its partial edits and do not repeat task 1.
|
||||
Resumption remains a user action even after the account has recovered.
|
||||
|
||||
## Recovery and limits
|
||||
|
||||
Ignored private state is under `artifacts/gsd-quota/`. `quota.json` stores only
|
||||
counters/timestamps. Per-thread registration stores thread/turn identifiers, not
|
||||
conversation contents. At low quota, `recovery-<reset>/record.json` saves the Git
|
||||
HEAD, status, tracked patch (capped at 2 MiB), and existing GSD state/handoff text
|
||||
(capped at 256 KiB each). It supplements and never replaces the canonical GSD
|
||||
handoff. Untracked file contents remain in the working tree; this is not a full
|
||||
backup. Source changes after that one-per-window snapshot rely on the working
|
||||
tree and the model's final handoff. Files are private to the local account.
|
||||
|
||||
The pause is **cooperative**. Polling may lag usage by a minute, a long active
|
||||
model request may consume the reserve, and no model can guarantee writing a
|
||||
handoff after the provider exhausts quota. The independent recovery record still
|
||||
runs without model tokens. Tool hooks cover supported local tool paths; launching
|
||||
workers through unrelated shell scripts or other clients bypasses this specific
|
||||
dispatch policy. Weekly and specialty-model limits are not guarded. Inactive or
|
||||
mismatched turns are never restarted by the watcher.
|
||||
|
||||
Verified against local Codex 0.169.0: real Unix WebSocket quota read, project hook
|
||||
discovery (five hooks awaiting trust), singleton background watcher, and 24 targeted
|
||||
unit tests. Simulated low-quota tests verify pause steering and snapshot behavior;
|
||||
no real quota depletion or active gameplay work was induced to test the threshold.
|
||||
See the [official app-server protocol](https://learn.chatgpt.com/docs/app-server)
|
||||
and [hook contracts](https://learn.chatgpt.com/docs/hooks).
|
||||
|
||||
Source-control note: guard changes are committed locally. Push is deferred until
|
||||
the earlier unfinished plan01-05 task2 RED checkpoint is green; pushing this
|
||||
branch now would also publish that unverified ancestor.
|
||||
@@ -1,5 +1,13 @@
|
||||
# Compatibility and validation
|
||||
|
||||
## Workflow quota guard — 2026-11-13
|
||||
|
||||
Local Codex0.169.0 Linux Unix-WebSocket quota read and watcher verified; five
|
||||
project hooks discovered, awaiting user trust. Targeted simulated quota, policy,
|
||||
transport and recovery tests pass. Real exhaustion/active-turn pause is not
|
||||
induced; pause remains cooperative. Windows hook override is a no-op. No game
|
||||
build, package or original/native Windows acceptance claim changes.
|
||||
|
||||
## Phase 1 wave 2 bounded observer evidence
|
||||
|
||||
Full authorized wave2 `scripts/test.sh` exits0:462 Python+2 HUD checks,
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
# Decisions
|
||||
|
||||
## Quota reserve — 2026-11-13
|
||||
|
||||
Use actual300-minute account quota and server reset time. Reserve20% for
|
||||
existing workers and15% for handoff. Poll locally without model turns; preserve
|
||||
a private independent recovery snapshot and request cooperative GSD pause.
|
||||
Never auto-resume, commit partial source, restart Codex, or bypass hook trust.
|
||||
See [guard contract](codex-quota-guard.md).
|
||||
|
||||
## Passive evidence observation — phase 1 wave 2
|
||||
|
||||
Keep one optional application-owned clock across Run continuations. Record raw,
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
# Progress — 2026-11-07
|
||||
|
||||
## Local quota guard — 2026-11-13
|
||||
|
||||
Installed Linux quota watcher and five project hooks; real local account quota
|
||||
read succeeds, watcher runs, and targeted policy/recovery tests pass. Hook trust
|
||||
remains a user `/hooks` action. At20% remaining new workers are denied; at15%
|
||||
remaining recovery is saved and registered active turns receive a cooperative
|
||||
GSD pause request. See [setup and limits](codex-quota-guard.md). Phase01-05 task2
|
||||
partial edits remain untouched; no gameplay or acceptance status changed.
|
||||
|
||||
## Phase 1 wave 2 — review, ledger and passive observation
|
||||
|
||||
Full authorized wave2 `scripts/test.sh` exits0:462 Python+2 HUD checks,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# Tasks
|
||||
|
||||
## Quota-aware GSD continuation — 2026-11-13
|
||||
|
||||
- [x] Install/test local60-second quota watcher,20% dispatch gate,15% pause request.
|
||||
- [ ] User review/trust the five project quota hooks via `/hooks`.
|
||||
- [ ] Resume `$gsd-resume-work` when desired; retain01-05 task2 partial edits.
|
||||
|
||||
## Phase 1 execution — wave 2
|
||||
|
||||
- [x]01-02 paired local review:21 Python/27 Node checks and real Chromium
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Quota policy and recovery tests; no model calls or quota consumption."""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from tools.workflow import codex_quota_guard as guard
|
||||
|
||||
|
||||
def limits(used=79, duration=300, reset=2000):
|
||||
return {'rateLimits': {'primary': {'usedPercent': used, 'windowDurationMins': duration,
|
||||
'resetsAt': reset}}}
|
||||
|
||||
|
||||
class QuotaTests(unittest.TestCase):
|
||||
def test_boundary_thresholds(self):
|
||||
for used, expected in [(0, 'ready'), (79, 'ready'), (80, 'drain'), (84, 'drain'),
|
||||
(85, 'pause'), (100, 'pause')]:
|
||||
with self.subTest(used=used):
|
||||
self.assertEqual(guard.stage(guard.quota(limits(used), 1000), 1000), expected)
|
||||
|
||||
def test_actual_window_secondary(self):
|
||||
data = limits(99, 10080)
|
||||
data['rateLimits']['secondary'] = limits(40)['rateLimits']['primary']
|
||||
self.assertEqual(guard.quota(data, 1000)['remaining'], 60)
|
||||
|
||||
def test_general_bucket_preferred(self):
|
||||
data = limits(99)
|
||||
data['rateLimitsByLimitId'] = {'codex': limits(10)['rateLimits'],
|
||||
'specialty': limits(100)['rateLimits']}
|
||||
self.assertEqual(guard.quota(data, 1000)['remaining'], 90)
|
||||
|
||||
def test_invalid_windows_never_look_healthy(self):
|
||||
for data in [limits(50, 10080), limits(50, reset=999), limits(float('nan')),
|
||||
limits(-1), limits(101), limits(True), {}, {'rateLimits': None}]:
|
||||
with self.subTest(data=data), self.assertRaises(ValueError):
|
||||
guard.quota(data, 1000)
|
||||
|
||||
def test_stale_and_expired_unknown(self):
|
||||
sample = guard.quota(limits(), 1000)
|
||||
self.assertEqual(guard.stage(sample, 1181), 'unknown')
|
||||
self.assertEqual(guard.stage(dict(sample, checkedAt=1999), 2000), 'unknown')
|
||||
self.assertEqual(guard.stage(None, 1000), 'unknown')
|
||||
|
||||
def test_reset_clears_pause_only_after_fresh_quota(self):
|
||||
sample = guard.quota(limits(99), 1000)
|
||||
self.assertEqual(guard.stage(sample, 2000), 'unknown')
|
||||
fresh = guard.quota(limits(1, reset=20000), 2001)
|
||||
self.assertEqual(guard.stage(fresh, 2001), 'ready')
|
||||
|
||||
def test_worker_denials_leave_checkpoint_tools_available(self):
|
||||
for used in (80, 85, 100):
|
||||
sample = guard.quota(limits(used), 1000)
|
||||
for tool in ('Agent', 'spawn_agent', 'followup_task', 'mcp__local__spawn_agent',
|
||||
'collaboration.spawn_agent'):
|
||||
answer = guard.hook_response('PreToolUse', tool, sample, 1000)
|
||||
self.assertEqual(answer['hookSpecificOutput']['permissionDecision'], 'deny')
|
||||
for tool in ('Bash', 'apply_patch', 'send_message', 'interrupt_agent'):
|
||||
answer = guard.hook_response('PreToolUse', tool, sample, 1000)
|
||||
self.assertNotIn('permissionDecision', answer.get('hookSpecificOutput', {}))
|
||||
|
||||
def test_unknown_denies_new_workers(self):
|
||||
answer = guard.hook_response('PreToolUse', 'spawn_agent', None, 1000)
|
||||
self.assertEqual(answer['hookSpecificOutput']['permissionDecision'], 'deny')
|
||||
|
||||
def test_ready_allows_dispatch(self):
|
||||
self.assertEqual(guard.hook_response('PreToolUse', 'spawn_agent',
|
||||
guard.quota(limits(), 1000), 1000), {})
|
||||
|
||||
def test_diagnostic_event_does_not_return_context(self):
|
||||
self.assertEqual(guard.hook_response('PreModelRequest', '',
|
||||
guard.quota(limits(90), 1000), 1000), {})
|
||||
|
||||
|
||||
class StateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.state = Path(self.temp.name) / 'state'
|
||||
self.patcher = patch.object(guard, 'STATE', self.state)
|
||||
self.patcher.start()
|
||||
self.addCleanup(self.patcher.stop)
|
||||
|
||||
def test_atomic_is_private_and_replaces(self):
|
||||
path = self.state / 'value.json'
|
||||
guard.atomic(path, {'one': 1})
|
||||
guard.atomic(path, {'two': 2})
|
||||
self.assertEqual(guard.read(path), {'two': 2})
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
|
||||
|
||||
def test_thread_registration_uses_agent_id(self):
|
||||
guard.register({'session_id': 'root', 'agent_id': 'child', 'turn_id': 'turn'})
|
||||
entries = list((self.state / 'threads').glob('*.json'))
|
||||
self.assertEqual(guard.read(entries[0])['threadId'], 'child')
|
||||
|
||||
def test_missing_turn_never_guessed(self):
|
||||
guard.register({'session_id': 'root'})
|
||||
self.assertFalse((self.state / 'threads').exists())
|
||||
|
||||
def test_context_only_once_per_turn(self):
|
||||
guard.atomic(self.state / 'quota.json', guard.quota(limits(90), 1000))
|
||||
payload = {'hook_event_name': 'PostToolUse', 'turn_id': 'turn'}
|
||||
with patch.object(guard.time, 'time', return_value=1000):
|
||||
self.assertIn('hookSpecificOutput', guard.handle_hook(payload))
|
||||
self.assertEqual(guard.handle_hook(payload), {})
|
||||
payload['turn_id'] = 'next'
|
||||
self.assertIn('hookSpecificOutput', guard.handle_hook(payload))
|
||||
|
||||
def test_only_unambiguous_events_register_thread(self):
|
||||
with patch.object(guard, 'register') as register:
|
||||
guard.handle_hook({'hook_event_name': 'PreToolUse', 'session_id': 'root', 'turn_id': 'child'})
|
||||
register.assert_not_called()
|
||||
guard.handle_hook({'hook_event_name': 'PreModelRequest', 'session_id': 'root', 'turn_id': 'root-turn'})
|
||||
register.assert_called_once()
|
||||
|
||||
def test_pause_saves_snapshot_and_steers_once(self):
|
||||
class FakeRPC:
|
||||
calls = []
|
||||
def call(self, method, params):
|
||||
self.calls.append((method, params))
|
||||
return limits(90)
|
||||
def close(self):
|
||||
pass
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', FakeRPC), patch.object(guard, 'snapshot') as snapshot, \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(snapshot.call_count, 2)
|
||||
steers = [p for m, p in FakeRPC.calls if m == 'turn/steer']
|
||||
self.assertEqual(len(steers), 1)
|
||||
self.assertEqual(steers[0]['expectedTurnId'], 'turn')
|
||||
self.assertEqual(steers[0]['threadId'], 'root')
|
||||
self.assertFalse(any(m == 'turn/start' for m, _ in FakeRPC.calls))
|
||||
|
||||
def test_inactive_turn_is_not_restarted(self):
|
||||
class FakeRPC:
|
||||
calls = []
|
||||
def call(self, method, params):
|
||||
self.calls.append(method)
|
||||
if method == 'turn/steer':
|
||||
raise RuntimeError('inactive')
|
||||
return limits(90)
|
||||
def close(self):
|
||||
pass
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', FakeRPC), patch.object(guard, 'snapshot'), \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(FakeRPC.calls.count('turn/steer'), 1)
|
||||
self.assertNotIn('turn/start', FakeRPC.calls)
|
||||
|
||||
def test_reset_timestamp_jitter_does_not_repeat_pause(self):
|
||||
class FakeRPC:
|
||||
steers = 0
|
||||
reads = 0
|
||||
def call(self, method, params):
|
||||
if method == 'turn/steer':
|
||||
self.steers += 1
|
||||
return {}
|
||||
self.reads += 1
|
||||
return limits(90, reset=2000 + self.reads)
|
||||
def close(self):
|
||||
pass
|
||||
rpc = FakeRPC()
|
||||
guard.register({'session_id': 'root', 'turn_id': 'turn'})
|
||||
with patch.object(guard, 'RPC', return_value=rpc), patch.object(guard, 'snapshot'), \
|
||||
patch.object(guard.time, 'time', return_value=1000):
|
||||
guard.tick()
|
||||
guard.tick()
|
||||
self.assertEqual(rpc.steers, 1)
|
||||
|
||||
def test_recovery_preserves_canonical_handoff(self):
|
||||
project = Path(self.temp.name) / 'project'
|
||||
(project / '.planning').mkdir(parents=True)
|
||||
handoff = project / '.planning/HANDOFF.json'
|
||||
handoff.write_text('{"precise": "continuation"}')
|
||||
process = unittest.mock.MagicMock()
|
||||
process.__enter__.return_value = process
|
||||
process.stdout.read.return_value = b'test git output'
|
||||
with patch.object(guard, 'ROOT', project), patch.object(guard.subprocess, 'Popen', return_value=process):
|
||||
guard.snapshot(guard.quota(limits(90), 1000))
|
||||
guard.snapshot(guard.quota(limits(90, reset=2001), 1000))
|
||||
self.assertEqual(handoff.read_text(), '{"precise": "continuation"}')
|
||||
self.assertFalse((self.state / 'recovery-2001').exists())
|
||||
record = guard.read(self.state / 'recovery-2000/record.json')
|
||||
self.assertIn('continuation', record['gsd']['.planning/HANDOFF.json'])
|
||||
|
||||
|
||||
class FrameTests(unittest.TestCase):
|
||||
def rpc(self, data=b''):
|
||||
rpc = guard.RPC.__new__(guard.RPC)
|
||||
rpc.sock = unittest.mock.Mock()
|
||||
rpc.buffer = data
|
||||
rpc.sequence = 0
|
||||
return rpc
|
||||
|
||||
def test_eof_terminates_instead_of_spinning(self):
|
||||
rpc = self.rpc()
|
||||
rpc.sock.recv.return_value = b''
|
||||
with self.assertRaises(EOFError):
|
||||
rpc.take(1)
|
||||
|
||||
def test_oversize_rejected_before_payload_read(self):
|
||||
rpc = self.rpc(b'\x81\x7f' + struct.pack('!Q', 2**21))
|
||||
with self.assertRaises(ValueError):
|
||||
rpc.message()
|
||||
rpc.sock.recv.assert_not_called()
|
||||
|
||||
def test_ping_and_fragmented_text(self):
|
||||
rpc = self.rpc(b'\x89\x01x' + b'\x01\x05{"a":' + b'\x80\x021}')
|
||||
self.assertEqual(rpc.message(), {'a': 1})
|
||||
rpc.sock.sendall.assert_called_once()
|
||||
|
||||
def test_client_masks_outgoing_frames(self):
|
||||
rpc = self.rpc()
|
||||
rpc.send({'test': 'value'})
|
||||
wire = rpc.sock.sendall.call_args.args[0]
|
||||
self.assertTrue(wire[1] & 128)
|
||||
mask = wire[2:6]
|
||||
raw = bytes(c ^ mask[i % 4] for i, c in enumerate(wire[6:]))
|
||||
self.assertEqual(json.loads(raw), {'test': 'value'})
|
||||
|
||||
def test_notifications_skipped_until_matching_response(self):
|
||||
rpc = self.rpc()
|
||||
with patch.object(rpc, 'message', side_effect=[{'method': 'notify'}, {'id': 1, 'result': {'ok': True}}]):
|
||||
self.assertEqual(rpc.call('test', {}), {'ok': True})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Linux Codex quota watcher. No model starts, auth-file reads, or commits."""
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError: # Keep repository-wide Python test discovery portable.
|
||||
fcntl = None
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
STATE = ROOT / 'artifacts/gsd-quota'
|
||||
PAUSE = ('The user enabled the GamePort quota guard. Five-hour quota has reached '
|
||||
'15% remaining or less. Stop dispatching new work. Reach the nearest safe '
|
||||
'checkpoint, preserve partial/unverified edits without committing them, '
|
||||
'and use $gsd-pause-work to save the exact continuation. End the turn after '
|
||||
'saving. Do not wait in a model loop or resume automatically.')
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, 'w') as out:
|
||||
json.dump(value, out, indent=2)
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def read(path, default=None):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class RPC:
|
||||
"""Bounded RFC6455 client for the existing Unix app-server socket."""
|
||||
def __init__(self, path=None):
|
||||
self.sock = socket.socket(socket.AF_UNIX)
|
||||
self.sock.settimeout(6)
|
||||
self.buffer = b''
|
||||
self.sequence = 0
|
||||
try:
|
||||
self.sock.connect(str(path or Path(os.environ.get('CODEX_HOME', Path.home() / '.codex')) /
|
||||
'app-server-control/app-server-control.sock'))
|
||||
key = base64.b64encode(os.urandom(16)).decode()
|
||||
self.sock.sendall(('GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\n'
|
||||
'Connection: Upgrade\r\nSec-WebSocket-Version: 13\r\n'
|
||||
f'Sec-WebSocket-Key: {key}\r\n\r\n').encode())
|
||||
while b'\r\n\r\n' not in self.buffer:
|
||||
self.receive()
|
||||
if len(self.buffer) > 16384:
|
||||
raise ValueError('Oversized handshake')
|
||||
header, self.buffer = self.buffer.split(b'\r\n\r\n', 1)
|
||||
expected = base64.b64encode(hashlib.sha1(
|
||||
(key + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest())
|
||||
if not header.startswith(b'HTTP/1.1 101 ') or expected.lower() not in header.lower():
|
||||
raise ValueError('Invalid WebSocket handshake')
|
||||
self.call('initialize', {'clientInfo': {'name': 'gameport-quota-guard', 'version': '1.0'}})
|
||||
self.send({'method': 'initialized'})
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
self.sock.close()
|
||||
|
||||
def receive(self):
|
||||
part = self.sock.recv(65536)
|
||||
if not part:
|
||||
raise EOFError('Codex daemon closed connection')
|
||||
self.buffer += part
|
||||
|
||||
def take(self, size):
|
||||
while len(self.buffer) < size:
|
||||
self.receive()
|
||||
result, self.buffer = self.buffer[:size], self.buffer[size:]
|
||||
return result
|
||||
|
||||
def frame(self, payload, opcode=1):
|
||||
mask = os.urandom(4)
|
||||
size = len(payload)
|
||||
length = bytes([size | 128]) if size < 126 else b'\xfe' + struct.pack('!H', size)
|
||||
self.sock.sendall(bytes([128 | opcode]) + length + mask +
|
||||
bytes(c ^ mask[i % 4] for i, c in enumerate(payload)))
|
||||
|
||||
def send(self, obj):
|
||||
self.frame(json.dumps(obj).encode())
|
||||
|
||||
def message(self):
|
||||
result = b''
|
||||
while True:
|
||||
first, second = self.take(2)
|
||||
length = second & 127
|
||||
if length == 126:
|
||||
length = struct.unpack('!H', self.take(2))[0]
|
||||
elif length == 127:
|
||||
length = struct.unpack('!Q', self.take(8))[0]
|
||||
if second & 128 or length + len(result) > 2**20:
|
||||
raise ValueError('Unsupported or oversized server frame')
|
||||
payload = self.take(length)
|
||||
opcode = first & 15
|
||||
if opcode == 8:
|
||||
raise EOFError('WebSocket closed')
|
||||
if opcode == 9:
|
||||
self.frame(payload, 10)
|
||||
continue
|
||||
if opcode == 10:
|
||||
continue
|
||||
if opcode not in (0, 1):
|
||||
raise ValueError('Unsupported WebSocket opcode')
|
||||
result += payload
|
||||
if first & 128:
|
||||
return json.loads(result)
|
||||
|
||||
def call(self, method, params):
|
||||
self.sequence += 1
|
||||
ident = self.sequence
|
||||
self.send({'id': ident, 'method': method, 'params': params})
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
response = self.message()
|
||||
if response.get('id') != ident:
|
||||
continue
|
||||
if 'error' in response:
|
||||
raise RuntimeError(str(response['error'].get('code', 'RPC error')))
|
||||
return response.get('result', {})
|
||||
raise TimeoutError('RPC deadline exceeded')
|
||||
|
||||
|
||||
def quota(result, now):
|
||||
buckets = result.get('rateLimitsByLimitId')
|
||||
# Do not let unused specialty-model buckets override the general Codex quota.
|
||||
if isinstance(buckets, dict) and 'codex' in buckets:
|
||||
snapshots = [buckets['codex']]
|
||||
else:
|
||||
snapshots = [result.get('rateLimits', {})]
|
||||
candidates = []
|
||||
for snapshot in snapshots:
|
||||
for name in ('primary', 'secondary'):
|
||||
window = (snapshot or {}).get(name) or {}
|
||||
used, reset = window.get('usedPercent'), window.get('resetsAt')
|
||||
if (window.get('windowDurationMins') == 300 and
|
||||
isinstance(used, (int, float)) and not isinstance(used, bool) and
|
||||
math.isfinite(used) and 0 <= used <= 100 and
|
||||
isinstance(reset, (int, float)) and reset > now):
|
||||
candidates.append({'remaining': 100 - used, 'resetsAt': reset})
|
||||
if not candidates:
|
||||
raise ValueError('No current, complete 300-minute quota window')
|
||||
return dict(min(candidates, key=lambda x: x['remaining']), checkedAt=now)
|
||||
|
||||
|
||||
def stage(sample, now):
|
||||
if not sample or now - sample.get('checkedAt', 0) > 180 or sample.get('resetsAt', 0) <= now:
|
||||
return 'unknown'
|
||||
remaining = sample['remaining']
|
||||
return 'pause' if remaining <= 15 else 'drain' if remaining <= 20 else 'ready'
|
||||
|
||||
|
||||
def snapshot(sample):
|
||||
"""Preserve a separate recovery record; never replace the detailed GSD handoff."""
|
||||
previous = read(STATE / 'snapshot.json', {})
|
||||
# Server reset estimates can move by a second between reads.
|
||||
if abs(previous.get('resetsAt', 0) - sample['resetsAt']) <= 120:
|
||||
return
|
||||
folder = STATE / ('recovery-' + str(int(sample['resetsAt'])))
|
||||
if (folder / 'record.json').exists():
|
||||
return
|
||||
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
def git(*args):
|
||||
# Bounded output, tracked source only; no originals/untracked asset copies.
|
||||
with subprocess.Popen(['git', *args], cwd=ROOT, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL) as proc:
|
||||
data = proc.stdout.read(2**21 + 1)
|
||||
if len(data) > 2**21:
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
return {'text': data[:2**21].decode('utf-8', 'replace'), 'truncated': len(data) > 2**21}
|
||||
atomic(folder / 'record.json', {
|
||||
'savedAt': time.time(), 'quota': sample, 'head': git('rev-parse', 'HEAD'),
|
||||
'status': git('status', '--short'), 'trackedPatch': git('diff', 'HEAD', '--no-ext-diff', '--no-textconv'),
|
||||
'gsd': {str(p.relative_to(ROOT)): p.read_text()[:262144] for p in
|
||||
(ROOT / '.planning/HANDOFF.json', ROOT / '.planning/STATE.md') if p.is_file()},
|
||||
'limits': 'Tracked patch capped at 2 MiB; untracked contents remain in working tree. '
|
||||
'This recovery record supplements the canonical GSD handoff.'})
|
||||
atomic(STATE / 'snapshot.json', {'resetsAt': sample['resetsAt'], 'path': str(folder)})
|
||||
|
||||
|
||||
def dispatch(tool):
|
||||
return tool.split('__')[-1].split('.')[-1] in ('Agent', 'spawn_agent', 'followup_task')
|
||||
|
||||
|
||||
def hook_response(event, tool, sample, now):
|
||||
level = stage(sample, now)
|
||||
if event == 'PreToolUse' and dispatch(tool) and level != 'ready':
|
||||
return {'hookSpecificOutput': {'hookEventName': event, 'permissionDecision': 'deny',
|
||||
'permissionDecisionReason': f'Quota guard: {level}. No new agent work until '
|
||||
'fresh five-hour quota is above 20%. Save a checkpoint if at 15% or less.'}}
|
||||
if event in ('PreToolUse', 'PostToolUse') and level == 'pause':
|
||||
return {'hookSpecificOutput': {'hookEventName': event, 'additionalContext': PAUSE}}
|
||||
return {}
|
||||
|
||||
|
||||
def handle_hook(payload):
|
||||
event = payload.get('hook_event_name')
|
||||
if event in ('PreModelRequest', 'UserPromptSubmit'):
|
||||
register(payload)
|
||||
sample = read(STATE / 'quota.json')
|
||||
response = hook_response(event, payload.get('tool_name', ''), sample, time.time())
|
||||
output = response.get('hookSpecificOutput', {})
|
||||
if 'additionalContext' in output:
|
||||
# One reminder per turn/window; tool denials are never suppressed.
|
||||
key = str(payload.get('turn_id'))
|
||||
marker = STATE / 'context' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
||||
if marker.exists():
|
||||
return {}
|
||||
atomic(marker, {'at': time.time()})
|
||||
return response
|
||||
|
||||
|
||||
def register(payload):
|
||||
# Per-thread files avoid races between root and child hooks. Never infer turns.
|
||||
thread = payload.get('agent_id') or payload.get('session_id')
|
||||
turn = payload.get('turn_id')
|
||||
if isinstance(thread, str) and isinstance(turn, str) and thread and turn:
|
||||
ident = hashlib.sha256(thread.encode()).hexdigest()
|
||||
atomic(STATE / 'threads' / (ident + '.json'),
|
||||
{'threadId': thread, 'turnId': turn, 'seenAt': time.time()})
|
||||
|
||||
|
||||
def tick():
|
||||
with contextlib.closing(RPC()) as rpc:
|
||||
sample = quota(rpc.call('account/rateLimits/read', {}), time.time())
|
||||
atomic(STATE / 'quota.json', sample)
|
||||
if stage(sample, time.time()) == 'pause':
|
||||
snapshot(sample)
|
||||
for path in (STATE / 'threads').glob('*.json'):
|
||||
target = read(path, {})
|
||||
if time.time() - target.get('seenAt', 0) > 86400:
|
||||
continue
|
||||
key = f"{target.get('threadId')}:{target.get('turnId')}"
|
||||
marker = STATE / 'sent' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
||||
if marker.exists():
|
||||
continue
|
||||
try:
|
||||
rpc.call('turn/steer', {'threadId': target['threadId'],
|
||||
'expectedTurnId': target['turnId'],
|
||||
'input': [{'type': 'text', 'text': PAUSE}]})
|
||||
atomic(marker, {'sentAt': time.time()})
|
||||
except RuntimeError:
|
||||
# Inactive/wrong turn: no new turn, no interrupt, no costly retry loop.
|
||||
atomic(marker, {'inactiveAt': time.time()})
|
||||
return sample
|
||||
|
||||
|
||||
def start():
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
with open(STATE / 'watch.log', 'ab') as log:
|
||||
os.chmod(STATE / 'watch.log', 0o600)
|
||||
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'watch'],
|
||||
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
||||
start_new_session=True, close_fds=True)
|
||||
|
||||
|
||||
def watch():
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
stop = STATE / 'stop'
|
||||
stop.unlink(missing_ok=True)
|
||||
atomic(STATE / 'watcher.json', {'pid': os.getpid(), 'startedAt': time.time()})
|
||||
while not stop.exists():
|
||||
try:
|
||||
tick()
|
||||
(STATE / 'error.json').unlink(missing_ok=True)
|
||||
except (OSError, ValueError, RuntimeError, EOFError, TimeoutError) as error:
|
||||
atomic(STATE / 'error.json', {'at': time.time(), 'type': type(error).__name__})
|
||||
for _ in range(60):
|
||||
if stop.exists():
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=['hook', 'watch', 'start', 'stop', 'status', 'refresh'])
|
||||
args = parser.parse_args()
|
||||
if fcntl is None:
|
||||
parser.error('The quota watcher runs on the Linux Codex host only.')
|
||||
if args.command == 'hook':
|
||||
payload = json.load(sys.stdin)
|
||||
if not Path(payload.get('cwd', '')).resolve().is_relative_to(ROOT):
|
||||
return
|
||||
start()
|
||||
print(json.dumps(handle_hook(payload)))
|
||||
elif args.command == 'watch':
|
||||
watch()
|
||||
elif args.command == 'start':
|
||||
start()
|
||||
elif args.command == 'stop':
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
(STATE / 'stop').touch(mode=0o600)
|
||||
elif args.command == 'refresh':
|
||||
print(json.dumps(tick()))
|
||||
else:
|
||||
sample = read(STATE / 'quota.json')
|
||||
running = False
|
||||
if (STATE / 'watch.lock').exists():
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
running = True
|
||||
print(json.dumps({'stage': stage(sample, time.time()), 'quota': sample,
|
||||
'running': running,
|
||||
'watcher': read(STATE / 'watcher.json'),
|
||||
'error': read(STATE / 'error.json')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user