feat(workflow): checkpoint GSD before five-hour quota exhaustion
This commit is contained in:
1 parent
73954e675d
commit
c9b9f2eba9
9 files changed
+758
-1
No files matched your search
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Linux Codex quota watcher. No model starts, auth-file reads, or commits."""
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError: # Keep repository-wide Python test discovery portable.
|
||||
fcntl = None
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
STATE = ROOT / 'artifacts/gsd-quota'
|
||||
PAUSE = ('The user enabled the GamePort quota guard. Five-hour quota has reached '
|
||||
'15% remaining or less. Stop dispatching new work. Reach the nearest safe '
|
||||
'checkpoint, preserve partial/unverified edits without committing them, '
|
||||
'and use $gsd-pause-work to save the exact continuation. End the turn after '
|
||||
'saving. Do not wait in a model loop or resume automatically.')
|
||||
|
||||
|
||||
def atomic(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, 'w') as out:
|
||||
json.dump(value, out, indent=2)
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def read(path, default=None):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class RPC:
|
||||
"""Bounded RFC6455 client for the existing Unix app-server socket."""
|
||||
def __init__(self, path=None):
|
||||
self.sock = socket.socket(socket.AF_UNIX)
|
||||
self.sock.settimeout(6)
|
||||
self.buffer = b''
|
||||
self.sequence = 0
|
||||
try:
|
||||
self.sock.connect(str(path or Path(os.environ.get('CODEX_HOME', Path.home() / '.codex')) /
|
||||
'app-server-control/app-server-control.sock'))
|
||||
key = base64.b64encode(os.urandom(16)).decode()
|
||||
self.sock.sendall(('GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\n'
|
||||
'Connection: Upgrade\r\nSec-WebSocket-Version: 13\r\n'
|
||||
f'Sec-WebSocket-Key: {key}\r\n\r\n').encode())
|
||||
while b'\r\n\r\n' not in self.buffer:
|
||||
self.receive()
|
||||
if len(self.buffer) > 16384:
|
||||
raise ValueError('Oversized handshake')
|
||||
header, self.buffer = self.buffer.split(b'\r\n\r\n', 1)
|
||||
expected = base64.b64encode(hashlib.sha1(
|
||||
(key + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest())
|
||||
if not header.startswith(b'HTTP/1.1 101 ') or expected.lower() not in header.lower():
|
||||
raise ValueError('Invalid WebSocket handshake')
|
||||
self.call('initialize', {'clientInfo': {'name': 'gameport-quota-guard', 'version': '1.0'}})
|
||||
self.send({'method': 'initialized'})
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
self.sock.close()
|
||||
|
||||
def receive(self):
|
||||
part = self.sock.recv(65536)
|
||||
if not part:
|
||||
raise EOFError('Codex daemon closed connection')
|
||||
self.buffer += part
|
||||
|
||||
def take(self, size):
|
||||
while len(self.buffer) < size:
|
||||
self.receive()
|
||||
result, self.buffer = self.buffer[:size], self.buffer[size:]
|
||||
return result
|
||||
|
||||
def frame(self, payload, opcode=1):
|
||||
mask = os.urandom(4)
|
||||
size = len(payload)
|
||||
length = bytes([size | 128]) if size < 126 else b'\xfe' + struct.pack('!H', size)
|
||||
self.sock.sendall(bytes([128 | opcode]) + length + mask +
|
||||
bytes(c ^ mask[i % 4] for i, c in enumerate(payload)))
|
||||
|
||||
def send(self, obj):
|
||||
self.frame(json.dumps(obj).encode())
|
||||
|
||||
def message(self):
|
||||
result = b''
|
||||
while True:
|
||||
first, second = self.take(2)
|
||||
length = second & 127
|
||||
if length == 126:
|
||||
length = struct.unpack('!H', self.take(2))[0]
|
||||
elif length == 127:
|
||||
length = struct.unpack('!Q', self.take(8))[0]
|
||||
if second & 128 or length + len(result) > 2**20:
|
||||
raise ValueError('Unsupported or oversized server frame')
|
||||
payload = self.take(length)
|
||||
opcode = first & 15
|
||||
if opcode == 8:
|
||||
raise EOFError('WebSocket closed')
|
||||
if opcode == 9:
|
||||
self.frame(payload, 10)
|
||||
continue
|
||||
if opcode == 10:
|
||||
continue
|
||||
if opcode not in (0, 1):
|
||||
raise ValueError('Unsupported WebSocket opcode')
|
||||
result += payload
|
||||
if first & 128:
|
||||
return json.loads(result)
|
||||
|
||||
def call(self, method, params):
|
||||
self.sequence += 1
|
||||
ident = self.sequence
|
||||
self.send({'id': ident, 'method': method, 'params': params})
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
response = self.message()
|
||||
if response.get('id') != ident:
|
||||
continue
|
||||
if 'error' in response:
|
||||
raise RuntimeError(str(response['error'].get('code', 'RPC error')))
|
||||
return response.get('result', {})
|
||||
raise TimeoutError('RPC deadline exceeded')
|
||||
|
||||
|
||||
def quota(result, now):
|
||||
buckets = result.get('rateLimitsByLimitId')
|
||||
# Do not let unused specialty-model buckets override the general Codex quota.
|
||||
if isinstance(buckets, dict) and 'codex' in buckets:
|
||||
snapshots = [buckets['codex']]
|
||||
else:
|
||||
snapshots = [result.get('rateLimits', {})]
|
||||
candidates = []
|
||||
for snapshot in snapshots:
|
||||
for name in ('primary', 'secondary'):
|
||||
window = (snapshot or {}).get(name) or {}
|
||||
used, reset = window.get('usedPercent'), window.get('resetsAt')
|
||||
if (window.get('windowDurationMins') == 300 and
|
||||
isinstance(used, (int, float)) and not isinstance(used, bool) and
|
||||
math.isfinite(used) and 0 <= used <= 100 and
|
||||
isinstance(reset, (int, float)) and reset > now):
|
||||
candidates.append({'remaining': 100 - used, 'resetsAt': reset})
|
||||
if not candidates:
|
||||
raise ValueError('No current, complete 300-minute quota window')
|
||||
return dict(min(candidates, key=lambda x: x['remaining']), checkedAt=now)
|
||||
|
||||
|
||||
def stage(sample, now):
|
||||
if not sample or now - sample.get('checkedAt', 0) > 180 or sample.get('resetsAt', 0) <= now:
|
||||
return 'unknown'
|
||||
remaining = sample['remaining']
|
||||
return 'pause' if remaining <= 15 else 'drain' if remaining <= 20 else 'ready'
|
||||
|
||||
|
||||
def snapshot(sample):
|
||||
"""Preserve a separate recovery record; never replace the detailed GSD handoff."""
|
||||
previous = read(STATE / 'snapshot.json', {})
|
||||
# Server reset estimates can move by a second between reads.
|
||||
if abs(previous.get('resetsAt', 0) - sample['resetsAt']) <= 120:
|
||||
return
|
||||
folder = STATE / ('recovery-' + str(int(sample['resetsAt'])))
|
||||
if (folder / 'record.json').exists():
|
||||
return
|
||||
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
def git(*args):
|
||||
# Bounded output, tracked source only; no originals/untracked asset copies.
|
||||
with subprocess.Popen(['git', *args], cwd=ROOT, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL) as proc:
|
||||
data = proc.stdout.read(2**21 + 1)
|
||||
if len(data) > 2**21:
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
return {'text': data[:2**21].decode('utf-8', 'replace'), 'truncated': len(data) > 2**21}
|
||||
atomic(folder / 'record.json', {
|
||||
'savedAt': time.time(), 'quota': sample, 'head': git('rev-parse', 'HEAD'),
|
||||
'status': git('status', '--short'), 'trackedPatch': git('diff', 'HEAD', '--no-ext-diff', '--no-textconv'),
|
||||
'gsd': {str(p.relative_to(ROOT)): p.read_text()[:262144] for p in
|
||||
(ROOT / '.planning/HANDOFF.json', ROOT / '.planning/STATE.md') if p.is_file()},
|
||||
'limits': 'Tracked patch capped at 2 MiB; untracked contents remain in working tree. '
|
||||
'This recovery record supplements the canonical GSD handoff.'})
|
||||
atomic(STATE / 'snapshot.json', {'resetsAt': sample['resetsAt'], 'path': str(folder)})
|
||||
|
||||
|
||||
def dispatch(tool):
|
||||
return tool.split('__')[-1].split('.')[-1] in ('Agent', 'spawn_agent', 'followup_task')
|
||||
|
||||
|
||||
def hook_response(event, tool, sample, now):
|
||||
level = stage(sample, now)
|
||||
if event == 'PreToolUse' and dispatch(tool) and level != 'ready':
|
||||
return {'hookSpecificOutput': {'hookEventName': event, 'permissionDecision': 'deny',
|
||||
'permissionDecisionReason': f'Quota guard: {level}. No new agent work until '
|
||||
'fresh five-hour quota is above 20%. Save a checkpoint if at 15% or less.'}}
|
||||
if event in ('PreToolUse', 'PostToolUse') and level == 'pause':
|
||||
return {'hookSpecificOutput': {'hookEventName': event, 'additionalContext': PAUSE}}
|
||||
return {}
|
||||
|
||||
|
||||
def handle_hook(payload):
|
||||
event = payload.get('hook_event_name')
|
||||
if event in ('PreModelRequest', 'UserPromptSubmit'):
|
||||
register(payload)
|
||||
sample = read(STATE / 'quota.json')
|
||||
response = hook_response(event, payload.get('tool_name', ''), sample, time.time())
|
||||
output = response.get('hookSpecificOutput', {})
|
||||
if 'additionalContext' in output:
|
||||
# One reminder per turn/window; tool denials are never suppressed.
|
||||
key = str(payload.get('turn_id'))
|
||||
marker = STATE / 'context' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
||||
if marker.exists():
|
||||
return {}
|
||||
atomic(marker, {'at': time.time()})
|
||||
return response
|
||||
|
||||
|
||||
def register(payload):
|
||||
# Per-thread files avoid races between root and child hooks. Never infer turns.
|
||||
thread = payload.get('agent_id') or payload.get('session_id')
|
||||
turn = payload.get('turn_id')
|
||||
if isinstance(thread, str) and isinstance(turn, str) and thread and turn:
|
||||
ident = hashlib.sha256(thread.encode()).hexdigest()
|
||||
atomic(STATE / 'threads' / (ident + '.json'),
|
||||
{'threadId': thread, 'turnId': turn, 'seenAt': time.time()})
|
||||
|
||||
|
||||
def tick():
|
||||
with contextlib.closing(RPC()) as rpc:
|
||||
sample = quota(rpc.call('account/rateLimits/read', {}), time.time())
|
||||
atomic(STATE / 'quota.json', sample)
|
||||
if stage(sample, time.time()) == 'pause':
|
||||
snapshot(sample)
|
||||
for path in (STATE / 'threads').glob('*.json'):
|
||||
target = read(path, {})
|
||||
if time.time() - target.get('seenAt', 0) > 86400:
|
||||
continue
|
||||
key = f"{target.get('threadId')}:{target.get('turnId')}"
|
||||
marker = STATE / 'sent' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
||||
if marker.exists():
|
||||
continue
|
||||
try:
|
||||
rpc.call('turn/steer', {'threadId': target['threadId'],
|
||||
'expectedTurnId': target['turnId'],
|
||||
'input': [{'type': 'text', 'text': PAUSE}]})
|
||||
atomic(marker, {'sentAt': time.time()})
|
||||
except RuntimeError:
|
||||
# Inactive/wrong turn: no new turn, no interrupt, no costly retry loop.
|
||||
atomic(marker, {'inactiveAt': time.time()})
|
||||
return sample
|
||||
|
||||
|
||||
def start():
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
with open(STATE / 'watch.log', 'ab') as log:
|
||||
os.chmod(STATE / 'watch.log', 0o600)
|
||||
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'watch'],
|
||||
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
||||
start_new_session=True, close_fds=True)
|
||||
|
||||
|
||||
def watch():
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
stop = STATE / 'stop'
|
||||
stop.unlink(missing_ok=True)
|
||||
atomic(STATE / 'watcher.json', {'pid': os.getpid(), 'startedAt': time.time()})
|
||||
while not stop.exists():
|
||||
try:
|
||||
tick()
|
||||
(STATE / 'error.json').unlink(missing_ok=True)
|
||||
except (OSError, ValueError, RuntimeError, EOFError, TimeoutError) as error:
|
||||
atomic(STATE / 'error.json', {'at': time.time(), 'type': type(error).__name__})
|
||||
for _ in range(60):
|
||||
if stop.exists():
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=['hook', 'watch', 'start', 'stop', 'status', 'refresh'])
|
||||
args = parser.parse_args()
|
||||
if fcntl is None:
|
||||
parser.error('The quota watcher runs on the Linux Codex host only.')
|
||||
if args.command == 'hook':
|
||||
payload = json.load(sys.stdin)
|
||||
if not Path(payload.get('cwd', '')).resolve().is_relative_to(ROOT):
|
||||
return
|
||||
start()
|
||||
print(json.dumps(handle_hook(payload)))
|
||||
elif args.command == 'watch':
|
||||
watch()
|
||||
elif args.command == 'start':
|
||||
start()
|
||||
elif args.command == 'stop':
|
||||
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
(STATE / 'stop').touch(mode=0o600)
|
||||
elif args.command == 'refresh':
|
||||
print(json.dumps(tick()))
|
||||
else:
|
||||
sample = read(STATE / 'quota.json')
|
||||
running = False
|
||||
if (STATE / 'watch.lock').exists():
|
||||
with open(STATE / 'watch.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
running = True
|
||||
print(json.dumps({'stage': stage(sample, time.time()), 'quota': sample,
|
||||
'running': running,
|
||||
'watcher': read(STATE / 'watcher.json'),
|
||||
'error': read(STATE / 'error.json')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user