Files
Neutron/tools/workflow/codex_quota_guard.py
T

341 lines
14 KiB
Python

#!/usr/bin/env python3
"""Local Linux Codex quota watcher. No model starts, auth-file reads, or commits."""
import argparse
import base64
import contextlib
import hashlib
import json
import math
import os
from pathlib import Path
import socket
import struct
import subprocess
import sys
import time
try:
import fcntl
except ImportError: # Keep repository-wide Python test discovery portable.
fcntl = None
ROOT = Path(__file__).resolve().parents[2]
STATE = ROOT / 'artifacts/gsd-quota'
PAUSE = ('The user enabled the GamePort quota guard. Five-hour quota has reached '
'15% remaining or less. Stop dispatching new work. Reach the nearest safe '
'checkpoint, preserve partial/unverified edits without committing them, '
'and use $gsd-pause-work to save the exact continuation. End the turn after '
'saving. Do not wait in a model loop or resume automatically.')
def atomic(path, value):
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w') as out:
json.dump(value, out, indent=2)
os.replace(temp, path)
def read(path, default=None):
try:
return json.loads(path.read_text())
except (OSError, ValueError):
return default
class RPC:
"""Bounded RFC6455 client for the existing Unix app-server socket."""
def __init__(self, path=None):
self.sock = socket.socket(socket.AF_UNIX)
self.sock.settimeout(6)
self.buffer = b''
self.sequence = 0
try:
self.sock.connect(str(path or Path(os.environ.get('CODEX_HOME', Path.home() / '.codex')) /
'app-server-control/app-server-control.sock'))
key = base64.b64encode(os.urandom(16)).decode()
self.sock.sendall(('GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\n'
'Connection: Upgrade\r\nSec-WebSocket-Version: 13\r\n'
f'Sec-WebSocket-Key: {key}\r\n\r\n').encode())
while b'\r\n\r\n' not in self.buffer:
self.receive()
if len(self.buffer) > 16384:
raise ValueError('Oversized handshake')
header, self.buffer = self.buffer.split(b'\r\n\r\n', 1)
expected = base64.b64encode(hashlib.sha1(
(key + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest())
if not header.startswith(b'HTTP/1.1 101 ') or expected.lower() not in header.lower():
raise ValueError('Invalid WebSocket handshake')
self.call('initialize', {'clientInfo': {'name': 'gameport-quota-guard', 'version': '1.0'}})
self.send({'method': 'initialized'})
except Exception:
self.close()
raise
def close(self):
self.sock.close()
def receive(self):
part = self.sock.recv(65536)
if not part:
raise EOFError('Codex daemon closed connection')
self.buffer += part
def take(self, size):
while len(self.buffer) < size:
self.receive()
result, self.buffer = self.buffer[:size], self.buffer[size:]
return result
def frame(self, payload, opcode=1):
mask = os.urandom(4)
size = len(payload)
length = bytes([size | 128]) if size < 126 else b'\xfe' + struct.pack('!H', size)
self.sock.sendall(bytes([128 | opcode]) + length + mask +
bytes(c ^ mask[i % 4] for i, c in enumerate(payload)))
def send(self, obj):
self.frame(json.dumps(obj).encode())
def message(self):
result = b''
while True:
first, second = self.take(2)
length = second & 127
if length == 126:
length = struct.unpack('!H', self.take(2))[0]
elif length == 127:
length = struct.unpack('!Q', self.take(8))[0]
if second & 128 or length + len(result) > 2**20:
raise ValueError('Unsupported or oversized server frame')
payload = self.take(length)
opcode = first & 15
if opcode == 8:
raise EOFError('WebSocket closed')
if opcode == 9:
self.frame(payload, 10)
continue
if opcode == 10:
continue
if opcode not in (0, 1):
raise ValueError('Unsupported WebSocket opcode')
result += payload
if first & 128:
return json.loads(result)
def call(self, method, params):
self.sequence += 1
ident = self.sequence
self.send({'id': ident, 'method': method, 'params': params})
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
response = self.message()
if response.get('id') != ident:
continue
if 'error' in response:
raise RuntimeError(str(response['error'].get('code', 'RPC error')))
return response.get('result', {})
raise TimeoutError('RPC deadline exceeded')
def quota(result, now):
buckets = result.get('rateLimitsByLimitId')
# Do not let unused specialty-model buckets override the general Codex quota.
if isinstance(buckets, dict) and 'codex' in buckets:
snapshots = [buckets['codex']]
else:
snapshots = [result.get('rateLimits', {})]
candidates = []
for snapshot in snapshots:
for name in ('primary', 'secondary'):
window = (snapshot or {}).get(name) or {}
used, reset = window.get('usedPercent'), window.get('resetsAt')
if (window.get('windowDurationMins') == 300 and
isinstance(used, (int, float)) and not isinstance(used, bool) and
math.isfinite(used) and 0 <= used <= 100 and
isinstance(reset, (int, float)) and reset > now):
candidates.append({'remaining': 100 - used, 'resetsAt': reset})
if not candidates:
raise ValueError('No current, complete 300-minute quota window')
return dict(min(candidates, key=lambda x: x['remaining']), checkedAt=now)
def stage(sample, now):
if not sample or now - sample.get('checkedAt', 0) > 180 or sample.get('resetsAt', 0) <= now:
return 'unknown'
remaining = sample['remaining']
return 'pause' if remaining <= 15 else 'drain' if remaining <= 20 else 'ready'
def snapshot(sample):
"""Preserve a separate recovery record; never replace the detailed GSD handoff."""
previous = read(STATE / 'snapshot.json', {})
# Server reset estimates can move by a second between reads.
if abs(previous.get('resetsAt', 0) - sample['resetsAt']) <= 120:
return
folder = STATE / ('recovery-' + str(int(sample['resetsAt'])))
if (folder / 'record.json').exists():
return
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
def git(*args):
# Bounded output, tracked source only; no originals/untracked asset copies.
with subprocess.Popen(['git', *args], cwd=ROOT, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL) as proc:
data = proc.stdout.read(2**21 + 1)
if len(data) > 2**21:
proc.kill()
proc.wait(timeout=10)
return {'text': data[:2**21].decode('utf-8', 'replace'), 'truncated': len(data) > 2**21}
atomic(folder / 'record.json', {
'savedAt': time.time(), 'quota': sample, 'head': git('rev-parse', 'HEAD'),
'status': git('status', '--short'), 'trackedPatch': git('diff', 'HEAD', '--no-ext-diff', '--no-textconv'),
'gsd': {str(p.relative_to(ROOT)): p.read_text()[:262144] for p in
(ROOT / '.planning/HANDOFF.json', ROOT / '.planning/STATE.md') if p.is_file()},
'limits': 'Tracked patch capped at 2 MiB; untracked contents remain in working tree. '
'This recovery record supplements the canonical GSD handoff.'})
atomic(STATE / 'snapshot.json', {'resetsAt': sample['resetsAt'], 'path': str(folder)})
def dispatch(tool):
return tool.split('__')[-1].split('.')[-1] in ('Agent', 'spawn_agent', 'followup_task')
def hook_response(event, tool, sample, now):
level = stage(sample, now)
if event == 'PreToolUse' and dispatch(tool) and level != 'ready':
return {'hookSpecificOutput': {'hookEventName': event, 'permissionDecision': 'deny',
'permissionDecisionReason': f'Quota guard: {level}. No new agent work until '
'fresh five-hour quota is above 20%. Save a checkpoint if at 15% or less.'}}
if event in ('PreToolUse', 'PostToolUse') and level == 'pause':
return {'hookSpecificOutput': {'hookEventName': event, 'additionalContext': PAUSE}}
return {}
def handle_hook(payload):
event = payload.get('hook_event_name')
if event in ('PreModelRequest', 'UserPromptSubmit'):
register(payload)
sample = read(STATE / 'quota.json')
response = hook_response(event, payload.get('tool_name', ''), sample, time.time())
output = response.get('hookSpecificOutput', {})
if 'additionalContext' in output:
# One reminder per turn/window; tool denials are never suppressed.
key = str(payload.get('turn_id'))
marker = STATE / 'context' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
if marker.exists():
return {}
atomic(marker, {'at': time.time()})
return response
def register(payload):
# Per-thread files avoid races between root and child hooks. Never infer turns.
thread = payload.get('agent_id') or payload.get('session_id')
turn = payload.get('turn_id')
if isinstance(thread, str) and isinstance(turn, str) and thread and turn:
ident = hashlib.sha256(thread.encode()).hexdigest()
atomic(STATE / 'threads' / (ident + '.json'),
{'threadId': thread, 'turnId': turn, 'seenAt': time.time()})
def tick():
with contextlib.closing(RPC()) as rpc:
sample = quota(rpc.call('account/rateLimits/read', {}), time.time())
atomic(STATE / 'quota.json', sample)
if stage(sample, time.time()) == 'pause':
snapshot(sample)
for path in (STATE / 'threads').glob('*.json'):
target = read(path, {})
if time.time() - target.get('seenAt', 0) > 86400:
continue
key = f"{target.get('threadId')}:{target.get('turnId')}"
marker = STATE / 'sent' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
if marker.exists():
continue
try:
rpc.call('turn/steer', {'threadId': target['threadId'],
'expectedTurnId': target['turnId'],
'input': [{'type': 'text', 'text': PAUSE}]})
atomic(marker, {'sentAt': time.time()})
except RuntimeError:
# Inactive/wrong turn: no new turn, no interrupt, no costly retry loop.
atomic(marker, {'inactiveAt': time.time()})
return sample
def start():
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
with open(STATE / 'watch.lock', 'a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return
with open(STATE / 'watch.log', 'ab') as log:
os.chmod(STATE / 'watch.log', 0o600)
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'watch'],
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
start_new_session=True, close_fds=True)
def watch():
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
with open(STATE / 'watch.lock', 'a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return
stop = STATE / 'stop'
stop.unlink(missing_ok=True)
atomic(STATE / 'watcher.json', {'pid': os.getpid(), 'startedAt': time.time()})
while not stop.exists():
try:
tick()
(STATE / 'error.json').unlink(missing_ok=True)
except (OSError, ValueError, RuntimeError, EOFError, TimeoutError) as error:
atomic(STATE / 'error.json', {'at': time.time(), 'type': type(error).__name__})
for _ in range(60):
if stop.exists():
return
time.sleep(1)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=['hook', 'watch', 'start', 'stop', 'status', 'refresh'])
args = parser.parse_args()
if fcntl is None:
parser.error('The quota watcher runs on the Linux Codex host only.')
if args.command == 'hook':
payload = json.load(sys.stdin)
if not Path(payload.get('cwd', '')).resolve().is_relative_to(ROOT):
return
start()
print(json.dumps(handle_hook(payload)))
elif args.command == 'watch':
watch()
elif args.command == 'start':
start()
elif args.command == 'stop':
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
(STATE / 'stop').touch(mode=0o600)
elif args.command == 'refresh':
print(json.dumps(tick()))
else:
sample = read(STATE / 'quota.json')
running = False
if (STATE / 'watch.lock').exists():
with open(STATE / 'watch.lock', 'a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
running = True
print(json.dumps({'stage': stage(sample, time.time()), 'quota': sample,
'running': running,
'watcher': read(STATE / 'watcher.json'),
'error': read(STATE / 'error.json')}))
if __name__ == '__main__':
main()