341 lines
14 KiB
Python
341 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Local Linux Codex quota watcher. No model starts, auth-file reads, or commits."""
|
|
import argparse
|
|
import base64
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
try:
|
|
import fcntl
|
|
except ImportError: # Keep repository-wide Python test discovery portable.
|
|
fcntl = None
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
STATE = ROOT / 'artifacts/gsd-quota'
|
|
PAUSE = ('The user enabled the GamePort quota guard. Five-hour quota has reached '
|
|
'15% remaining or less. Stop dispatching new work. Reach the nearest safe '
|
|
'checkpoint, preserve partial/unverified edits without committing them, '
|
|
'and use $gsd-pause-work to save the exact continuation. End the turn after '
|
|
'saving. Do not wait in a model loop or resume automatically.')
|
|
|
|
|
|
def atomic(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
|
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, 'w') as out:
|
|
json.dump(value, out, indent=2)
|
|
os.replace(temp, path)
|
|
|
|
|
|
def read(path, default=None):
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
|
|
class RPC:
|
|
"""Bounded RFC6455 client for the existing Unix app-server socket."""
|
|
def __init__(self, path=None):
|
|
self.sock = socket.socket(socket.AF_UNIX)
|
|
self.sock.settimeout(6)
|
|
self.buffer = b''
|
|
self.sequence = 0
|
|
try:
|
|
self.sock.connect(str(path or Path(os.environ.get('CODEX_HOME', Path.home() / '.codex')) /
|
|
'app-server-control/app-server-control.sock'))
|
|
key = base64.b64encode(os.urandom(16)).decode()
|
|
self.sock.sendall(('GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\n'
|
|
'Connection: Upgrade\r\nSec-WebSocket-Version: 13\r\n'
|
|
f'Sec-WebSocket-Key: {key}\r\n\r\n').encode())
|
|
while b'\r\n\r\n' not in self.buffer:
|
|
self.receive()
|
|
if len(self.buffer) > 16384:
|
|
raise ValueError('Oversized handshake')
|
|
header, self.buffer = self.buffer.split(b'\r\n\r\n', 1)
|
|
expected = base64.b64encode(hashlib.sha1(
|
|
(key + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest())
|
|
if not header.startswith(b'HTTP/1.1 101 ') or expected.lower() not in header.lower():
|
|
raise ValueError('Invalid WebSocket handshake')
|
|
self.call('initialize', {'clientInfo': {'name': 'gameport-quota-guard', 'version': '1.0'}})
|
|
self.send({'method': 'initialized'})
|
|
except Exception:
|
|
self.close()
|
|
raise
|
|
|
|
def close(self):
|
|
self.sock.close()
|
|
|
|
def receive(self):
|
|
part = self.sock.recv(65536)
|
|
if not part:
|
|
raise EOFError('Codex daemon closed connection')
|
|
self.buffer += part
|
|
|
|
def take(self, size):
|
|
while len(self.buffer) < size:
|
|
self.receive()
|
|
result, self.buffer = self.buffer[:size], self.buffer[size:]
|
|
return result
|
|
|
|
def frame(self, payload, opcode=1):
|
|
mask = os.urandom(4)
|
|
size = len(payload)
|
|
length = bytes([size | 128]) if size < 126 else b'\xfe' + struct.pack('!H', size)
|
|
self.sock.sendall(bytes([128 | opcode]) + length + mask +
|
|
bytes(c ^ mask[i % 4] for i, c in enumerate(payload)))
|
|
|
|
def send(self, obj):
|
|
self.frame(json.dumps(obj).encode())
|
|
|
|
def message(self):
|
|
result = b''
|
|
while True:
|
|
first, second = self.take(2)
|
|
length = second & 127
|
|
if length == 126:
|
|
length = struct.unpack('!H', self.take(2))[0]
|
|
elif length == 127:
|
|
length = struct.unpack('!Q', self.take(8))[0]
|
|
if second & 128 or length + len(result) > 2**20:
|
|
raise ValueError('Unsupported or oversized server frame')
|
|
payload = self.take(length)
|
|
opcode = first & 15
|
|
if opcode == 8:
|
|
raise EOFError('WebSocket closed')
|
|
if opcode == 9:
|
|
self.frame(payload, 10)
|
|
continue
|
|
if opcode == 10:
|
|
continue
|
|
if opcode not in (0, 1):
|
|
raise ValueError('Unsupported WebSocket opcode')
|
|
result += payload
|
|
if first & 128:
|
|
return json.loads(result)
|
|
|
|
def call(self, method, params):
|
|
self.sequence += 1
|
|
ident = self.sequence
|
|
self.send({'id': ident, 'method': method, 'params': params})
|
|
deadline = time.monotonic() + 10
|
|
while time.monotonic() < deadline:
|
|
response = self.message()
|
|
if response.get('id') != ident:
|
|
continue
|
|
if 'error' in response:
|
|
raise RuntimeError(str(response['error'].get('code', 'RPC error')))
|
|
return response.get('result', {})
|
|
raise TimeoutError('RPC deadline exceeded')
|
|
|
|
|
|
def quota(result, now):
|
|
buckets = result.get('rateLimitsByLimitId')
|
|
# Do not let unused specialty-model buckets override the general Codex quota.
|
|
if isinstance(buckets, dict) and 'codex' in buckets:
|
|
snapshots = [buckets['codex']]
|
|
else:
|
|
snapshots = [result.get('rateLimits', {})]
|
|
candidates = []
|
|
for snapshot in snapshots:
|
|
for name in ('primary', 'secondary'):
|
|
window = (snapshot or {}).get(name) or {}
|
|
used, reset = window.get('usedPercent'), window.get('resetsAt')
|
|
if (window.get('windowDurationMins') == 300 and
|
|
isinstance(used, (int, float)) and not isinstance(used, bool) and
|
|
math.isfinite(used) and 0 <= used <= 100 and
|
|
isinstance(reset, (int, float)) and reset > now):
|
|
candidates.append({'remaining': 100 - used, 'resetsAt': reset})
|
|
if not candidates:
|
|
raise ValueError('No current, complete 300-minute quota window')
|
|
return dict(min(candidates, key=lambda x: x['remaining']), checkedAt=now)
|
|
|
|
|
|
def stage(sample, now):
|
|
if not sample or now - sample.get('checkedAt', 0) > 180 or sample.get('resetsAt', 0) <= now:
|
|
return 'unknown'
|
|
remaining = sample['remaining']
|
|
return 'pause' if remaining <= 15 else 'drain' if remaining <= 20 else 'ready'
|
|
|
|
|
|
def snapshot(sample):
|
|
"""Preserve a separate recovery record; never replace the detailed GSD handoff."""
|
|
previous = read(STATE / 'snapshot.json', {})
|
|
# Server reset estimates can move by a second between reads.
|
|
if abs(previous.get('resetsAt', 0) - sample['resetsAt']) <= 120:
|
|
return
|
|
folder = STATE / ('recovery-' + str(int(sample['resetsAt'])))
|
|
if (folder / 'record.json').exists():
|
|
return
|
|
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
def git(*args):
|
|
# Bounded output, tracked source only; no originals/untracked asset copies.
|
|
with subprocess.Popen(['git', *args], cwd=ROOT, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL) as proc:
|
|
data = proc.stdout.read(2**21 + 1)
|
|
if len(data) > 2**21:
|
|
proc.kill()
|
|
proc.wait(timeout=10)
|
|
return {'text': data[:2**21].decode('utf-8', 'replace'), 'truncated': len(data) > 2**21}
|
|
atomic(folder / 'record.json', {
|
|
'savedAt': time.time(), 'quota': sample, 'head': git('rev-parse', 'HEAD'),
|
|
'status': git('status', '--short'), 'trackedPatch': git('diff', 'HEAD', '--no-ext-diff', '--no-textconv'),
|
|
'gsd': {str(p.relative_to(ROOT)): p.read_text()[:262144] for p in
|
|
(ROOT / '.planning/HANDOFF.json', ROOT / '.planning/STATE.md') if p.is_file()},
|
|
'limits': 'Tracked patch capped at 2 MiB; untracked contents remain in working tree. '
|
|
'This recovery record supplements the canonical GSD handoff.'})
|
|
atomic(STATE / 'snapshot.json', {'resetsAt': sample['resetsAt'], 'path': str(folder)})
|
|
|
|
|
|
def dispatch(tool):
|
|
return tool.split('__')[-1].split('.')[-1] in ('Agent', 'spawn_agent', 'followup_task')
|
|
|
|
|
|
def hook_response(event, tool, sample, now):
|
|
level = stage(sample, now)
|
|
if event == 'PreToolUse' and dispatch(tool) and level != 'ready':
|
|
return {'hookSpecificOutput': {'hookEventName': event, 'permissionDecision': 'deny',
|
|
'permissionDecisionReason': f'Quota guard: {level}. No new agent work until '
|
|
'fresh five-hour quota is above 20%. Save a checkpoint if at 15% or less.'}}
|
|
if event in ('PreToolUse', 'PostToolUse') and level == 'pause':
|
|
return {'hookSpecificOutput': {'hookEventName': event, 'additionalContext': PAUSE}}
|
|
return {}
|
|
|
|
|
|
def handle_hook(payload):
|
|
event = payload.get('hook_event_name')
|
|
if event in ('PreModelRequest', 'UserPromptSubmit'):
|
|
register(payload)
|
|
sample = read(STATE / 'quota.json')
|
|
response = hook_response(event, payload.get('tool_name', ''), sample, time.time())
|
|
output = response.get('hookSpecificOutput', {})
|
|
if 'additionalContext' in output:
|
|
# One reminder per turn/window; tool denials are never suppressed.
|
|
key = str(payload.get('turn_id'))
|
|
marker = STATE / 'context' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
|
if marker.exists():
|
|
return {}
|
|
atomic(marker, {'at': time.time()})
|
|
return response
|
|
|
|
|
|
def register(payload):
|
|
# Per-thread files avoid races between root and child hooks. Never infer turns.
|
|
thread = payload.get('agent_id') or payload.get('session_id')
|
|
turn = payload.get('turn_id')
|
|
if isinstance(thread, str) and isinstance(turn, str) and thread and turn:
|
|
ident = hashlib.sha256(thread.encode()).hexdigest()
|
|
atomic(STATE / 'threads' / (ident + '.json'),
|
|
{'threadId': thread, 'turnId': turn, 'seenAt': time.time()})
|
|
|
|
|
|
def tick():
|
|
with contextlib.closing(RPC()) as rpc:
|
|
sample = quota(rpc.call('account/rateLimits/read', {}), time.time())
|
|
atomic(STATE / 'quota.json', sample)
|
|
if stage(sample, time.time()) == 'pause':
|
|
snapshot(sample)
|
|
for path in (STATE / 'threads').glob('*.json'):
|
|
target = read(path, {})
|
|
if time.time() - target.get('seenAt', 0) > 86400:
|
|
continue
|
|
key = f"{target.get('threadId')}:{target.get('turnId')}"
|
|
marker = STATE / 'sent' / (hashlib.sha256(key.encode()).hexdigest() + '.json')
|
|
if marker.exists():
|
|
continue
|
|
try:
|
|
rpc.call('turn/steer', {'threadId': target['threadId'],
|
|
'expectedTurnId': target['turnId'],
|
|
'input': [{'type': 'text', 'text': PAUSE}]})
|
|
atomic(marker, {'sentAt': time.time()})
|
|
except RuntimeError:
|
|
# Inactive/wrong turn: no new turn, no interrupt, no costly retry loop.
|
|
atomic(marker, {'inactiveAt': time.time()})
|
|
return sample
|
|
|
|
|
|
def start():
|
|
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
with open(STATE / 'watch.lock', 'a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return
|
|
with open(STATE / 'watch.log', 'ab') as log:
|
|
os.chmod(STATE / 'watch.log', 0o600)
|
|
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'watch'],
|
|
cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
|
start_new_session=True, close_fds=True)
|
|
|
|
|
|
def watch():
|
|
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
with open(STATE / 'watch.lock', 'a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return
|
|
stop = STATE / 'stop'
|
|
stop.unlink(missing_ok=True)
|
|
atomic(STATE / 'watcher.json', {'pid': os.getpid(), 'startedAt': time.time()})
|
|
while not stop.exists():
|
|
try:
|
|
tick()
|
|
(STATE / 'error.json').unlink(missing_ok=True)
|
|
except (OSError, ValueError, RuntimeError, EOFError, TimeoutError) as error:
|
|
atomic(STATE / 'error.json', {'at': time.time(), 'type': type(error).__name__})
|
|
for _ in range(60):
|
|
if stop.exists():
|
|
return
|
|
time.sleep(1)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command', choices=['hook', 'watch', 'start', 'stop', 'status', 'refresh'])
|
|
args = parser.parse_args()
|
|
if fcntl is None:
|
|
parser.error('The quota watcher runs on the Linux Codex host only.')
|
|
if args.command == 'hook':
|
|
payload = json.load(sys.stdin)
|
|
if not Path(payload.get('cwd', '')).resolve().is_relative_to(ROOT):
|
|
return
|
|
start()
|
|
print(json.dumps(handle_hook(payload)))
|
|
elif args.command == 'watch':
|
|
watch()
|
|
elif args.command == 'start':
|
|
start()
|
|
elif args.command == 'stop':
|
|
STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
(STATE / 'stop').touch(mode=0o600)
|
|
elif args.command == 'refresh':
|
|
print(json.dumps(tick()))
|
|
else:
|
|
sample = read(STATE / 'quota.json')
|
|
running = False
|
|
if (STATE / 'watch.lock').exists():
|
|
with open(STATE / 'watch.lock', 'a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
running = True
|
|
print(json.dumps({'stage': stage(sample, time.time()), 'quota': sample,
|
|
'running': running,
|
|
'watcher': read(STATE / 'watcher.json'),
|
|
'error': read(STATE / 'error.json')}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|