Files
Neutron/tools/recon/fidelity_review.py

334 lines
16 KiB
Python

"""Export validated local receipts; serve only the generated, read-only tree.
CLI: export --original IMPORT/receipt.json --port IMPORT/receipt.json --output NEW
serve --root NEW [--port 8765]
Either side may be omitted. Exports are review aids, never acceptance receipts.
The cooperative offline filesystem boundary matches fidelity_receipts: revalidate
before review; hostile concurrent filesystem replacement is outside this model.
"""
import argparse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import mimetypes
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
from urllib.parse import quote, unquote, urlsplit
# A bundled CLI must not create bytecode inside the sealed checkpoint.
if __name__ == '__main__':
sys.dont_write_bytecode = True
from fidelity_receipts import (MAX_JSON_BYTES, canonical_bytes, contained_path,
identify_file, read_json, read_receipt, separated_roots,
validate_receipt)
VIEWER = Path(__file__).with_name('fidelity_viewer')
PROJECT = Path(__file__).resolve().parents[2]
STATIC = ('index.html', 'viewer.mjs', 'style.css')
MAX_RANGE = 16 * 1024 * 1024
def _side(path, side):
if path is None:
return {'missing': 'Missing ' + side + ' reference recording', 'media': None,
'artifacts': [], 'events': [], 'derivatives': []}, []
path = Path(path)
if path.is_symlink():
raise ValueError('Receipt must not be a symlink')
path = path.resolve(strict=True)
value = read_receipt(path)
validation = validate_receipt(value, path.parent)
prefix = 'sides/' + side + '/'
paths = {a['path'] for a in value['artifacts']}
paths.update(a['path'] for a in value['identity'].values() if isinstance(a, dict) and 'path' in a)
if any(name.casefold() in ('receipt.json', 'receipt.sha256') for name in paths):
raise ValueError('Artifact collides with exported receipt')
artifacts = [dict(a, url=quote(prefix + a['path'], safe='/')) for a in value['artifacts']]
videos = [a for a in artifacts if a['role'] in ('video', 'preview')]
# A preview is explicit derivative provenance; a raw recording remains linked.
media = next((a for a in videos if a['role'] == 'preview'), next(iter(videos), None))
events = [e for e in value['events'] if media and e['artifact'] == media['path']]
result = {key: value.get(key) for key in ('receipt_id', 'session_id', 'case_id', 'scope',
'evidence_kind', 'status', 'platform', 'historical', 'limitations', 'identity', 'clocks')}
result.update(artifacts=artifacts, media=media, events=events,
derivatives=value['derivatives'], receipt_url=prefix + 'receipt.json',
missing_evidence=validation['missing_evidence'],
missing='' if media else 'Missing ' + side + ' playable video/preview',
reference_warning=('No genuine original reference: this is ' + value['evidence_kind'] + ' evidence')
if side == 'original' and value['evidence_kind'] != 'original_dxwnd' else '',
acceptance={'state': 'open', 'remaining': ['Review export is not acceptance']})
copies = [(contained_path(path.parent, name), prefix + name) for name in sorted(paths)]
copies.append((path, prefix + 'receipt.json'))
return result, copies
def _events(sides):
indexed = {}
for side, record in sides.items():
seen = set()
for event in record['events']:
if event['name'] in seen:
raise ValueError('Ambiguous duplicate event on selected media: ' + event['name'])
seen.add(event['name'])
indexed.setdefault(event['name'], {'name': event['name'], 'original': None, 'port': None})[side] = event
return list(indexed.values())
def _ledger(path):
if path is None:
return {'schema_version': 1, 'capabilities': []}, []
data = read_json(path)
if data.get('schema_version') != 1 or not isinstance(data.get('capabilities'), list):
raise ValueError('Unsupported capability export')
result = subprocess.run(['git', '-C', str(PROJECT), 'ls-files', '-z'],
capture_output=True, check=True, timeout=20)
if len(result.stdout) > MAX_JSON_BYTES:
raise ValueError('Tracked source inventory exceeds bound')
tracked = set(result.stdout.decode('utf8').split('\0'))
copies = {}
for capability in data['capabilities']:
if not isinstance(capability, dict) or not isinstance(capability.get('source_links', []), list):
raise ValueError('Capability source_links must be an array')
links = []
for item in capability.get('source_links', []):
if not isinstance(item, dict) or not isinstance(item.get('label'), str):
raise ValueError('Source link requires label and repo-relative path')
source = contained_path(PROJECT, item.get('path'))
relative = source.relative_to(PROJECT).as_posix()
if relative.split('/')[0] in {'Original', 'original', 'artifacts', 'dist', '.git', '.tools', '.nuget'}:
raise ValueError('Source links cannot expose original/generated/private roots')
link = {'label': item['label'], 'path': relative, 'href': None,
'validated_local': False, 'missing': None}
if relative not in tracked or not source.is_file():
link['missing'] = 'Missing tracked source reference; acceptance remains open'
else:
actual = identify_file(source)
if actual['size'] > 1024 * 1024:
raise ValueError('Source text exceeds 1 MiB bound')
text = source.read_bytes()
text.decode('utf8')
if b'\0' in text:
raise ValueError('Source reference must be text, not an executable or package')
if item.get('sha256') and item['sha256'] != actual['sha256']:
raise ValueError('Source reference hash mismatch')
exported = 'sources/' + relative
copies[exported] = source
link.update(href=quote(exported, safe='/'), validated_local=True,
sha256=actual['sha256'], size=actual['size'])
links.append(link)
capability['source_links'] = links
return data, [(source, name) for name, source in sorted(copies.items())]
def export_review(original, port, output, ledger=None):
"""Return review model and atomically publish a NEW self-contained directory."""
output = Path(output)
if output.is_symlink() or output.exists():
raise ValueError('Review output must be a new directory')
output = output.resolve()
separated_roots(PROJECT.parent / 'Original', output)
sides, copies = {}, []
for side, path in [('original', original), ('port', port)]:
if path:
separated_roots(Path(path).resolve().parent, output)
sides[side], files = _side(path, side)
copies += files
present = [s for s in sides.values() if 'case_id' in s]
if len({(s['case_id'], s['scope']) for s in present}) > 1:
raise ValueError('Paired receipts must describe the same case and scope')
ledger_model, source_copies = _ledger(ledger)
copies += source_copies
model = {'schema_version': 1, 'case_id': present[0]['case_id'] if present else 'empty',
'sides': sides, 'events': _events(sides), 'ledger': ledger_model,
'acceptance': {'state': 'open', 'remaining': ['Actual identified runtime and audiovisual review required']}}
encoded = canonical_bytes(model)
if len(encoded) > MAX_JSON_BYTES:
raise ValueError('Review JSON exceeds bound')
output.parent.mkdir(parents=True, exist_ok=True)
temporary = Path(tempfile.mkdtemp(prefix='.' + output.name + '-incomplete-', dir=output.parent))
# Failure retains the temporary tree for diagnosis; never partially publish.
(temporary / 'incomplete.json').write_text('{"status":"incomplete"}\n')
for source, relative in copies:
target = temporary / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
if identify_file(source) != identify_file(target):
raise ValueError('Artifact changed during review copy')
for name in STATIC:
shutil.copyfile(VIEWER / name, temporary / name)
(temporary / 'review.json').write_bytes(encoded)
files = list(STATIC) + ['review.json'] + [name for _, name in copies]
manifest = {'schema_version': 1, 'tool': 'fidelity_review', 'files':
[dict(path=name, **identify_file(temporary / name)) for name in files]}
(temporary / '.review-root.json').write_bytes(canonical_bytes(manifest))
(temporary / 'incomplete.json').unlink()
# Recheck the receipt/artifact copies with the shared validator at publication.
for side in ('original', 'port'):
receipt = temporary / 'sides' / side / 'receipt.json'
if receipt.exists():
validate_receipt(read_receipt(receipt), receipt.parent)
if output.exists():
raise ValueError('Review output allocated by another writer')
os.rename(temporary, output)
return model
class ReviewHandler(BaseHTTPRequestHandler):
"""No directory listing, upload, repository root, symlink or remote access."""
def log_message(self, *_args):
pass
def end_headers(self):
self.send_header('X-Content-Type-Options', 'nosniff')
self.send_header('Content-Security-Policy', "default-src 'none'; script-src 'self'; style-src 'self'; "
"media-src 'self'; img-src 'self'; connect-src 'self'; base-uri 'none'; "
"frame-ancestors 'none'; form-action 'none'; object-src 'none'")
self.send_header('Referrer-Policy', 'no-referrer')
self.send_header('Cache-Control', 'no-store')
super().end_headers()
def do_GET(self):
self._read(False)
def do_HEAD(self):
self._read(True)
def _reject(self):
self.send_response(405)
self.send_header('Allow', 'GET, HEAD')
self.send_header('Content-Length', '0')
self.end_headers()
do_POST = do_PUT = do_DELETE = do_PATCH = do_OPTIONS = do_TRACE = do_CONNECT = _reject
def _read(self, head):
try:
parsed = urlsplit(self.path)
if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment:
raise ValueError('Only generated local paths are readable')
name = unquote(parsed.path, errors='strict')
name = 'index.html' if name == '/' else name.removeprefix('/')
path = contained_path(self.server.root, name)
if name not in self.server.allowed or not path.is_file():
self.send_error(404)
return
size = path.stat().st_size
start, end, status = 0, size - 1, 200
range_value = self.headers.get('Range')
if range_value:
match = re.fullmatch(r'bytes=([0-9]{0,20})-([0-9]{0,20})', range_value)
if not match or not any(match.groups()) or size == 0:
raise IndexError('Invalid range')
left, right = match.groups()
if left:
start = int(left)
end = min(int(right), size - 1) if right else size - 1
else:
suffix = int(right)
if suffix <= 0:
raise IndexError('Invalid suffix')
start = max(0, size - suffix)
if start >= size or start > end:
raise IndexError('Unsatisfiable range')
end = min(end, start + MAX_RANGE - 1)
status = 206
self.send_response(status)
mime = 'text/javascript' if name.endswith('.mjs') else mimetypes.guess_type(name)[0]
mime = mime or 'application/octet-stream'
if name.startswith('sources/'):
mime = 'text/plain; charset=utf-8'
# Imported HTML/SVG/scripts never become active same-origin content.
if name not in STATIC and not name.startswith('sources/') and not (mime.startswith(('video/', 'audio/')) or
mime in ('image/png', 'image/jpeg', 'image/webp', 'application/json')):
mime = 'application/octet-stream'
self.send_header('Content-Disposition', 'attachment')
self.send_header('Content-Type', mime)
self.send_header('Accept-Ranges', 'bytes')
self.send_header('Content-Length', str(max(0, end - start + 1)))
if status == 206:
self.send_header('Content-Range', f'bytes {start}-{end}/{size}')
self.end_headers()
if not head:
with path.open('rb') as stream:
stream.seek(start)
remaining = end - start + 1
while remaining > 0:
block = stream.read(min(65536, remaining))
if not block:
break
self.wfile.write(block)
remaining -= len(block)
except IndexError:
self.send_response(416)
self.send_header('Content-Range', f'bytes */{size}')
self.send_header('Content-Length', '0')
self.end_headers()
except (BrokenPipeError, ConnectionResetError):
# Browsers cancel range reads while seeking; the response has already
# started. A second error response would itself write to a dead peer.
return
except (ValueError, UnicodeError, OSError):
self.send_error(403)
def make_server(root, port=8765, host='127.0.0.1'):
if host != '127.0.0.1':
raise ValueError('Viewer binds literal IPv4 loopback only')
root = Path(root)
if root.is_symlink():
raise ValueError('Review root must not be a symlink')
root = root.resolve(strict=True)
separated_roots(PROJECT.parent / 'Original', root)
manifest = read_json(contained_path(root, '.review-root.json'))
if manifest.get('tool') != 'fidelity_review' or manifest.get('schema_version') != 1:
raise ValueError('Only generated review trees can be served')
allowed = set()
for item in manifest.get('files', []):
path = contained_path(root, item['path'])
actual = identify_file(path)
if item['sha256'] != actual['sha256'] or item['size'] != actual['size']:
raise ValueError('Generated review changed; export again')
allowed.add(item['path'])
if not set(STATIC + ('review.json',)).issubset(allowed):
raise ValueError('Incomplete generated review')
server = ThreadingHTTPServer((host, port), ReviewHandler)
server.daemon_threads = True
server.root, server.allowed = root, allowed
return server
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
export = commands.add_parser('export')
export.add_argument('--original', type=Path)
export.add_argument('--port', type=Path)
export.add_argument('--output', type=Path, required=True)
export.add_argument('--ledger', type=Path)
serve = commands.add_parser('serve')
serve.add_argument('--root', type=Path, required=True)
serve.add_argument('--port', type=int, default=8765)
args = parser.parse_args()
try:
if args.command == 'export':
model = export_review(args.original, args.port, args.output, args.ledger)
print(json.dumps({'path': str(args.output / 'index.html'),
'events': len(model['events']), 'acceptance': 'OPEN'}))
else:
with make_server(args.root, args.port) as server:
print(f'Local review: http://127.0.0.1:{server.server_port}/ — acceptance OPEN', flush=True)
server.serve_forever()
except (ValueError, OSError, subprocess.SubprocessError) as error:
parser.exit(2, 'Review error: ' + str(error) + '\n')
except KeyboardInterrupt:
parser.exit(0)
if __name__ == '__main__':
main()