Files
Neutron/tools/recon/fidelity_checkpoint.py
T
UnixNight aa4b862a12 fix(01-05): retain executable permissions in offline copies
Preserve source read/write/execute bits without privileged mode bits. Verify copied Linux apphost byte identity and permissions.
2026-11-13 11:43:51 -05:00

329 lines
18 KiB
Python

#!/usr/bin/env python3
"""Prepare immutable, local-only fidelity checkpoints. READY is never acceptance.
The Windows collector needs no Python or SDK. This module assembles/transfers
identities and imports explicitly synthetic fixture results; it never executes
an original. Cooperative locks/incomplete siblings are retained after failure.
"""
import argparse
import hashlib
import os
from pathlib import Path
import shutil
import subprocess
import sys
import uuid
from fidelity_receipts import (canonical_bytes, contained_path, identify_file,
identify_tree, read_json, relative_name, separated_roots, source_identity,
utc_now, validate_receipt, validate_scenario_contract, verify_inventory,
_id, _write_json)
from fidelity_review import export_review
PROJECT = Path(__file__).resolve().parents[2]
RIDS = ('linux-x64', 'win-x64')
FLAGS = ['--fidelity-output', '--fidelity-session', '--fidelity-case']
MAX_FILE_BYTES = 8 * 1024**3
MUTABLE = ('results', 'original-runs')
MATERIALS = ('scripts/collect-fidelity-windows.ps1',
'scripts/collect-fidelity-windows.cmd',
'tests/fidelity-windows.Tests.ps1', 'docs/evidence/windows-session.md')
def _copy_file(source, target):
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
if identify_file(source) != identify_file(target):
raise ValueError('Source changed during copy: ' + str(source))
# Byte-identical Unix apphosts still need their source execution permission.
# Do not propagate setuid/setgid/sticky bits into a generated checkpoint.
target.chmod(Path(source).stat().st_mode & 0o777)
def _copy_tree(source, target):
tree = identify_tree(source)
target.mkdir(parents=True)
for entry in tree['files']:
if entry['size'] > MAX_FILE_BYTES:
raise ValueError('File exceeds 8 GiB bound: ' + entry['path'])
_copy_file(contained_path(source, entry['path']), target / entry['path'])
if identify_tree(target)['files'] != tree['files']:
raise ValueError('Copied tree differs: ' + str(source))
def _package_files(package):
return [f for f in identify_tree(package)['files'] if f['path'] != 'fidelity-build.json']
def seal_package(package, rid, source, tooling=False, build_argv=None):
"""Called immediately after the local publish by our wrapper, never on old DLLs.
Build source is captured BEFORE publishing and checked again afterward. This
record binds those claims to bytes, without pretending the hash proves their
compiler provenance. A synthetic seal is permanently tooling-only.
"""
package = Path(package)
if rid not in RIDS:
raise ValueError('Unsupported package RID')
required = ['GamePort.dll', 'GamePort.runtimeconfig.json', 'GamePort.deps.json',
'BUILD-INFO.txt', 'GamePort.exe' if rid == 'win-x64' else 'GamePort',
'SDL3.dll' if rid == 'win-x64' else 'libSDL3.so.0',
'assets/NewGame.tsk', 'scenes/level1b/scene.json', 'scenes/VR04/scene.json',
'scenes/level1/scene.json', 'scenes/animations/jimstop/animation.json']
for name in required:
identify_file(contained_path(package, name))
for name in ('assets', 'scenes', 'audio', 'Shaders'):
identify_tree(package / name)
if not tooling:
dll = (package / 'GamePort.dll').read_bytes()
if not all(flag.encode('utf-16le') in dll for flag in FLAGS):
raise ValueError('Stale DLL: observer flags absent')
if not build_argv or not source.get('commit') or not source.get('diff_sha256'):
raise ValueError('Missing actual pre-publish source/command identity')
record = {'schema_version': 1, 'rid': rid, 'source': source,
'built_utc': utc_now(), 'tooling_only': tooling,
'build_argv': build_argv or ['synthetic fixture, no compiler'],
'observer_flags': FLAGS, 'files': _package_files(package)}
_write_json(package / 'fidelity-build.json', record)
return record
def _package(package, rid, tooling):
record = read_json(package / 'fidelity-build.json')
if (record.get('schema_version') != 1 or record.get('rid') != rid or
record.get('observer_flags') != FLAGS or
record.get('files') != _package_files(package)):
raise ValueError('Stale/missing package bytes or observer identity: ' + rid)
if record.get('tooling_only') and not tooling:
raise ValueError('Synthetic package cannot qualify as runtime materials')
return {'build_info': identify_file(package / 'BUILD-INFO.txt'),
'build': {k: v for k, v in record.items() if k != 'files'},
'package_tree_sha256': hashlib.sha256(canonical_bytes(record['files'])).hexdigest(),
'dll': identify_file(package / 'GamePort.dll'),
'executable': identify_file(package / ('GamePort.exe' if rid == 'win-x64' else 'GamePort')),
'resources': {k: {'tree_sha256': identify_tree(package / k)['tree_sha256']}
for k in ('assets', 'scenes', 'audio', 'Shaders')}}
def prepare_checkpoint(original, original_manifest, packages, output, tooling=False):
unresolved = Path(output).absolute()
if any(p.is_symlink() for p in (unresolved, *unresolved.parents)):
raise ValueError('Checkpoint output cannot use symlink aliases')
original, output = separated_roots(original, output)
# Resolve aliases before allocating; reject symlink ancestry even for missing leaves.
candidate = Path(output)
if any(p.is_symlink() for p in (candidate, *candidate.parents)) or candidate.exists():
raise ValueError('Checkpoint output must be a fresh non-symlink directory')
separated_roots(PROJECT.parent / 'Original', output)
if set(packages) != set(RIDS):
raise ValueError('Both identified offline packages are required')
before = verify_inventory(original, original_manifest)
protocol = read_json(PROJECT / 'docs/evidence/scenarios.json')
validate_scenario_contract(protocol)
identities = {}
for rid in RIDS:
package = Path(packages[rid]).resolve(strict=True)
separated_roots(package, output)
identities[rid] = _package(package, rid, tooling)
# Every raw asset shipped by the package must match the original baseline.
if not tooling:
baseline = {f['path'].casefold(): f for f in before['files']}
for f in identify_tree(package / 'assets')['files']:
expected = baseline.get(f['path'].casefold())
if not expected or any(f[k] != expected[k] for k in ('sha256', 'size', 'header')):
raise ValueError('Package raw resource differs from original: ' + f['path'])
output.parent.mkdir(parents=True, exist_ok=True)
lock = output.with_name('.' + output.name.casefold() + '.lock')
try:
lock.mkdir()
except FileExistsError as error:
raise ValueError('Checkpoint reserved; inspect incomplete assembly, never steal lock') from error
temporary = output.with_name('.' + output.name + '.incomplete-' + uuid.uuid4().hex)
temporary.mkdir()
_write_json(temporary / 'incomplete.json', {'status': 'incomplete', 'started_utc': utc_now()})
# On failure retain both the diagnostic sibling and reservation.
_copy_tree(original, temporary / 'original-pristine')
for rid in RIDS:
_copy_tree(packages[rid], temporary / 'port' / rid)
for name in MATERIALS:
if (PROJECT / name).is_file():
_copy_file(PROJECT / name, temporary / name)
else:
raise ValueError('Missing collector material: ' + name)
_copy_file(original_manifest, temporary / 'protocol/original-manifest.json')
for name in ('scenarios.json', 'capabilities.json'):
_copy_file(PROJECT / 'docs/evidence' / name, temporary / 'protocol' / name)
export_review(None, None, temporary / 'review', PROJECT / 'docs/evidence/capabilities.json')
for name in MUTABLE:
(temporary / name).mkdir()
# All 80 Windows obligations are prepared; Linux's remaining40 use the same protocol.
agenda = [{'side': side, 'case_id': cell['id'], 'obligation_id': obligation,
'status': 'pending'} for side in ('original-windows-dxwnd', 'port-windows-native')
for cell in protocol['cells'] for obligation in cell['obligations']]
_write_json(temporary / 'protocol/agenda.json', {'schema_version': 1,
'runs': agenda, 'acceptance': {'state': 'open', 'required_all_platforms': 120}})
_write_json(temporary / 'protocol/settings-template.json', {
'schema_version': 1, 'diagnostic': True,
'tools': {k: {'source': '', 'version': '', 'settings': ''}
for k in ('dxwnd', 'recorder', 'browser', 'media_probe', 'derivation')},
'capture': {'source': '', 'monitor': '', 'crop': [], 'aspect': '',
'resolution': [], 'frame_rate': '', 'audio_devices': [],
'encoder': '', 'container': 'MKV', 'timing_limitations': ''},
'preflight': {'original': {'visible_moving_game': False, 'intelligible_game_audio': False,
'correct_crop_aspect': False},
'port': {'visible_moving_game': False, 'intelligible_game_audio': False,
'correct_crop_aspect': False},
'browser': {'actual_playback': False, 'seeking': False,
'exclusive_audio': False}},
'notes': 'Fill actual observed values; false/empty fields remain pending, never acceptance.'})
_write_json(temporary / 'protocol/annotation-template.json', {
'schema_version': 1, 'status': 'incomplete', 'diagnostic': True,
'artifacts': [], 'events': [], 'derivatives': [], 'continuity': {},
'limitations': ['Actual media clocks, uncertainty and human review still required'],
'notes': 'Use windows-session.md field contract; empty template cannot finalize capture.'})
verify_inventory(original, original_manifest)
(temporary / 'incomplete.json').unlink()
files = identify_tree(temporary)['files']
manifest = {'schema_version': 1, 'checkpoint_id': output.name,
'created_utc': utc_now(), 'tooling_only': tooling, 'packages': identities,
'original_manifest': identify_file(original_manifest),
'original_pre': {'tree_sha256': before['tree_sha256'], 'count': len(before['files'])},
'files': files, 'mutable_roots': list(MUTABLE),
'acceptance': {'state': 'open', 'remaining': ['Actual original/native captures and human review']}}
_write_json(temporary / 'manifest.json', manifest)
_write_json(temporary / 'READY.json', {'schema_version': 1, 'status': 'prepared',
'manifest': dict(path='manifest.json', **identify_file(temporary / 'manifest.json')),
'acceptance': {'state': 'open'}, 'meaning': 'Materials prepared only; no runtime acceptance'})
preflight(temporary)
if output.exists():
raise ValueError('Output allocated concurrently')
os.rename(temporary, output)
lock.rmdir()
return manifest
def preflight(root, dependencies=None):
root = Path(root)
if root.is_symlink():
raise ValueError('Checkpoint root cannot be a symlink')
ready = read_json(root / 'READY.json')
if ready.get('schema_version') != 1 or ready.get('status') != 'prepared':
raise ValueError('Checkpoint is incomplete, not READY')
actual = identify_file(root / 'manifest.json')
if any(ready['manifest'].get(k) != actual[k] for k in ('sha256', 'size')):
raise ValueError('Manifest identity differs from READY')
manifest = read_json(root / 'manifest.json')
expected = {item['path']: item for item in manifest['files']}
for name, item in expected.items():
actual = identify_file(contained_path(root, name))
if any(item[k] != actual[k] for k in ('sha256', 'size', 'header')):
raise ValueError('Tampered checkpoint material: ' + name)
# Permit only owned result/copy directories to grow after publication.
for directory, dirs, names in os.walk(root, followlinks=False):
for name in dirs + names:
path = Path(directory) / name
relative = path.relative_to(root).as_posix()
if path.is_symlink():
raise ValueError('Symlink in checkpoint: ' + relative)
if path.is_file() and relative.split('/')[0] not in MUTABLE:
if relative not in expected and relative not in ('READY.json', 'manifest.json'):
raise ValueError('Unexpected immutable material: ' + relative)
failures = []
tools = {}
for label, path in (dependencies or {}).items():
try:
tools[label] = dict(path=str(Path(path).resolve()), **identify_file(path))
except (ValueError, OSError):
failures.append('Missing local dependency ' + label + ': select an existing local file (' + str(path) + ')')
return {'schema_version': 1, 'status': 'blocked' if failures else 'ready',
'checked_files': len(expected), 'tooling_only': manifest['tooling_only'],
'tools': tools, 'failures': failures, 'acceptance': {'state': 'open'}}
def fixture_result(bundle, run, session_id, case_id, scope, artifacts, status='captured'):
"""Tooling-only end-to-end importer fixture; cannot mint native evidence."""
preflight(bundle)
if status not in ('captured', 'failed', 'skipped', 'aborted'):
raise ValueError('Unknown fixture result status')
_id(session_id)
protocol = read_json(Path(bundle) / 'protocol/scenarios.json')
if case_id not in {c['id'] for c in protocol['cells']} or scope not in protocol['obligations']:
raise ValueError('Unknown case/scope')
run = Path(run)
items = [dict(path=str(relative_name(p)), role='diagnostic', synthetic=True,
**identify_file(contained_path(run, p))) for p in artifacts]
receipt = {'schema_version': 1, 'receipt_id': session_id, 'session_id': session_id,
'case_id': case_id, 'obligation_id': scope, 'scope': scope,
'evidence_kind': 'synthetic', 'status': status, 'historical': False,
'platform': {'os': 'test', 'execution': 'synthetic'}, 'identity': {},
'invocation': {'argv': ['fixture result; no executable launched'], 'cwd': str(run)},
'clocks': {'wall_start': utc_now(), 'wall_end': utc_now(),
'monotonic_start': 0, 'monotonic_end': 0},
'artifacts': items, 'derivatives': [], 'events': [],
'limitations': ['Synthetic tooling only'], 'missing_evidence': ['No native/reference execution'],
'acceptance': {'state': 'open', 'remaining': ['Actual identified captures and human review']}}
validate_receipt(receipt, run)
_write_json(run / 'receipt.json', receipt)
return receipt
def publish_packages(output, base):
"""Isolated equivalent of existing builds, preserving historical dist packages."""
output, base = Path(output).resolve(), Path(base).resolve()
if output.exists():
raise ValueError('Publish root must be fresh')
separated_roots(base, output)
source = source_identity(PROJECT)
output.mkdir(parents=True)
for rid in RIDS:
target = output / rid
_copy_tree(base / rid, target)
# The old note identifies historical resource packaging, not this compilation.
(target / 'BUILD-INFO.txt').rename(target / 'RESOURCE-BASE-BUILD-INFO.txt')
old_seal = target / 'fidelity-build.json'
if old_seal.exists():
old_seal.unlink()
argv = ['dotnet', 'publish', 'GamePort.csproj', '-c', 'Release', '-r', rid,
'--self-contained', 'true', '--no-restore', '-o', str(target)]
subprocess.run(argv, cwd=PROJECT, check=True, timeout=600)
if source_identity(PROJECT) != source:
raise ValueError('Source changed during publish; preserve diagnostics and republish')
(target / 'BUILD-INFO.txt').write_text('source_commit=' + source['commit'] +
'\nsource_dirty=' + ('yes' if source['dirty'] else 'no') +
'\nsource_diff_sha256=' + source['diff_sha256'] + '\npublished_utc=' + utc_now() + '\n')
seal_package(target, rid, source, build_argv=argv)
return {'status': 'published', 'output': str(output), 'source': source}
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='command', required=True)
prepare = sub.add_parser('prepare')
prepare.add_argument('--original', type=Path, default=PROJECT.parent / 'Original')
prepare.add_argument('--original-manifest', type=Path, default=PROJECT / 'docs/original-manifest.json')
prepare.add_argument('--packages', type=Path, required=True)
prepare.add_argument('--output', type=Path, required=True)
check = sub.add_parser('preflight')
check.add_argument('--checkpoint', type=Path, required=True)
publish = sub.add_parser('publish')
publish.add_argument('--output', type=Path, required=True)
publish.add_argument('--base', type=Path, default=PROJECT / 'dist')
args = parser.parse_args()
try:
if args.command == 'prepare':
result = prepare_checkpoint(args.original, args.original_manifest,
{rid: args.packages / rid for rid in RIDS}, args.output)
print('Prepared checkpoint; acceptance OPEN; files:', len(result['files']))
elif args.command == 'publish':
print(canonical_bytes(publish_packages(args.output, args.base)).decode())
else:
print(canonical_bytes(preflight(args.checkpoint)).decode())
return 0
except (ValueError, OSError, subprocess.SubprocessError) as error:
print('Checkpoint failed (acceptance OPEN): ' + str(error), file=sys.stderr)
return 2
if __name__ == '__main__':
sys.exit(main())