Map all 33 requirements across six groups and reconcile 33 hashed source rows. Preserve startup current/history, unimplemented work and open native/reference acceptance; bind imported receipt identities and generate navigation.
340 lines
18 KiB
Python
340 lines
18 KiB
Python
import copy
|
|
import importlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tools/recon'))
|
|
TOOL = ROOT / 'tools/recon/fidelity_ledger.py'
|
|
LEDGER = ROOT / 'docs/evidence/capabilities.json'
|
|
|
|
|
|
def evidence(identifier='proof-a', **changes):
|
|
value = {
|
|
'id': identifier, 'receipt_id': identifier, 'evidence_kind': 'port_fixture',
|
|
'method': 'bounded fixture', 'revision': 'a' * 40, 'historical': True,
|
|
'content': {'runtime_sha256': '1' * 64, 'resources_sha256': '2' * 64},
|
|
'platform': {'os': 'linux', 'execution': 'native'},
|
|
'scope': ['Carl first neutral input'], 'status': 'current',
|
|
'claim': {'key': 'neutral-return', 'result': 'upright'},
|
|
'supersedes': [], 'reason': None, 'artifacts': [],
|
|
'missing_evidence': ['Original and native Windows comparison absent'],
|
|
'report_path': 'docs/gameplay/main-menu-player-lifetime.md',
|
|
}
|
|
value.update(changes)
|
|
return value
|
|
|
|
|
|
def target(record=None):
|
|
row = record or evidence()
|
|
return {key: copy.deepcopy(row[key]) for key in ('content', 'platform', 'scope')}
|
|
|
|
|
|
def capability(identifier='startup-order'):
|
|
row = evidence()
|
|
return {
|
|
'id': identifier, 'group': 'movement', 'requirement_ids': ['MOVE-04'],
|
|
'summary': 'TSK story entry preserves retained Jimmy-before-input order.',
|
|
'verified_scope': 'Historical Linux fixture; direct fresh scene remains separate.',
|
|
'differences': ['Whole-original audiovisual comparison remains absent.'],
|
|
'acceptance': {'status': 'open', 'remaining': ['Original and native Windows comparison']},
|
|
'source_links': [{'label': 'Source lifetime', 'path': row['report_path']}],
|
|
'scenario_ids': ['fresh-natural-ordinary'], 'implementation': {'revision': 'a' * 40,
|
|
'paths': ['src/GamePort/Program.cs'], 'status': 'bounded'},
|
|
'targets': [dict(id='historical-linux', **target(row))],
|
|
'evidence': [row], 'discrepancies': [{'id': 'reference-gap',
|
|
'impact': 'missing_verification', 'summary': 'Missing whole-original recording'}],
|
|
}
|
|
|
|
|
|
def ledger(*rows):
|
|
return {'schema_version': 1, 'capabilities': list(rows)}
|
|
|
|
|
|
class LedgerTests(unittest.TestCase):
|
|
def api(self):
|
|
self.assertTrue(TOOL.is_file(), 'Capability ledger CLI must exist')
|
|
return importlib.import_module('fidelity_ledger')
|
|
|
|
def test_real_startup_capability_cli_and_viewer_roundtrip(self):
|
|
self.assertTrue(TOOL.is_file(), 'Capability ledger CLI must exist')
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
exported = Path(temporary) / 'ledger.json'
|
|
result = subprocess.run([sys.executable, str(TOOL), 'export', '--ledger', str(LEDGER),
|
|
'--output', str(exported)], text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
model = json.loads(exported.read_text())
|
|
row = next(row for row in model['capabilities'] if row['id'] == 'startup-order')
|
|
self.assertIn('already corrected', row['summary'])
|
|
self.assertEqual(row['acceptance']['status'], 'open')
|
|
self.assertTrue(row['current'])
|
|
self.assertTrue(row['history'])
|
|
from fidelity_review import export_review
|
|
reviewed = export_review(None, None, Path(temporary) / 'review', exported)
|
|
loaded = json.loads((Path(temporary) / 'review/review.json').read_text())
|
|
self.assertEqual(reviewed, loaded)
|
|
nav = next(row for row in loaded['ledger']['capabilities'] if row['id'] == 'startup-order')
|
|
self.assertEqual(nav['current'], row['current'])
|
|
self.assertEqual(nav['history'], row['history'])
|
|
self.assertTrue(all(link['validated_local'] for link in nav['source_links']))
|
|
for link in nav['source_links']:
|
|
self.assertTrue((Path(temporary) / 'review' / link['href']).is_file())
|
|
self.assertIn('native Windows', str(nav['acceptance']))
|
|
self.assertEqual(loaded['acceptance']['state'], 'open')
|
|
self.assertEqual(len(loaded['ledger']['capabilities']), len(model['capabilities']))
|
|
for exported_row in model['capabilities']:
|
|
viewed = next(r for r in loaded['ledger']['capabilities'] if r['id'] == exported_row['id'])
|
|
for key in ('current', 'history', 'scenario_ids', 'acceptance', 'discrepancies'):
|
|
self.assertEqual(viewed[key], exported_row[key])
|
|
self.assertTrue(all(link['validated_local'] for link in viewed['source_links']))
|
|
self.assertTrue(all((Path(temporary) / 'review' / link['href']).is_file()
|
|
for link in viewed['source_links']))
|
|
|
|
def test_empty_ledger_fails(self):
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(), ROOT)
|
|
|
|
def test_single_capability_renders_with_open_acceptance(self):
|
|
api = self.api()
|
|
text = api.render_markdown(ledger(capability()), ROOT)
|
|
self.assertIn('TSK story entry', text)
|
|
self.assertIn('OPEN', text)
|
|
self.assertIn('History', text)
|
|
|
|
def test_duplicate_capability_conflict_rejected(self):
|
|
row = capability()
|
|
other = copy.deepcopy(row)
|
|
other['summary'] = 'contradictory'
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row, other), ROOT)
|
|
|
|
def test_equal_record_ids_merge_only_equal_records(self):
|
|
row = capability()
|
|
row['evidence'].append(copy.deepcopy(row['evidence'][0]))
|
|
model = self.api().export_ledger(ledger(row), ROOT)
|
|
self.assertEqual(len(model['capabilities'][0]['evidence']), 1)
|
|
row['evidence'][-1]['method'] = 'changed bytes'
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_conflicting_shared_receipt_identity_rejected(self):
|
|
row = capability()
|
|
row['evidence'].append(evidence('other-id', receipt_id='proof-a', method='different'))
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_equal_priority_has_stable_id_order(self):
|
|
api = self.api()
|
|
a, b = capability('zeta'), capability('alpha')
|
|
self.assertEqual([r['id'] for r in api.export_ledger(ledger(a, b), ROOT)['capabilities']],
|
|
['alpha', 'zeta'])
|
|
|
|
def test_supersession_cycle_rejected(self):
|
|
row = capability()
|
|
row['evidence'] = [evidence('a', supersedes=['b'], reason='replace'),
|
|
evidence('b', supersedes=['a'], reason='replace')]
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_unresolved_contradictory_current_claims_fail(self):
|
|
row = capability()
|
|
row['evidence'].append(evidence('other', claim={'key': 'neutral-return', 'result': 'tilted'}))
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_explicit_supersession_retains_prior_claim(self):
|
|
row = capability()
|
|
row['evidence'][0].update(status='superseded')
|
|
row['evidence'].append(evidence('new', supersedes=['proof-a'], reason='Source correction',
|
|
claim={'key': 'neutral-return', 'result': 'corrected'}))
|
|
model = self.api().export_ledger(ledger(row), ROOT)['capabilities'][0]
|
|
self.assertEqual([r['id'] for r in model['current']], ['new'])
|
|
self.assertEqual([r['id'] for r in model['history']], ['proof-a'])
|
|
|
|
def test_unsafe_dead_and_tampered_source_links_fail(self):
|
|
api = self.api()
|
|
for path in ('../Original/Neutron.exe', '/etc/passwd', 'C:/secret',
|
|
'artifacts/private.json', 'docs/nonexistent-ledger-source.md'):
|
|
with self.subTest(path=path):
|
|
row = capability()
|
|
row['source_links'][0]['path'] = path
|
|
with self.assertRaises(ValueError): api.validate_ledger(ledger(row), ROOT)
|
|
row = capability()
|
|
row['source_links'][0]['sha256'] = '0' * 64
|
|
with self.assertRaises(ValueError): api.validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_docs_revision_does_not_change_content_applicability(self):
|
|
api = self.api()
|
|
row = evidence()
|
|
current = target(row)
|
|
current['source_revision'] = 'b' * 40
|
|
self.assertEqual(api.select_current_evidence([row], current), [row])
|
|
|
|
def test_changed_package_or_resources_is_not_applicable(self):
|
|
api = self.api()
|
|
for key in ('runtime_sha256', 'resources_sha256', 'package_sha256'):
|
|
row = evidence()
|
|
changed = target(row)
|
|
changed['content'][key] = '3' * 64
|
|
self.assertEqual(api.select_current_evidence([row], changed), [])
|
|
|
|
def test_mismatched_platform_or_scope_is_not_applicable(self):
|
|
api = self.api()
|
|
for key, value in (('platform', {'os': 'windows', 'execution': 'native'}),
|
|
('scope', ['whole campaign'])):
|
|
changed = target()
|
|
changed[key] = value
|
|
self.assertEqual(api.select_current_evidence([evidence()], changed), [])
|
|
|
|
def test_missing_artifact_is_explicit_and_never_accepted(self):
|
|
row = capability()
|
|
row['evidence'][0]['artifacts'] = [{'path': 'artifacts/fidelity/never-captured.webm',
|
|
'missing_reason': 'No recording was made'}]
|
|
exported = self.api().export_ledger(ledger(row), ROOT)['capabilities'][0]
|
|
self.assertEqual(exported['acceptance']['status'], 'open')
|
|
self.assertIn('No recording', str(exported['evidence']))
|
|
row['acceptance']['status'] = 'accepted'
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_invalid_receipt_classification_rejected(self):
|
|
row = capability()
|
|
row['evidence'][0]['evidence_kind'] = 'native_port'
|
|
row['evidence'][0]['platform'] = {'os': 'windows', 'execution': 'wine'}
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_nonfinite_or_oversized_input_rejected(self):
|
|
row = capability()
|
|
row['evidence'][0]['elapsed'] = float('nan')
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
row = capability()
|
|
row['summary'] = 'x' * 20000
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_duplicate_history_is_not_silently_overwritten(self):
|
|
row = capability()
|
|
row['evidence'] = [evidence('old', status='historical'), evidence('old', status='superseded')]
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_full_game_has_33_requirements_and_six_groups(self):
|
|
api = self.api()
|
|
model = json.loads(LEDGER.read_text())
|
|
required = set(__import__('re').findall(r'^- \[[ x]\] \*\*([A-Z]+-[0-9]{2})\*\*:',
|
|
(ROOT / '.planning/REQUIREMENTS.md').read_text(), __import__('re').M))
|
|
mapped = {req for row in model['capabilities'] for req in row['requirement_ids']}
|
|
self.assertEqual(mapped, required, 'Every full-game requirement needs capability coverage')
|
|
counts = api.validate_ledger(model, ROOT, ROOT / '.planning/REQUIREMENTS.md')
|
|
self.assertEqual((counts['requirements'], counts['unmapped'], counts['groups']), (33, 0, 6))
|
|
|
|
def test_unmapped_requirements_fail_full_validation(self):
|
|
with self.assertRaises(ValueError):
|
|
self.api().validate_ledger(ledger(capability()), ROOT, ROOT / '.planning/REQUIREMENTS.md')
|
|
|
|
def test_unimplemented_required_rows_remain_visible_and_open(self):
|
|
model = json.loads(LEDGER.read_text())
|
|
rows = [row for row in model['capabilities'] if row['implementation']['status'] == 'unimplemented']
|
|
self.assertTrue(rows, 'Unimplemented full-game work must be visible')
|
|
self.assertTrue(all(row['acceptance']['status'] == 'open' for row in rows))
|
|
self.assertIn('CAMP-03', {req for row in rows for req in row['requirement_ids']})
|
|
|
|
def test_checkpoint_source_rows_are_exhaustively_reconciled(self):
|
|
api = self.api()
|
|
data = json.loads(LEDGER.read_text())
|
|
self.assertIn('reconciliation', data, 'Checkpoint source-row accounting must be explicit')
|
|
expected = api.checkpoint_rows(ROOT / 'docs/checkpoint/coverage.md')
|
|
self.assertGreaterEqual(len(expected), 32)
|
|
self.assertEqual({r['source_row']: r['source_sha256'] for r in data['reconciliation']}, expected)
|
|
data['reconciliation'].pop()
|
|
with self.assertRaises(ValueError): api.validate_ledger(data, ROOT, ROOT / '.planning/REQUIREMENTS.md')
|
|
|
|
def test_discrepancy_impact_and_stable_ties(self):
|
|
api = self.api()
|
|
row = capability()
|
|
row['discrepancies'] = [{'id': name, 'impact': impact, 'summary': name}
|
|
for name, impact in [('z', 'missing_verification'), ('b', 'blocking'),
|
|
('a', 'blocking'), ('d', 'subtle'), ('c', 'disruptive')]]
|
|
actual = api.export_ledger(ledger(row), ROOT)['capabilities'][0]['discrepancies']
|
|
self.assertEqual([r['id'] for r in actual], ['a', 'b', 'c', 'd', 'z'])
|
|
|
|
def test_unknown_scenario_and_missing_critical_fields_fail(self):
|
|
api = self.api()
|
|
row = capability()
|
|
row['scenario_ids'] = ['invented-scenario']
|
|
with self.assertRaises(ValueError): api.validate_ledger(ledger(row), ROOT)
|
|
for key in ('summary', 'verified_scope', 'implementation', 'acceptance', 'source_links'):
|
|
broken = capability()
|
|
del broken[key]
|
|
with self.assertRaises(ValueError): api.validate_ledger(ledger(broken), ROOT)
|
|
|
|
def test_generated_readme_exactly_matches_authoritative_model(self):
|
|
api = self.api()
|
|
self.assertEqual((ROOT / 'docs/evidence/README.md').read_text(),
|
|
api.render_markdown(json.loads(LEDGER.read_text()), ROOT))
|
|
|
|
def test_local_artifact_tamper_fails_validation(self):
|
|
api = self.api()
|
|
with tempfile.TemporaryDirectory(dir=ROOT / 'artifacts') as temporary:
|
|
artifact = Path(temporary) / 'proof.log'
|
|
artifact.write_text('original fixture log')
|
|
import hashlib
|
|
row = capability()
|
|
row['evidence'][0]['artifacts'] = [{'path': artifact.relative_to(ROOT).as_posix(),
|
|
'sha256': hashlib.sha256(artifact.read_bytes()).hexdigest(), 'size': artifact.stat().st_size}]
|
|
api.validate_ledger(ledger(row), ROOT)
|
|
artifact.write_text('tampered fixture log')
|
|
with self.assertRaises(ValueError): api.validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_receipt_identity_conflict_across_capabilities_rejected(self):
|
|
a, b = capability('alpha'), capability('beta')
|
|
self.api().validate_ledger(ledger(a, b), ROOT)
|
|
b['evidence'][0]['method'] = 'Conflicting bytes under a shared receipt ID'
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(a, b), ROOT)
|
|
|
|
def test_component_and_inferred_categories_cannot_claim_native_execution(self):
|
|
for kind in ('component', 'managed', 'synthetic', 'inferred'):
|
|
with self.subTest(kind=kind):
|
|
row = capability()
|
|
row['evidence'][0]['evidence_kind'] = kind
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(row), ROOT)
|
|
|
|
def test_reconciliation_requires_actual_retained_source_link(self):
|
|
model = json.loads(LEDGER.read_text())
|
|
identifier = model['reconciliation'][0]['capability_ids'][0]
|
|
row = next(row for row in model['capabilities'] if row['id'] == identifier)
|
|
row['source_links'] = [link for link in row['source_links'] if link['path'] != 'docs/checkpoint/coverage.md']
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(model, ROOT)
|
|
|
|
def test_cli_full_model_rejects_unmapped_requirement_without_optional_flag(self):
|
|
model = json.loads(LEDGER.read_text())
|
|
for row in model['capabilities']:
|
|
row['requirement_ids'] = [req for req in row['requirement_ids'] if req != 'EVID-03']
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
path = Path(temporary) / 'broken.json'
|
|
path.write_text(json.dumps(model))
|
|
result = subprocess.run([sys.executable, str(TOOL), 'validate', '--ledger', str(path)],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn('Unmapped or unknown requirements', result.stderr)
|
|
|
|
def test_real_imported_receipt_reference_binds_bytes_scope_and_platform(self):
|
|
from test_fidelity_review import fixture
|
|
from fidelity_receipts import identify_file
|
|
with tempfile.TemporaryDirectory(dir=ROOT / 'artifacts') as temporary:
|
|
receipt_path, receipt = fixture(Path(temporary), 'ledger')
|
|
row = capability()
|
|
record = row['evidence'][0]
|
|
record.update(receipt_id=receipt['receipt_id'], evidence_kind='synthetic', historical=False,
|
|
platform=receipt['platform'], scope=[receipt['scope']],
|
|
receipt_path=receipt_path.relative_to(ROOT).as_posix(),
|
|
content={'receipt_sha256': identify_file(receipt_path)['sha256']})
|
|
row['targets'] = [dict(id='imported-test', **target(record))]
|
|
self.api().validate_ledger(ledger(row), ROOT)
|
|
for key, value in [('scope', ['wrong scope']),
|
|
('platform', {'os':'wrong host','execution':'synthetic'}),
|
|
('content', {'receipt_sha256':'0'*64})]:
|
|
broken = copy.deepcopy(row)
|
|
broken['evidence'][0][key] = value
|
|
broken['targets'] = [dict(id='imported-test', **target(broken['evidence'][0]))]
|
|
with self.assertRaises(ValueError): self.api().validate_ledger(ledger(broken), ROOT)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|