296 lines
16 KiB
Python
296 lines
16 KiB
Python
"""Acceptance metadata doubles are synthetic tests, never runtime proof."""
|
|
import copy
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import test_fidelity_scenarios
|
|
import test_fidelity_checkpoint
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tools/recon'))
|
|
import fidelity_receipts as receipts
|
|
from fidelity_review import export_review
|
|
|
|
try:
|
|
acceptance = importlib.import_module('fidelity_acceptance')
|
|
except ModuleNotFoundError:
|
|
acceptance = None
|
|
|
|
|
|
class AcceptanceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.fixture = test_fidelity_scenarios.FidelityScenarios()
|
|
self.fixture.setUp()
|
|
self.addCleanup(self.fixture.doCleanups)
|
|
self.root = self.fixture.base
|
|
self.protocol = self.fixture.protocol()
|
|
self.record, self.record_root = self.fixture.receipt()
|
|
identity = self.record['identity']
|
|
self.package = {'executable': identity['executable'],
|
|
'build_info': identity['source'],
|
|
'resources': {'assets': {'tree_sha256': 'a' * 64}}}
|
|
self.package['build'] = {'source': {k: identity['source'][k]
|
|
for k in ('commit', 'dirty', 'diff_sha256')}}
|
|
# Bind real hashed metadata files in the test instead of accepting labels.
|
|
self.original_files = [dict(path='Neutron.exe', **receipts.identify_file(self.record_root / 'setup.txt'))]
|
|
self.original_inventory = {'files': self.original_files,
|
|
'tree_sha256': hashlib.sha256(receipts.canonical_bytes(self.original_files)).hexdigest()}
|
|
for key, data in [('original_pre', self.original_inventory),
|
|
('original_post', self.original_inventory),
|
|
('original-package', self.original_inventory),
|
|
('original-resources', self.original_inventory),
|
|
('package', {'source': self.package['build']['source']}),
|
|
('resources', self.package['resources'])]:
|
|
path = self.record_root / (key + '.json')
|
|
path.write_bytes(receipts.canonical_bytes(data))
|
|
identity[key] = dict(receipts.identify_file(path), path=path.name)
|
|
self.package['seal'] = identity['package']
|
|
self.manifest = {'tooling_only': False, 'packages': {
|
|
'linux-x64': self.package, 'win-x64': self.package},
|
|
'original_pre': {'tree_sha256': self.original_inventory['tree_sha256'], 'count': 1}}
|
|
self.review = {'schema_version': 1,
|
|
'checks': {key: {'observed': True, 'reviewer': 'metadata test',
|
|
'notes': 'Synthetic unit assertion only'} for key in
|
|
('A1', 'A2', 'EVID-02-unclassified', 'EVID-04-unclassified',
|
|
'P1', 'P2', 'P3', 'browser', 'paired_review', 'physical_input')},
|
|
'receipt_ids': [self.record['receipt_id']]}
|
|
|
|
def api(self):
|
|
self.assertIsNotNone(acceptance, 'strict readiness/acceptance API must exist')
|
|
return acceptance
|
|
|
|
def audit(self, records=None, review=None):
|
|
return self.api().audit_records(self.protocol, self.manifest,
|
|
[(self.record, self.record_root)] if records is None else records,
|
|
self.review if review is None else review, self.original_inventory)
|
|
|
|
def original_record(self):
|
|
value = copy.deepcopy(self.record)
|
|
value['evidence_kind'] = 'original_dxwnd'
|
|
value['platform'].update(os='windows', execution='dxwnd')
|
|
value['identity']['executable']['original_member'] = 'Neutron.exe'
|
|
for key in ('package', 'resources'):
|
|
value['identity'][key] = value['identity']['original-' + key]
|
|
return value
|
|
|
|
def test_empty_matrix_never_passes(self):
|
|
report = self.audit([])
|
|
self.assertEqual(report['acceptance'], 'human_needed')
|
|
self.assertEqual(len(report['matrix']['missing_obligations']), 120)
|
|
|
|
def test_identified_valid_cell_covers_only_one_slot(self):
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 1)
|
|
self.assertEqual(self.audit()['acceptance'], 'human_needed')
|
|
|
|
def test_complete_metadata_matrix_with_human_findings(self):
|
|
records = []
|
|
for cell in self.protocol['cells']:
|
|
for obligation in cell['obligations']:
|
|
for kind, os_name in [('native_port', 'linux'), ('native_port', 'windows'),
|
|
('original_dxwnd', 'windows')]:
|
|
value, root = self.fixture.receipt(cell['id'], obligation, kind, os_name)
|
|
value['identity'] = copy.deepcopy(self.record['identity'])
|
|
if kind == 'original_dxwnd':
|
|
value['identity'] = self.original_record()['identity']
|
|
for path in self.record_root.iterdir():
|
|
if path.is_file() and root != self.record_root:
|
|
(root / path.name).write_bytes(path.read_bytes())
|
|
records.append((value, root))
|
|
review = copy.deepcopy(self.review)
|
|
review['receipt_ids'] = [r['receipt_id'] for r, _ in records]
|
|
self.assertEqual(self.audit(records, review)['acceptance'], 'passed')
|
|
|
|
def test_original_executable_must_match_named_pristine_member(self):
|
|
value = self.original_record()
|
|
self.assertEqual(self.audit([(value, self.record_root)])['matrix']['accepted'], 1)
|
|
wrong = self.record_root / 'wrong.exe'
|
|
wrong.write_bytes(b'synthetic wrong original executable')
|
|
value['identity']['executable'] = dict(receipts.identify_file(wrong), path=wrong.name,
|
|
original_member='Neutron.exe')
|
|
self.assertEqual(self.audit([(value, self.record_root)])['matrix']['accepted'], 0)
|
|
value = self.original_record()
|
|
value['identity']['executable']['original_member'] = 'unselected.exe'
|
|
self.assertEqual(self.audit([(value, self.record_root)])['matrix']['accepted'], 0)
|
|
|
|
def test_original_inventories_require_actual_files_and_digest(self):
|
|
for key in ('original_pre', 'original_post', 'package', 'resources'):
|
|
with self.subTest(key=key):
|
|
value = self.original_record()
|
|
path = self.record_root / ('altered-' + key + '.json')
|
|
forged = copy.deepcopy(self.original_inventory)
|
|
forged['files'][0]['sha256'] = 'f' * 64
|
|
path.write_bytes(receipts.canonical_bytes(forged))
|
|
value['identity'][key] = dict(receipts.identify_file(path), path=path.name)
|
|
self.assertEqual(self.audit([(value, self.record_root)])['matrix']['accepted'], 0)
|
|
|
|
def test_generated_copy_changes_are_separate_from_original_preservation(self):
|
|
value = self.original_record()
|
|
path = self.record_root / 'changed-copy.json'
|
|
path.write_bytes(receipts.canonical_bytes({'files': [], 'generated_change': True}))
|
|
value['identity']['copy_post'] = dict(receipts.identify_file(path), path=path.name)
|
|
self.assertEqual(self.audit([(value, self.record_root)])['matrix']['accepted'], 1)
|
|
|
|
def test_aborted_case_stays_open(self):
|
|
self.record['status'] = 'aborted'
|
|
self.record['acceptance'] = {'state': 'open', 'remaining': ['aborted']}
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 0)
|
|
|
|
def test_wine_label_cannot_fill_native(self):
|
|
self.record['platform']['execution'] = 'wine'
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 0)
|
|
|
|
def test_wrong_package_rejected(self):
|
|
self.package['seal'] = {'sha256': '0' * 64}
|
|
self.assertTrue(self.audit()['errors'])
|
|
|
|
def test_wrong_resource_tree_rejected(self):
|
|
self.package['resources']['assets']['tree_sha256'] = '0' * 64
|
|
self.assertTrue(self.audit()['errors'])
|
|
|
|
def test_windows_seal_source_and_resource_format_is_supported(self):
|
|
source = self.record['identity']['source']
|
|
self.record['identity']['source'] = dict(self.record['identity']['package'],
|
|
**{k: source[k] for k in ('commit', 'dirty', 'diff_sha256')})
|
|
self.record['identity']['resources'] = self.record['identity']['package']
|
|
self.package['build']['source']['status_sha256'] = 'not copied into receipt'
|
|
path = self.record_root / 'package.json'
|
|
path.write_bytes(receipts.canonical_bytes({'source': self.package['build']['source']}))
|
|
updated = dict(receipts.identify_file(path), path=path.name)
|
|
self.package['seal'] = updated
|
|
for key in ('package', 'resources', 'source'):
|
|
self.record['identity'][key].update(updated)
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 1)
|
|
|
|
def test_mismatched_source_revision_rejected(self):
|
|
self.record['identity']['source']['commit'] = 'wrong revision'
|
|
self.assertTrue(self.audit()['errors'])
|
|
|
|
def test_synthetic_original_rejected(self):
|
|
self.record['artifacts'][0]['synthetic'] = True
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 0)
|
|
|
|
def test_absent_visible_audible_reviews_rejected(self):
|
|
self.record['review']['audible'] = False
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 0)
|
|
|
|
def test_physical_input_disposition_required(self):
|
|
del self.review['checks']['physical_input']
|
|
self.assertIn('physical_input', self.audit()['human_needed'])
|
|
|
|
def test_duplicate_history_conflict_rejected(self):
|
|
self.assertTrue(self.audit([(self.record, self.record_root)] * 2)['errors'])
|
|
|
|
def test_historical_receipt_cannot_be_current(self):
|
|
self.record['historical'] = True
|
|
self.assertEqual(self.audit()['matrix']['accepted'], 0)
|
|
|
|
def test_both_unclassified_assumptions_stay_visible(self):
|
|
for key in ('EVID-02-unclassified', 'EVID-04-unclassified'):
|
|
self.review['checks'][key]['observed'] = False
|
|
self.assertTrue(set(('EVID-02-unclassified', 'EVID-04-unclassified')) <=
|
|
set(self.audit()['human_needed']))
|
|
|
|
def test_backstops_and_prohibitions_cannot_be_auto_dismissed(self):
|
|
self.assertTrue(set(('A1', 'A2', 'P1', 'P2', 'P3')) <=
|
|
set(self.audit(review={})['human_needed']))
|
|
|
|
def test_review_must_bind_actual_receipt_ids(self):
|
|
self.review['receipt_ids'] = ['different']
|
|
self.assertTrue(self.audit()['errors'])
|
|
|
|
def test_review_digest_binds_selected_filename_without_sidecar(self):
|
|
checkpoint = self.root / 'checkpoint'
|
|
(checkpoint / 'protocol').mkdir(parents=True)
|
|
(checkpoint / 'manifest.json').write_bytes(receipts.canonical_bytes(
|
|
dict(self.manifest, packages={})))
|
|
(checkpoint / 'protocol/scenarios.json').write_bytes(receipts.canonical_bytes(self.protocol))
|
|
(checkpoint / 'protocol/original-manifest.json').write_bytes(receipts.canonical_bytes(self.original_inventory))
|
|
selected = self.record_root / 'selected.json'
|
|
selected.write_bytes(receipts.canonical_bytes(self.record))
|
|
sibling = self.record_root / 'receipt.json'
|
|
sibling.write_text('{"unrelated": true}')
|
|
(self.root / 'current.json').write_bytes(receipts.canonical_bytes(
|
|
{'receipts': [selected.relative_to(self.root).as_posix()]}))
|
|
for reviewed, expected_pending in ((sibling, True), (None, True), (selected, False)):
|
|
review = dict(self.review, receipt_sha256={} if reviewed is None else {
|
|
self.record['receipt_id']: receipts.identify_file(reviewed)['sha256']})
|
|
(self.root / 'human-review.json').write_bytes(receipts.canonical_bytes(review))
|
|
with mock.patch.object(self.api(), 'ready'), mock.patch.object(self.api(), '_bind'):
|
|
report = self.api().audit(checkpoint, self.root)
|
|
self.assertEqual(bool(report['human_needed']), expected_pending)
|
|
self.assertEqual(report['matrix']['accepted'], 1)
|
|
|
|
def test_actual_failed_linux_receipt_import_and_review_remains_open(self):
|
|
source = ROOT / 'artifacts/fidelity/linux-current-receipts/linux-01-06-matrix-preflight-linux'
|
|
if not source.exists():
|
|
self.assertTrue(self.audit([])['matrix']['missing_obligations'])
|
|
return # Local ignored evidence is not a prerequisite on other checkouts.
|
|
imported = receipts.import_receipt(source / 'receipt.json', source, self.root / 'imports')
|
|
path = Path(imported['path'])
|
|
value = receipts.read_receipt(path)
|
|
report = self.audit([(value, path.parent)], {})
|
|
self.assertEqual(report['matrix']['accepted'], 0)
|
|
self.assertEqual(report['acceptance'], 'human_needed')
|
|
export_review(None, path, self.root / 'review')
|
|
self.assertTrue((self.root / 'review/index.html').is_file())
|
|
|
|
def test_cli_strict_nonzero_and_allow_open_explicit(self):
|
|
api = self.api()
|
|
report = self.audit([])
|
|
with mock.patch.object(api, 'audit', return_value=report):
|
|
self.assertEqual(api.main(['audit', '--checkpoint', str(self.root),
|
|
'--results', str(self.root), '--require-complete']), 1)
|
|
self.assertEqual(api.main(['audit', '--checkpoint', str(self.root),
|
|
'--results', str(self.root), '--allow-open']), 0)
|
|
|
|
def test_ready_cannot_accept_old_incomplete_material_set(self):
|
|
with mock.patch.object(self.api(), 'preflight', return_value={'status': 'ready'}):
|
|
with self.assertRaises((ValueError, FileNotFoundError)):
|
|
self.api().ready(self.root)
|
|
|
|
def prepared(self):
|
|
fixture = test_fidelity_checkpoint.FidelityCheckpoint()
|
|
fixture.setUp()
|
|
self.addCleanup(fixture.doCleanups)
|
|
fixture.prepare()
|
|
root = fixture.output
|
|
manifest = receipts.read_json(root / 'manifest.json')
|
|
manifest['tooling_only'] = False
|
|
(root / 'manifest.json').write_bytes(receipts.canonical_bytes(manifest))
|
|
prepared = receipts.read_json(root / 'READY.json')
|
|
prepared['manifest'] = dict(receipts.identify_file(root / 'manifest.json'), path='manifest.json')
|
|
(root / 'READY.json').write_bytes(receipts.canonical_bytes(prepared))
|
|
return root, manifest
|
|
|
|
def test_ready_validates_all_materials_separately_from_acceptance(self):
|
|
root, manifest = self.prepared()
|
|
with mock.patch.object(self.api(), '_package', side_effect=lambda _, rid, __: manifest['packages'][rid]):
|
|
report = self.api().ready(root)
|
|
self.assertTrue(report['materials_ready'])
|
|
self.assertEqual(report['acceptance'], 'human_needed')
|
|
|
|
def test_ready_rejects_tampered_final_checklist(self):
|
|
root, _ = self.prepared()
|
|
(root / 'docs/evidence/checkpoint-session.md').write_text('changed')
|
|
with self.assertRaises(ValueError):
|
|
self.api().ready(root)
|
|
|
|
def test_audit_explicit_selection_reads_actual_failed_linux_hashes(self):
|
|
root, manifest = self.prepared()
|
|
source = ROOT / 'artifacts/fidelity/linux-current-receipts/linux-01-06-matrix-preflight-linux'
|
|
paths = []
|
|
if source.exists():
|
|
imported = receipts.import_receipt(source / 'receipt.json', source, root / 'results/imported')
|
|
paths.append(Path(imported['path']).relative_to(root / 'results').as_posix())
|
|
(root / 'results/current.json').write_bytes(receipts.canonical_bytes({'receipts': paths}))
|
|
with mock.patch.object(self.api(), '_package', side_effect=lambda _, rid, __: manifest['packages'][rid]):
|
|
report = self.api().audit(root, root / 'results')
|
|
self.assertEqual(report['acceptance'], 'human_needed')
|
|
self.assertEqual(report['matrix']['accepted'], 0)
|