Files
Neutron/tests/test_fidelity_acceptance.py
T

226 lines
11 KiB
Python

"""Acceptance metadata doubles are synthetic tests, never runtime proof."""
import copy
import importlib
import json
from pathlib import Path
import subprocess
import sys
import unittest
from unittest import mock
import test_fidelity_scenarios
import test_fidelity_checkpoint
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tools/recon'))
import fidelity_receipts as receipts
from fidelity_review import export_review
try:
acceptance = importlib.import_module('fidelity_acceptance')
except ModuleNotFoundError:
acceptance = None
class AcceptanceTests(unittest.TestCase):
def setUp(self):
self.fixture = test_fidelity_scenarios.FidelityScenarios()
self.fixture.setUp()
self.addCleanup(self.fixture.doCleanups)
self.root = self.fixture.base
self.protocol = self.fixture.protocol()
self.record, self.record_root = self.fixture.receipt()
identity = self.record['identity']
self.package = {'executable': identity['executable'],
'build_info': identity['source'],
'resources': {'assets': {'tree_sha256': 'a' * 64}}}
self.package['build'] = {'source': {k: identity['source'][k]
for k in ('commit', 'dirty', 'diff_sha256')}}
# Bind real hashed metadata files in the test instead of accepting labels.
for key, data in [('original_pre', {'tree_sha256': 'c' * 64}),
('package', {'source': self.package['build']['source']}),
('resources', self.package['resources'])]:
path = self.record_root / (key + '.json')
path.write_bytes(receipts.canonical_bytes(data))
identity[key] = dict(receipts.identify_file(path), path=path.name)
self.package['seal'] = identity['package']
self.manifest = {'tooling_only': False, 'packages': {
'linux-x64': self.package, 'win-x64': self.package},
'original_pre': {'tree_sha256': 'c' * 64}}
self.review = {'schema_version': 1,
'checks': {key: {'observed': True, 'reviewer': 'metadata test',
'notes': 'Synthetic unit assertion only'} for key in
('A1', 'A2', 'EVID-02-unclassified', 'EVID-04-unclassified',
'P1', 'P2', 'P3', 'browser', 'paired_review', 'physical_input')},
'receipt_ids': [self.record['receipt_id']]}
def api(self):
self.assertIsNotNone(acceptance, 'strict readiness/acceptance API must exist')
return acceptance
def audit(self, records=None, review=None):
return self.api().audit_records(self.protocol, self.manifest,
[(self.record, self.record_root)] if records is None else records,
self.review if review is None else review)
def test_empty_matrix_never_passes(self):
report = self.audit([])
self.assertEqual(report['acceptance'], 'human_needed')
self.assertEqual(len(report['matrix']['missing_obligations']), 120)
def test_identified_valid_cell_covers_only_one_slot(self):
self.assertEqual(self.audit()['matrix']['accepted'], 1)
self.assertEqual(self.audit()['acceptance'], 'human_needed')
def test_complete_metadata_matrix_with_human_findings(self):
records = []
for cell in self.protocol['cells']:
for obligation in cell['obligations']:
for kind, os_name in [('native_port', 'linux'), ('native_port', 'windows'),
('original_dxwnd', 'windows')]:
value, root = self.fixture.receipt(cell['id'], obligation, kind, os_name)
value['identity'] = copy.deepcopy(self.record['identity'])
for path in self.record_root.iterdir():
if path.is_file() and root != self.record_root:
(root / path.name).write_bytes(path.read_bytes())
records.append((value, root))
review = copy.deepcopy(self.review)
review['receipt_ids'] = [r['receipt_id'] for r, _ in records]
self.assertEqual(self.audit(records, review)['acceptance'], 'passed')
def test_aborted_case_stays_open(self):
self.record['status'] = 'aborted'
self.record['acceptance'] = {'state': 'open', 'remaining': ['aborted']}
self.assertEqual(self.audit()['matrix']['accepted'], 0)
def test_wine_label_cannot_fill_native(self):
self.record['platform']['execution'] = 'wine'
self.assertEqual(self.audit()['matrix']['accepted'], 0)
def test_wrong_package_rejected(self):
self.package['seal'] = {'sha256': '0' * 64}
self.assertTrue(self.audit()['errors'])
def test_wrong_resource_tree_rejected(self):
self.package['resources']['assets']['tree_sha256'] = '0' * 64
self.assertTrue(self.audit()['errors'])
def test_windows_seal_source_and_resource_format_is_supported(self):
source = self.record['identity']['source']
self.record['identity']['source'] = dict(self.record['identity']['package'],
**{k: source[k] for k in ('commit', 'dirty', 'diff_sha256')})
self.record['identity']['resources'] = self.record['identity']['package']
self.package['build']['source']['status_sha256'] = 'not copied into receipt'
path = self.record_root / 'package.json'
path.write_bytes(receipts.canonical_bytes({'source': self.package['build']['source']}))
updated = dict(receipts.identify_file(path), path=path.name)
self.package['seal'] = updated
for key in ('package', 'resources', 'source'):
self.record['identity'][key].update(updated)
self.assertEqual(self.audit()['matrix']['accepted'], 1)
def test_mismatched_source_revision_rejected(self):
self.record['identity']['source']['commit'] = 'wrong revision'
self.assertTrue(self.audit()['errors'])
def test_synthetic_original_rejected(self):
self.record['artifacts'][0]['synthetic'] = True
self.assertEqual(self.audit()['matrix']['accepted'], 0)
def test_absent_visible_audible_reviews_rejected(self):
self.record['review']['audible'] = False
self.assertEqual(self.audit()['matrix']['accepted'], 0)
def test_physical_input_disposition_required(self):
del self.review['checks']['physical_input']
self.assertIn('physical_input', self.audit()['human_needed'])
def test_duplicate_history_conflict_rejected(self):
self.assertTrue(self.audit([(self.record, self.record_root)] * 2)['errors'])
def test_historical_receipt_cannot_be_current(self):
self.record['historical'] = True
self.assertEqual(self.audit()['matrix']['accepted'], 0)
def test_both_unclassified_assumptions_stay_visible(self):
for key in ('EVID-02-unclassified', 'EVID-04-unclassified'):
self.review['checks'][key]['observed'] = False
self.assertTrue(set(('EVID-02-unclassified', 'EVID-04-unclassified')) <=
set(self.audit()['human_needed']))
def test_backstops_and_prohibitions_cannot_be_auto_dismissed(self):
self.assertTrue(set(('A1', 'A2', 'P1', 'P2', 'P3')) <=
set(self.audit(review={})['human_needed']))
def test_review_must_bind_actual_receipt_ids(self):
self.review['receipt_ids'] = ['different']
self.assertTrue(self.audit()['errors'])
def test_actual_failed_linux_receipt_import_and_review_remains_open(self):
source = ROOT / 'artifacts/fidelity/linux-current-receipts/linux-01-06-matrix-preflight-linux'
if not source.exists():
self.assertTrue(self.audit([])['matrix']['missing_obligations'])
return # Local ignored evidence is not a prerequisite on other checkouts.
imported = receipts.import_receipt(source / 'receipt.json', source, self.root / 'imports')
path = Path(imported['path'])
value = receipts.read_receipt(path)
report = self.audit([(value, path.parent)], {})
self.assertEqual(report['matrix']['accepted'], 0)
self.assertEqual(report['acceptance'], 'human_needed')
export_review(None, path, self.root / 'review')
self.assertTrue((self.root / 'review/index.html').is_file())
def test_cli_strict_nonzero_and_allow_open_explicit(self):
api = self.api()
report = self.audit([])
with mock.patch.object(api, 'audit', return_value=report):
self.assertEqual(api.main(['audit', '--checkpoint', str(self.root),
'--results', str(self.root), '--require-complete']), 1)
self.assertEqual(api.main(['audit', '--checkpoint', str(self.root),
'--results', str(self.root), '--allow-open']), 0)
def test_ready_cannot_accept_old_incomplete_material_set(self):
with mock.patch.object(self.api(), 'preflight', return_value={'status': 'ready'}):
with self.assertRaises((ValueError, FileNotFoundError)):
self.api().ready(self.root)
def prepared(self):
fixture = test_fidelity_checkpoint.FidelityCheckpoint()
fixture.setUp()
self.addCleanup(fixture.doCleanups)
fixture.prepare()
root = fixture.output
manifest = receipts.read_json(root / 'manifest.json')
manifest['tooling_only'] = False
(root / 'manifest.json').write_bytes(receipts.canonical_bytes(manifest))
prepared = receipts.read_json(root / 'READY.json')
prepared['manifest'] = dict(receipts.identify_file(root / 'manifest.json'), path='manifest.json')
(root / 'READY.json').write_bytes(receipts.canonical_bytes(prepared))
return root, manifest
def test_ready_validates_all_materials_separately_from_acceptance(self):
root, manifest = self.prepared()
with mock.patch.object(self.api(), '_package', side_effect=lambda _, rid, __: manifest['packages'][rid]):
report = self.api().ready(root)
self.assertTrue(report['materials_ready'])
self.assertEqual(report['acceptance'], 'human_needed')
def test_ready_rejects_tampered_final_checklist(self):
root, _ = self.prepared()
(root / 'docs/evidence/checkpoint-session.md').write_text('changed')
with self.assertRaises(ValueError):
self.api().ready(root)
def test_audit_explicit_selection_reads_actual_failed_linux_hashes(self):
root, manifest = self.prepared()
source = ROOT / 'artifacts/fidelity/linux-current-receipts/linux-01-06-matrix-preflight-linux'
paths = []
if source.exists():
imported = receipts.import_receipt(source / 'receipt.json', source, root / 'results/imported')
paths.append(Path(imported['path']).relative_to(root / 'results').as_posix())
(root / 'results/current.json').write_bytes(receipts.canonical_bytes({'receipts': paths}))
with mock.patch.object(self.api(), '_package', side_effect=lambda _, rid, __: manifest['packages'][rid]):
report = self.api().audit(root, root / 'results')
self.assertEqual(report['acceptance'], 'human_needed')
self.assertEqual(report['matrix']['accepted'], 0)