diff --git a/tests/test_fidelity_acceptance.py b/tests/test_fidelity_acceptance.py new file mode 100644 index 0000000..25dabda --- /dev/null +++ b/tests/test_fidelity_acceptance.py @@ -0,0 +1,166 @@ +"""Acceptance metadata doubles are synthetic tests, never runtime proof.""" +import copy +import importlib +import json +from pathlib import Path +import subprocess +import sys +import unittest +from unittest import mock + +import test_fidelity_scenarios + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tools/recon')) +import fidelity_receipts as receipts +from fidelity_review import export_review + +try: + acceptance = importlib.import_module('fidelity_acceptance') +except ModuleNotFoundError: + acceptance = None + + +class AcceptanceTests(unittest.TestCase): + def setUp(self): + self.fixture = test_fidelity_scenarios.FidelityScenarios() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.root = self.fixture.base + self.protocol = self.fixture.protocol() + self.record, self.record_root = self.fixture.receipt() + identity = self.record['identity'] + self.package = {'executable': identity['executable'], + 'build_info': identity['source'], + 'resources': {'assets': {'tree_sha256': 'a' * 64}}} + self.package['build'] = {'source': {k: identity['source'][k] + for k in ('commit', 'dirty', 'diff_sha256')}} + # Bind real hashed metadata files in the test instead of accepting labels. + for key, data in [('package', {'source': self.package['build']['source']}), + ('resources', self.package['resources'])]: + path = self.record_root / (key + '.json') + path.write_bytes(receipts.canonical_bytes(data)) + identity[key] = dict(receipts.identify_file(path), path=path.name) + self.package['seal'] = identity['package'] + self.manifest = {'tooling_only': False, 'packages': { + 'linux-x64': self.package, 'win-x64': self.package}, + 'original_pre': {'tree_sha256': 'c' * 64}} + self.review = {'schema_version': 1, + 'checks': {key: {'observed': True, 'reviewer': 'metadata test', + 'notes': 'Synthetic unit assertion only'} for key in + ('A1', 'A2', 'EVID-02-unclassified', 'EVID-04-unclassified', + 'P1', 'P2', 'P3', 'browser', 'paired_review', 'physical_input')}, + 'receipt_ids': [self.record['receipt_id']]} + + def api(self): + self.assertIsNotNone(acceptance, 'strict readiness/acceptance API must exist') + return acceptance + + def audit(self, records=None, review=None): + return self.api().audit_records(self.protocol, self.manifest, + [(self.record, self.record_root)] if records is None else records, + self.review if review is None else review) + + def test_empty_matrix_never_passes(self): + report = self.audit([]) + self.assertEqual(report['acceptance'], 'human_needed') + self.assertEqual(len(report['matrix']['missing_obligations']), 120) + + def test_identified_valid_cell_covers_only_one_slot(self): + self.assertEqual(self.audit()['matrix']['accepted'], 1) + self.assertEqual(self.audit()['acceptance'], 'human_needed') + + def test_complete_metadata_matrix_with_human_findings(self): + records = [] + for cell in self.protocol['cells']: + for obligation in cell['obligations']: + for kind, os_name in [('native_port', 'linux'), ('native_port', 'windows'), + ('original_dxwnd', 'windows')]: + value, root = self.fixture.receipt(cell['id'], obligation, kind, os_name) + value['identity'] = copy.deepcopy(self.record['identity']) + for path in self.record_root.iterdir(): + if path.is_file() and root != self.record_root: + (root / path.name).write_bytes(path.read_bytes()) + value['identity']['original_pre']['tree_sha256'] = 'c' * 64 + records.append((value, root)) + review = copy.deepcopy(self.review) + review['receipt_ids'] = [r['receipt_id'] for r, _ in records] + self.assertEqual(self.audit(records, review)['acceptance'], 'passed') + + def test_aborted_case_stays_open(self): + self.record['status'] = 'aborted' + self.record['acceptance'] = {'state': 'open', 'remaining': ['aborted']} + self.assertEqual(self.audit()['matrix']['accepted'], 0) + + def test_wine_label_cannot_fill_native(self): + self.record['platform']['execution'] = 'wine' + self.assertEqual(self.audit()['matrix']['accepted'], 0) + + def test_wrong_package_rejected(self): + self.package['seal'] = {'sha256': '0' * 64} + self.assertTrue(self.audit()['errors']) + + def test_wrong_resource_tree_rejected(self): + self.package['resources']['assets']['tree_sha256'] = '0' * 64 + self.assertTrue(self.audit()['errors']) + + def test_synthetic_original_rejected(self): + self.record['artifacts'][0]['synthetic'] = True + self.assertEqual(self.audit()['matrix']['accepted'], 0) + + def test_absent_visible_audible_reviews_rejected(self): + self.record['review']['audible'] = False + self.assertEqual(self.audit()['matrix']['accepted'], 0) + + def test_physical_input_disposition_required(self): + del self.review['checks']['physical_input'] + self.assertIn('physical_input', self.audit()['human_needed']) + + def test_duplicate_history_conflict_rejected(self): + self.assertTrue(self.audit([(self.record, self.record_root)] * 2)['errors']) + + def test_historical_receipt_cannot_be_current(self): + self.record['historical'] = True + self.assertEqual(self.audit()['matrix']['accepted'], 0) + + def test_both_unclassified_assumptions_stay_visible(self): + for key in ('EVID-02-unclassified', 'EVID-04-unclassified'): + self.review['checks'][key]['observed'] = False + self.assertTrue(set(('EVID-02-unclassified', 'EVID-04-unclassified')) <= + set(self.audit()['human_needed'])) + + def test_backstops_and_prohibitions_cannot_be_auto_dismissed(self): + self.assertTrue(set(('A1', 'A2', 'P1', 'P2', 'P3')) <= + set(self.audit(review={})['human_needed'])) + + def test_review_must_bind_actual_receipt_ids(self): + self.review['receipt_ids'] = ['different'] + self.assertTrue(self.audit()['errors']) + + def test_actual_failed_linux_receipt_import_and_review_remains_open(self): + source = ROOT / 'artifacts/fidelity/linux-current-receipts/linux-01-06-matrix-preflight-linux' + if not source.exists(): + self.assertTrue(self.audit([])['matrix']['missing_obligations']) + return # Local ignored evidence is not a prerequisite on other checkouts. + imported = receipts.import_receipt(source / 'receipt.json', source, self.root / 'imports') + path = Path(imported['path']) + value = receipts.read_receipt(path) + report = self.audit([(value, path.parent)], {}) + self.assertEqual(report['matrix']['accepted'], 0) + self.assertEqual(report['acceptance'], 'human_needed') + export_review(None, path, self.root / 'review') + self.assertTrue((self.root / 'review/index.html').is_file()) + + def test_cli_strict_nonzero_and_allow_open_explicit(self): + api = self.api() + report = self.audit([]) + with mock.patch.object(api, 'audit', return_value=report): + self.assertEqual(api.main(['audit', '--checkpoint', str(self.root), + '--results', str(self.root), '--require-complete']), 1) + self.assertEqual(api.main(['audit', '--checkpoint', str(self.root), + '--results', str(self.root), '--allow-open']), 0) + + def test_ready_cannot_accept_old_incomplete_material_set(self): + with mock.patch.object(self.api(), 'preflight', return_value={'status': 'ready'}): + with self.assertRaises((ValueError, FileNotFoundError)): + self.api().ready(self.root)