Files
Neutron/tools/extractors/entities.py
T

197 lines
9.9 KiB
Python

#!/usr/bin/env python3
"""Prepare source-backed GAM entity references without assigning gameplay rules.
Keeps source order/offsets, duplicate tags and exact typed property values. Optional
resource decoding exports only resources named by explicit database/index pairs.
It does not turn editor icons into rendered actors or infer class asset defaults.
"""
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
from dataclasses import asdict
import hashlib
import io
import json
from pathlib import Path
import re
import struct
import sys
import wave
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'levels'))
from gam import parse
from omt import Database, FormatError, read_canvas, read_shape, safe_name, write_png
SENTINELS = {'none', 'null', 'default', 'end', ''}
OBJECT_LINKS = {
'StartTrigger', 'NextTrigger', 'ToggleObject', 'ActivateObject', 'CameraTarget',
'PlayerControlled', 'FaceObject', 'AITarget', 'AIPatrol', 'AINewPos', 'PatrolPoint',
'NextPatrolPoint', 'CallObjectTag', 'ActivateBy', 'PickupLink', 'ActivateButton',
}
def resolve_asset(root: Path, relative: str) -> Path:
"""Portable case-insensitive lookup, rejecting traversal and ambiguities."""
root = root.resolve()
current = root
parts = relative.replace('\\', '/').split('/')
if not relative or any(p in ('', '.', '..') for p in parts):
raise FormatError(f'Invalid asset path: {relative!r}')
for part in parts:
matches = [p for p in current.iterdir() if p.name.casefold() == part.casefold()]
if len(matches) != 1:
raise FormatError(f'Missing or ambiguous asset: {relative}')
current = matches[0]
if not current.resolve().is_relative_to(root):
raise FormatError(f'Asset escapes original root: {relative}')
return current
def _object_link(name: str) -> bool:
return name in OBJECT_LINKS or bool(re.fullmatch(r'(CameraTarget|ActivateObject)\d+', name))
def resource_pairs(values: dict):
"""Explicit serialized pairings only; no implicit global sound bank assumption."""
for prefix, resource_type in (('Sprite', 'Canv'), ('Music', 'Wave'), ('Omt', '3DSh')):
database, index = values.get(prefix + 'Database'), values.get(prefix + 'Index')
if isinstance(database, str) and isinstance(index, int) and index >= 0 and database.casefold() not in SENTINELS:
yield prefix + 'Index', database, resource_type, index
database = values.get('SpriteDatabase')
if isinstance(database, str) and database.casefold() not in SENTINELS:
for name, index in values.items():
if re.fullmatch(r'Sprite[1-9]', name) and isinstance(index, int) and index >= 0:
yield name, database, 'Canv', index
database = values.get('SoundDatabase')
if isinstance(database, str) and database.casefold() not in SENTINELS:
for name, index in values.items():
if re.fullmatch(r'SoundIndex\d*', name) and isinstance(index, int) and index >= 0:
yield name, database, 'Wave', index
def prepare_entities(original: Path, level_file: str, output: Path, decode: bool = False) -> dict:
original = original.resolve()
output = output.resolve()
if output.is_relative_to(original) or original.is_relative_to(output):
raise FormatError('Output must be separate from the original directory')
source = resolve_asset(original, 'gam/' + level_file)
data = source.read_bytes()
level = parse(data)
output.mkdir(parents=True, exist_ok=True)
tags = defaultdict(list)
for ordinal, entity in enumerate(level.entities):
tag = entity.values().get('ObjectTag', '')
tags[str(tag).casefold()].append(ordinal)
databases, resources = {}, {}
entities = []
for ordinal, entity in enumerate(level.entities):
values = entity.values()
record = {
'ordinal': ordinal, 'kind': entity.kind, 'offset': entity.offset,
'tag': values.get('ObjectTag'),
'position': [values.get('Position' + axis, 0) for axis in 'XYZ'],
'rotation': [values.get('Rotation' + axis, 0) for axis in 'XYZ'],
'properties': values,
'propertyEvidence': {p.name: {'offset': p.offset, 'type': p.type, 'length': p.length, 'raw': p.raw}
for p in entity.properties},
'objectReferences': [], 'resourceReferences': [], 'fileReferences': [],
}
for name, value in values.items():
if _object_link(name) and isinstance(value, str) and value.casefold() not in SENTINELS:
candidates = tags.get(value.casefold(), [])
record['objectReferences'].append({'property': name, 'tag': value,
'candidateOrdinals': candidates})
if isinstance(value, str) and value.lower().endswith(('.ase', '.png')):
folder = 'ASE' if value.lower().endswith('.ase') else 'png'
try:
path = resolve_asset(original, folder + '/' + value)
record['fileReferences'].append({'property': name, 'source': path.relative_to(original).as_posix(),
'sha256': hashlib.sha256(path.read_bytes()).hexdigest()})
except FormatError as error:
record['fileReferences'].append({'property': name, 'source': value, 'unresolved': str(error)})
for property_name, database_name, resource_type, identity in resource_pairs(values):
database_path = resolve_asset(original, 'omt/' + database_name)
canonical = database_path.relative_to(original).as_posix()
if canonical not in databases:
databases[canonical] = Database(database_path)
database = databases[canonical]
key = (canonical, resource_type, identity)
if key not in resources:
matches = [e for e in database.entries if e.type == resource_type and e.id == identity]
resource = {'database': canonical, 'type': resource_type, 'id': identity,
'databaseSha256': hashlib.sha256(database.data).hexdigest()}
if len(matches) != 1:
resource['unresolved'] = 'No matching resource in explicitly named database'
else:
entry = matches[0]
resource.update(name=entry.name, offset=entry.offset, size=entry.size)
if decode:
_decode(database, entry, resource, output)
resources[key] = resource
record['resourceReferences'].append({'property': property_name, 'database': canonical,
'type': resource_type, 'id': identity})
entities.append(record)
report = {
'schema': 1, 'source': source.relative_to(original).as_posix(),
'sha256': hashlib.sha256(data).hexdigest(), 'level': level.code,
'policy': 'Exact source entity data and explicit references only; gameplay/visibility rules are not inferred.',
'counts': dict(sorted(Counter(e.kind for e in level.entities).items())),
'duplicateTags': {tag: ids for tag, ids in sorted(tags.items()) if len(ids) > 1},
'entities': entities, 'resources': [resources[key] for key in sorted(resources)],
}
(output / 'entities.json').write_text(json.dumps(report, indent=2, allow_nan=False) + '\n', encoding='utf8')
return report
def _decode(database, entry, resource: dict, output: Path):
folder = output / 'resources' / safe_name(database.path.stem)
folder.mkdir(parents=True, exist_ok=True)
stem = f'{entry.type}_{entry.id:08x}_{safe_name(entry.name)}'
payload = database.payload(entry)
if entry.type == 'Canv':
image = read_canvas(payload)
rgba = image.pop('rgba')
raw, png = folder / (stem + '.rgba'), folder / (stem + '.png')
raw.write_bytes(rgba)
write_png(png, image['width'], image['height'], rgba)
resource.update(image, path=raw.relative_to(output).as_posix(), png=png.relative_to(output).as_posix())
elif entry.type == '3DSh':
path = folder / (stem + '.json')
path.write_text(json.dumps(read_shape(payload), allow_nan=False) + '\n', encoding='utf8')
resource['path'] = path.relative_to(output).as_posix()
elif entry.type == 'Wave':
start = payload.find(b'RIFF')
if start < 0 or payload[start + 8:start + 12] != b'WAVE' or start + 12 > len(payload):
raise FormatError(f'Wave resource is missing bounded RIFF: {entry.id}')
size = struct.unpack_from('<I', payload, start + 4)[0] + 8
if size < 12 or start + size > len(payload):
raise FormatError(f'RIFF outside resource: {entry.id}')
audio = payload[start:start + size]
path = folder / (stem + '.wav')
path.write_bytes(audio)
resource['path'] = path.relative_to(output).as_posix()
try:
with wave.open(io.BytesIO(audio)) as wav:
resource['audio'] = {'rate': wav.getframerate(), 'channels': wav.getnchannels(),
'sampleWidth': wav.getsampwidth(), 'frames': wav.getnframes(),
'seconds': wav.getnframes() / wav.getframerate()}
except wave.Error:
resource['audio'] = {'encoding': 'non-PCM RIFF retained unchanged'}
def main():
cli = argparse.ArgumentParser(description=__doc__)
cli.add_argument('--original', type=Path, default=Path('../Original'))
cli.add_argument('--level', default='level1b.gam')
cli.add_argument('--output', type=Path, required=True)
cli.add_argument('--decode', action='store_true')
args = cli.parse_args()
report = prepare_entities(args.original, args.level, args.output, args.decode)
print(f"{report['level']}: {len(report['entities'])} source entities, {len(report['resources'])} explicit resource references")
if __name__ == '__main__':
main()