Files
Neutron/tools/extractors/omt.py
T

402 lines
19 KiB
Python

#!/usr/bin/env python3
"""Read-only Open Media Toolkit 0MF2 database recovery. No third-party modules."""
from __future__ import annotations
import argparse
from dataclasses import asdict, dataclass
import hashlib
import json
import math
from pathlib import Path
import re
import struct
import wave
import zlib
class FormatError(ValueError):
"""Malformed or unsupported original resource."""
class Reader:
def __init__(self, data: bytes, position: int = 0):
self.data = data
self.position = position
def take(self, size: int) -> bytes:
if size < 0 or self.position < 0 or self.position + size > len(self.data):
raise FormatError(f"Read outside resource at 0x{self.position:x}, size {size}")
result = self.data[self.position:self.position + size]
self.position += size
return result
def unpack(self, fmt: str):
return struct.unpack(fmt, self.take(struct.calcsize(fmt)))
def u8(self): return self.unpack('>B')[0]
def u16(self): return self.unpack('>H')[0]
def u32(self): return self.unpack('>I')[0]
def f32(self): return self.unpack('>f')[0]
def boolean(self): return self.u16() != 0
def vec3(self): return self.unpack('>3f')
def string(self): return self.take(self.u8()).decode('latin1')
def count(self, minimum_bytes: int):
count = self.u32()
if count > (len(self.data) - self.position) // minimum_bytes:
raise FormatError(f"Impossible count {count} at 0x{self.position - 4:x}")
return count
def link(self):
if not self.boolean(): return None
return {'type': self.take(4).decode('latin1'), 'id': self.u32()}
def end(self):
if self.position != len(self.data):
raise FormatError(f"Unconsumed resource bytes: {len(self.data)-self.position}")
@dataclass(frozen=True)
class Entry:
type: str
id: int
offset: int
size: int
name: str
class Database:
def __init__(self, path: Path | str):
self.path = Path(path)
self.data = self.path.read_bytes()
reader = Reader(self.data)
if reader.take(4) != b'0MF2':
raise FormatError('Expected 0MF2 database signature')
self.table_offset = reader.u32()
if not 8 <= self.table_offset < len(self.data):
raise FormatError('Database table offset is outside file')
reader.position = self.table_offset
groups = reader.count(8)
self.entries = []
keys = set()
for _ in range(groups):
tag = reader.take(4).decode('latin1')
for _ in range(reader.count(13)):
identity, offset, size = reader.unpack('>III')
name = reader.string()
if offset < 8 or offset + size > len(self.data):
raise FormatError(f'Resource {tag}:{identity} outside file')
if (tag, identity) in keys:
raise FormatError(f'Duplicate resource {tag}:{identity}')
keys.add((tag, identity))
self.entries.append(Entry(tag, identity, offset, size, name))
self.table_end = reader.position
def payload(self, entry: Entry) -> bytes:
return self.data[entry.offset:entry.offset + entry.size]
def read_shape(data: bytes) -> dict:
"""Decode 3DSP versions 0..4 using OMT2.dll read_class evidence."""
r = Reader(data)
if r.take(4) != b'3DSP': raise FormatError('Expected 3DSP shape')
version = r.u32()
if version > 4: raise FormatError(f'Unsupported 3DSP version {version}')
bounds = r.unpack('>8f')
legacy = {'unused': r.f32(), 'flag': r.boolean()} if 0 < version < 3 else None
vertices = [r.vec3() for _ in range(r.count(12))]
polygons = []
for _ in range(r.count(1)):
if version <= 2:
flags = {'first': r.boolean(), 'second': r.boolean(), 'legacy': r.u16()}
identifier = r.u32()
extra = r.unpack('>2f') if version else None
legacy_flag = r.u16()
corners = r.u16()
if corners > (len(data)-r.position)//12: raise FormatError('Impossible polygon corner count')
points = []
for _ in range(corners):
index = r.u32()
uv = r.unpack('>2f')
normal_index = r.u32() if version > 1 else index
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
points.append({'vertex': index, 'uv': uv, 'normal': normal_index})
normal = r.vec3()
# Old polygon format has an explicit material-present bool around link.
material = r.link() if r.boolean() else None
polygons.append({'flags': flags, 'id': identifier, 'extra': extra,
'legacy_flag': legacy_flag, 'corners': points,
'normal': normal, 'material': material})
else:
flags = r.u16()
identifier = r.u32()
points = []
for _ in range(r.count(20)):
index, normal_index, colour_index = r.unpack('>3I')
uv = r.unpack('>2f')
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
points.append({'vertex': index, 'normal': normal_index,
'colour': colour_index, 'uv': uv})
polygons.append({'flags': flags, 'id': identifier, 'corners': points,
'normal': r.vec3(), 'material': r.link()})
colours = [r.unpack('>4f') for _ in range(r.count(16))]
normals = [r.vec3() for _ in range(r.count(12))]
additional = [r.unpack('>4f') for _ in range(r.count(16))] if version > 2 else []
r.end()
if not all(math.isfinite(v) for point in vertices for v in point):
raise FormatError('Nonfinite shape coordinate')
return {'version': version, 'bounds': bounds, 'legacy': legacy, 'vertices': vertices,
'polygons': polygons, 'colours': colours, 'normals': normals,
'additional': additional}
def safe_name(value: str) -> str:
return re.sub(r'[^a-zA-Z0-9._-]+', '_', value).strip('._')[:100] or 'unnamed'
def read_material(data: bytes) -> dict:
r = Reader(data)
version = r.u16()
if version != 1: raise FormatError(f'Unsupported material version {version}')
style = r.u16()
if style > 2: raise FormatError(f'Unsupported material style {style}')
shininess, legacy_flag = r.f32(), r.boolean()
colours = {}
if style == 0:
colours['diffuse'] = [v/65535 for v in r.unpack('>3H')]
else:
for name in ('emission', 'diffuse', 'specular', 'ambient'):
colours[name] = [v/65535 for v in r.unpack('>3H')]
legacy_boolean = r.boolean()
texture, texture_override_flag = None, False
if style == 2:
texture_override_flag = r.boolean()
if r.u16() == 1: texture = r.link()
trailing = r.u16()
r.end()
return {'version': version, 'style': style, 'shininess': shininess,
'legacyFlag': legacy_flag, 'colours': colours,
'legacyBoolean': legacy_boolean if style else False, 'textureOverrideFlag': texture_override_flag,
'texture': texture, 'trailing': trailing}
def unpack_row(encoded: bytes, expected_size: int, unit: int) -> bytes:
"""OMT PackBits: LE signed control of unit width, payload in source pixel order."""
r = Reader(encoded)
output = bytearray()
while len(output) < expected_size:
control_bytes = r.take(unit)
# 32-bit variant tests the low signed 16 bits, as original routine does.
control = int.from_bytes(control_bytes[:min(unit, 2)], 'little', signed=True)
if control == -128: continue
count = control + 1 if control >= 0 else 1 - control
if len(output) + count*unit > expected_size:
raise FormatError('RLE run exceeds decoded row')
output.extend(r.take(count*unit) if control >= 0 else r.take(unit)*count)
r.end()
return bytes(output)
def read_canvas(data: bytes) -> dict:
r = Reader(data)
magic = r.take(4)
if magic != b'OmCv': raise FormatError(f'Unsupported canvas signature {magic!r}')
flags = r.u32()
if r.take(4) != b'OmGW': raise FormatError('Missing OmGW pixel header')
width, height, depth = r.unpack('>3H')
packing, key_type, row_bytes = r.u8(), r.u8(), r.u16()
if not width or not height or width*height > 16777216: raise FormatError('Invalid canvas dimensions')
if packing not in (0, 1, 2): raise FormatError('Unknown canvas packing')
if depth not in (8, 16, 32): raise FormatError(f'Unsupported canvas depth {depth}')
has_key = r.boolean()
key = None
if key_type == 0:
rgb = r.unpack('>3H')
if has_key:
channels = [v*(31 if depth == 16 else 255)//65535 for v in rgb]
key = ((channels[0]<<10)|(channels[1]<<5)|channels[2]) if depth == 16 else ((channels[0]<<16)|(channels[1]<<8)|channels[2])
elif has_key: key = r.u32()
palette = None
if r.boolean():
signature = r.take(4)
if signature not in (b'OPa2', b'OmPa'): raise FormatError('Invalid canvas palette signature')
r.string() # Palette name; absent for these originals.
count = r.u32()
if count > 256: raise FormatError('Palette exceeds 256 colours')
palette = [tuple(v & 255 for v in r.unpack('>3H')) for _ in range(count)]
if signature == b'OPa2': r.take(4096) # Legacy palette matching accelerator.
if count <= 2: palette = [(255,255,255), (0,0,0)]
if depth == 8 and palette is None: raise FormatError('Indexed canvas has no palette')
size = r.u32()
compressed = Reader(r.take(size))
r.end()
if row_bytes < width*(depth//8): raise FormatError('Canvas row stride too small')
pixels = bytearray()
for _ in range(height):
row = unpack_row(compressed.take(compressed.u16()), row_bytes, 1 << packing)
for x in range(width):
v = int.from_bytes(row[x*(depth//8):(x+1)*(depth//8)], 'big')
if depth == 8:
if v >= len(palette): raise FormatError('Pixel index outside palette')
rgb = palette[v]
elif depth == 16:
rgb = (((v>>10)&31)<<3, ((v>>5)&31)<<3, (v&31)<<3)
else: rgb = ((v>>16)&255, (v>>8)&255, v&255)
pixels.extend((*rgb, 0 if key is not None and v == key else 255))
compressed.end()
return {'width': width, 'height': height, 'depth': depth, 'flags': flags,
'packing': packing, 'rowBytes': row_bytes, 'colourKey': key,
'rgba': bytes(pixels)}
def write_png(path: Path, width: int, height: int, rgba: bytes):
if len(rgba) != width*height*4: raise ValueError('Incorrect RGBA byte count')
def chunk(tag, payload):
return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag+payload))
scanlines = b''.join(b'\0' + rgba[y*width*4:(y+1)*width*4] for y in range(height))
path.write_bytes(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b''))
def export_scene(database: Database, output: Path):
"""Expanded original corner data; triangle fans preserve source winding."""
validate_output(database, output)
output.mkdir(parents=True, exist_ok=True)
scene = {'source': database.path.name, 'sha256': hashlib.sha256(database.data).hexdigest(),
'coordinates': 'Original OMT world coordinates, unchanged',
'vertices': [], 'uvs': [], 'normals': [], 'indices': [], 'meshes': [],
'materials': [], 'textures': [], 'warnings': [], 'collisionPolygons': []}
material_names = {entry.id: entry.name for entry in database.entries if entry.type == '3DMa'}
for entry in database.entries:
payload = database.payload(entry)
if entry.type == '3DSh':
shape = read_shape(payload)
for polygon_index, poly in enumerate(shape['polygons']):
link = poly['material']
resolved_material = link and link['type'] == '3DMa' and link['id'] in material_names
# Player43AA40 compares the signed long stored in the database
# chunk, independently of the render material/texture index.
material_id = link['id'] if resolved_material else None
if material_id is not None and material_id >= 0x80000000:
material_id -= 0x100000000
first_index = len(scene['indices'])
first_vertex = len(scene['vertices'])
for corner in poly['corners']:
scene['vertices'].append(shape['vertices'][corner['vertex']])
scene['uvs'].append(corner['uv'])
ni = corner['normal']
scene['normals'].append(shape['normals'][ni] if ni < len(shape['normals']) else poly['normal'])
for i in range(1, len(poly['corners'])-1):
scene['indices'].extend((first_vertex, first_vertex+i, first_vertex+i+1))
scene['collisionPolygons'].append({'shapeId': entry.id, 'polygonIndex': polygon_index,
'normal': poly['normal'], 'materialName': material_names[link['id']] if resolved_material else None,
'materialId': material_id})
if len(poly['corners']) > 3:
scene['warnings'].append(f'{entry.id}: polygon with {len(poly["corners"])} corners fan triangulated; concavity not validated')
material = poly['material']['id'] if poly['material'] else -1
if scene['meshes'] and scene['meshes'][-1]['shapeId'] == entry.id and scene['meshes'][-1]['material'] == material:
scene['meshes'][-1]['indexCount'] += len(scene['indices'])-first_index
else:
scene['meshes'].append({'name': entry.name, 'shapeId': entry.id,
'firstIndex': first_index, 'indexCount': len(scene['indices'])-first_index, 'material': material})
elif entry.type == '3DMa':
material = read_material(payload)
scene['materials'].append({'id': entry.id, 'name': entry.name, **material})
elif entry.type == 'Canv':
canvas = read_canvas(payload)
base = f'texture_{entry.id:08x}'
(output / (base + '.rgba')).write_bytes(canvas.pop('rgba'))
rgba = (output / (base + '.rgba')).read_bytes()
write_png(output / (base + '.png'), canvas['width'], canvas['height'], rgba)
scene['textures'].append({'id': entry.id, 'name': entry.name,
'path': base + '.rgba', 'png': base + '.png', **canvas})
(output / 'scene.json').write_text(json.dumps(scene, separators=(',', ':')) + '\n', encoding='utf-8')
return scene
def write_obj(shape: dict, path: Path):
"""Preserve original positions/UVs/polygon normals in a standard interchange file."""
lines = ['# Original OMT coordinates; no transform or guessed smoothing applied']
lines.extend('v ' + ' '.join(format(v, '.9g') for v in p) for p in shape['vertices'])
for poly in shape['polygons']:
lines.extend('vt ' + ' '.join(format(v, '.9g') for v in c['uv']) for c in poly['corners'])
lines.extend('vn ' + ' '.join(format(v, '.9g') for v in p['normal']) for p in shape['polygons'])
uv_index = 1
for normal_index, poly in enumerate(shape['polygons'], 1):
if poly['material']: lines.append(f"usemtl material_{poly['material']['id']}")
corners = []
for corner in poly['corners']:
corners.append(f"{corner['vertex']+1}/{uv_index}/{normal_index}")
uv_index += 1
if len(corners) >= 3: lines.append('f ' + ' '.join(corners))
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
def extract(database: Database, output: Path, raw: bool = False):
output = output.resolve()
validate_output(database, output)
output.mkdir(parents=True, exist_ok=True)
report = {'source': str(database.path), 'sha256': hashlib.sha256(database.data).hexdigest(),
'table_offset': database.table_offset, 'entries': []}
for entry in database.entries:
info = asdict(entry)
payload = database.payload(entry)
base = f'{safe_name(entry.type)}_{entry.id:08x}_{safe_name(entry.name)}'
if raw and entry.type != '0HDR':
(output / (base + '.bin')).write_bytes(payload)
if entry.type == '3DSh':
try:
shape = read_shape(payload)
write_obj(shape, output / (base + '.obj'))
(output / (base + '.json')).write_text(json.dumps(shape), encoding='utf-8')
info.update(vertices=len(shape['vertices']), polygons=len(shape['polygons']))
except FormatError as ex: info['unsupported'] = str(ex)
if entry.type == 'Wave':
start = payload.find(b'RIFF')
if start >= 0 and payload[start+8:start+12] == b'WAVE':
size = struct.unpack_from('<I', payload, start+4)[0] + 8
if start + size <= len(payload):
target = output / (base + '.wav')
target.write_bytes(payload[start:start+size])
try:
with wave.open(str(target)) as wav:
info['audio'] = {'channels': wav.getnchannels(), 'rate': wav.getframerate(),
'sample_width': wav.getsampwidth(), 'frames': wav.getnframes()}
except wave.Error:
info['audio'] = {'encoding': 'non-PCM RIFF preserved unchanged'}
else: info['unsupported'] = 'RIFF size outside resource'
report['entries'].append(info)
(output / 'manifest.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
return report
def validate_output(database: Database, output: Path):
source_directory = database.path.parent.resolve()
target = output.resolve()
if target.is_relative_to(source_directory) or database.path.resolve().is_relative_to(target):
raise ValueError('Output must be separate from original input directory')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('database', type=Path)
parser.add_argument('--output', type=Path)
parser.add_argument('--raw', action='store_true', help='Also extract exact raw resources')
parser.add_argument('--scene', action='store_true', help='Export runtime scene.json with decoded textures')
args = parser.parse_args()
db = Database(args.database)
if args.output:
if args.scene:
scene = export_scene(db, args.output)
print(f"{len(scene['vertices'])} vertices, {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures; {args.output}")
return
report = extract(db, args.output, args.raw)
failures = [e for e in report['entries'] if 'unsupported' in e]
print(f'{len(db.entries)} resources; {len(failures)} unsupported conversions; {args.output}')
for fail in failures[:10]: print(f" {fail['type']}:{fail['id']} {fail['unsupported']}")
else:
for e in db.entries: print(f'{e.type} {e.id:10d} {e.offset:10d} {e.size:10d} {e.name}')
if __name__ == '__main__': main()