391 lines
18 KiB
Python
391 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only Open Media Toolkit 0MF2 database recovery. No third-party modules."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import asdict, dataclass
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import wave
|
|
import zlib
|
|
|
|
|
|
class FormatError(ValueError):
|
|
"""Malformed or unsupported original resource."""
|
|
|
|
|
|
class Reader:
|
|
def __init__(self, data: bytes, position: int = 0):
|
|
self.data = data
|
|
self.position = position
|
|
|
|
def take(self, size: int) -> bytes:
|
|
if size < 0 or self.position < 0 or self.position + size > len(self.data):
|
|
raise FormatError(f"Read outside resource at 0x{self.position:x}, size {size}")
|
|
result = self.data[self.position:self.position + size]
|
|
self.position += size
|
|
return result
|
|
|
|
def unpack(self, fmt: str):
|
|
return struct.unpack(fmt, self.take(struct.calcsize(fmt)))
|
|
|
|
def u8(self): return self.unpack('>B')[0]
|
|
def u16(self): return self.unpack('>H')[0]
|
|
def u32(self): return self.unpack('>I')[0]
|
|
def f32(self): return self.unpack('>f')[0]
|
|
def boolean(self): return self.u16() != 0
|
|
def vec3(self): return self.unpack('>3f')
|
|
def string(self): return self.take(self.u8()).decode('latin1')
|
|
|
|
def count(self, minimum_bytes: int):
|
|
count = self.u32()
|
|
if count > (len(self.data) - self.position) // minimum_bytes:
|
|
raise FormatError(f"Impossible count {count} at 0x{self.position - 4:x}")
|
|
return count
|
|
|
|
def link(self):
|
|
if not self.boolean(): return None
|
|
return {'type': self.take(4).decode('latin1'), 'id': self.u32()}
|
|
|
|
def end(self):
|
|
if self.position != len(self.data):
|
|
raise FormatError(f"Unconsumed resource bytes: {len(self.data)-self.position}")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Entry:
|
|
type: str
|
|
id: int
|
|
offset: int
|
|
size: int
|
|
name: str
|
|
|
|
|
|
class Database:
|
|
def __init__(self, path: Path | str):
|
|
self.path = Path(path)
|
|
self.data = self.path.read_bytes()
|
|
reader = Reader(self.data)
|
|
if reader.take(4) != b'0MF2':
|
|
raise FormatError('Expected 0MF2 database signature')
|
|
self.table_offset = reader.u32()
|
|
if not 8 <= self.table_offset < len(self.data):
|
|
raise FormatError('Database table offset is outside file')
|
|
reader.position = self.table_offset
|
|
groups = reader.count(8)
|
|
self.entries = []
|
|
keys = set()
|
|
for _ in range(groups):
|
|
tag = reader.take(4).decode('latin1')
|
|
for _ in range(reader.count(13)):
|
|
identity, offset, size = reader.unpack('>III')
|
|
name = reader.string()
|
|
if offset < 8 or offset + size > len(self.data):
|
|
raise FormatError(f'Resource {tag}:{identity} outside file')
|
|
if (tag, identity) in keys:
|
|
raise FormatError(f'Duplicate resource {tag}:{identity}')
|
|
keys.add((tag, identity))
|
|
self.entries.append(Entry(tag, identity, offset, size, name))
|
|
self.table_end = reader.position
|
|
|
|
def payload(self, entry: Entry) -> bytes:
|
|
return self.data[entry.offset:entry.offset + entry.size]
|
|
|
|
|
|
def read_shape(data: bytes) -> dict:
|
|
"""Decode 3DSP versions 0..4 using OMT2.dll read_class evidence."""
|
|
r = Reader(data)
|
|
if r.take(4) != b'3DSP': raise FormatError('Expected 3DSP shape')
|
|
version = r.u32()
|
|
if version > 4: raise FormatError(f'Unsupported 3DSP version {version}')
|
|
bounds = r.unpack('>8f')
|
|
legacy = {'unused': r.f32(), 'flag': r.boolean()} if 0 < version < 3 else None
|
|
vertices = [r.vec3() for _ in range(r.count(12))]
|
|
polygons = []
|
|
for _ in range(r.count(1)):
|
|
if version <= 2:
|
|
flags = {'first': r.boolean(), 'second': r.boolean(), 'legacy': r.u16()}
|
|
identifier = r.u32()
|
|
extra = r.unpack('>2f') if version else None
|
|
legacy_flag = r.u16()
|
|
corners = r.u16()
|
|
if corners > (len(data)-r.position)//12: raise FormatError('Impossible polygon corner count')
|
|
points = []
|
|
for _ in range(corners):
|
|
index = r.u32()
|
|
uv = r.unpack('>2f')
|
|
normal_index = r.u32() if version > 1 else index
|
|
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
|
|
points.append({'vertex': index, 'uv': uv, 'normal': normal_index})
|
|
normal = r.vec3()
|
|
# Old polygon format has an explicit material-present bool around link.
|
|
material = r.link() if r.boolean() else None
|
|
polygons.append({'flags': flags, 'id': identifier, 'extra': extra,
|
|
'legacy_flag': legacy_flag, 'corners': points,
|
|
'normal': normal, 'material': material})
|
|
else:
|
|
flags = r.u16()
|
|
identifier = r.u32()
|
|
points = []
|
|
for _ in range(r.count(20)):
|
|
index, normal_index, colour_index = r.unpack('>3I')
|
|
uv = r.unpack('>2f')
|
|
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
|
|
points.append({'vertex': index, 'normal': normal_index,
|
|
'colour': colour_index, 'uv': uv})
|
|
polygons.append({'flags': flags, 'id': identifier, 'corners': points,
|
|
'normal': r.vec3(), 'material': r.link()})
|
|
colours = [r.unpack('>4f') for _ in range(r.count(16))]
|
|
normals = [r.vec3() for _ in range(r.count(12))]
|
|
additional = [r.unpack('>4f') for _ in range(r.count(16))] if version > 2 else []
|
|
r.end()
|
|
if not all(math.isfinite(v) for point in vertices for v in point):
|
|
raise FormatError('Nonfinite shape coordinate')
|
|
return {'version': version, 'bounds': bounds, 'legacy': legacy, 'vertices': vertices,
|
|
'polygons': polygons, 'colours': colours, 'normals': normals,
|
|
'additional': additional}
|
|
|
|
|
|
def safe_name(value: str) -> str:
|
|
return re.sub(r'[^a-zA-Z0-9._-]+', '_', value).strip('._')[:100] or 'unnamed'
|
|
|
|
|
|
def read_material(data: bytes) -> dict:
|
|
r = Reader(data)
|
|
version = r.u16()
|
|
if version != 1: raise FormatError(f'Unsupported material version {version}')
|
|
style = r.u16()
|
|
if style > 2: raise FormatError(f'Unsupported material style {style}')
|
|
shininess, legacy_flag = r.f32(), r.boolean()
|
|
colours = {}
|
|
if style == 0:
|
|
colours['diffuse'] = [v/65535 for v in r.unpack('>3H')]
|
|
else:
|
|
for name in ('emission', 'diffuse', 'specular', 'ambient'):
|
|
colours[name] = [v/65535 for v in r.unpack('>3H')]
|
|
legacy_boolean = r.boolean()
|
|
texture, texture_override_flag = None, False
|
|
if style == 2:
|
|
texture_override_flag = r.boolean()
|
|
if r.u16() == 1: texture = r.link()
|
|
trailing = r.u16()
|
|
r.end()
|
|
return {'version': version, 'style': style, 'shininess': shininess,
|
|
'legacyFlag': legacy_flag, 'colours': colours,
|
|
'legacyBoolean': legacy_boolean if style else False, 'textureOverrideFlag': texture_override_flag,
|
|
'texture': texture, 'trailing': trailing}
|
|
|
|
|
|
def unpack_row(encoded: bytes, expected_size: int, unit: int) -> bytes:
|
|
"""OMT PackBits: LE signed control of unit width, payload in source pixel order."""
|
|
r = Reader(encoded)
|
|
output = bytearray()
|
|
while len(output) < expected_size:
|
|
control_bytes = r.take(unit)
|
|
# 32-bit variant tests the low signed 16 bits, as original routine does.
|
|
control = int.from_bytes(control_bytes[:min(unit, 2)], 'little', signed=True)
|
|
if control == -128: continue
|
|
count = control + 1 if control >= 0 else 1 - control
|
|
if len(output) + count*unit > expected_size:
|
|
raise FormatError('RLE run exceeds decoded row')
|
|
output.extend(r.take(count*unit) if control >= 0 else r.take(unit)*count)
|
|
r.end()
|
|
return bytes(output)
|
|
|
|
|
|
def read_canvas(data: bytes) -> dict:
|
|
r = Reader(data)
|
|
magic = r.take(4)
|
|
if magic != b'OmCv': raise FormatError(f'Unsupported canvas signature {magic!r}')
|
|
flags = r.u32()
|
|
if r.take(4) != b'OmGW': raise FormatError('Missing OmGW pixel header')
|
|
width, height, depth = r.unpack('>3H')
|
|
packing, key_type, row_bytes = r.u8(), r.u8(), r.u16()
|
|
if not width or not height or width*height > 16777216: raise FormatError('Invalid canvas dimensions')
|
|
if packing not in (0, 1, 2): raise FormatError('Unknown canvas packing')
|
|
if depth not in (8, 16, 32): raise FormatError(f'Unsupported canvas depth {depth}')
|
|
has_key = r.boolean()
|
|
key = None
|
|
if key_type == 0:
|
|
rgb = r.unpack('>3H')
|
|
if has_key:
|
|
channels = [v*(31 if depth == 16 else 255)//65535 for v in rgb]
|
|
key = ((channels[0]<<10)|(channels[1]<<5)|channels[2]) if depth == 16 else ((channels[0]<<16)|(channels[1]<<8)|channels[2])
|
|
elif has_key: key = r.u32()
|
|
palette = None
|
|
if r.boolean():
|
|
signature = r.take(4)
|
|
if signature not in (b'OPa2', b'OmPa'): raise FormatError('Invalid canvas palette signature')
|
|
r.string() # Palette name; absent for these originals.
|
|
count = r.u32()
|
|
if count > 256: raise FormatError('Palette exceeds 256 colours')
|
|
palette = [tuple(v & 255 for v in r.unpack('>3H')) for _ in range(count)]
|
|
if signature == b'OPa2': r.take(4096) # Legacy palette matching accelerator.
|
|
if count <= 2: palette = [(255,255,255), (0,0,0)]
|
|
if depth == 8 and palette is None: raise FormatError('Indexed canvas has no palette')
|
|
size = r.u32()
|
|
compressed = Reader(r.take(size))
|
|
r.end()
|
|
if row_bytes < width*(depth//8): raise FormatError('Canvas row stride too small')
|
|
pixels = bytearray()
|
|
for _ in range(height):
|
|
row = unpack_row(compressed.take(compressed.u16()), row_bytes, 1 << packing)
|
|
for x in range(width):
|
|
v = int.from_bytes(row[x*(depth//8):(x+1)*(depth//8)], 'big')
|
|
if depth == 8:
|
|
if v >= len(palette): raise FormatError('Pixel index outside palette')
|
|
rgb = palette[v]
|
|
elif depth == 16:
|
|
rgb = (((v>>10)&31)<<3, ((v>>5)&31)<<3, (v&31)<<3)
|
|
else: rgb = ((v>>16)&255, (v>>8)&255, v&255)
|
|
pixels.extend((*rgb, 0 if key is not None and v == key else 255))
|
|
compressed.end()
|
|
return {'width': width, 'height': height, 'depth': depth, 'flags': flags,
|
|
'packing': packing, 'rowBytes': row_bytes, 'colourKey': key,
|
|
'rgba': bytes(pixels)}
|
|
|
|
|
|
def write_png(path: Path, width: int, height: int, rgba: bytes):
|
|
if len(rgba) != width*height*4: raise ValueError('Incorrect RGBA byte count')
|
|
def chunk(tag, payload):
|
|
return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag+payload))
|
|
scanlines = b''.join(b'\0' + rgba[y*width*4:(y+1)*width*4] for y in range(height))
|
|
path.write_bytes(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0))
|
|
+ chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b''))
|
|
|
|
|
|
def export_scene(database: Database, output: Path):
|
|
"""Expanded original corner data; triangle fans preserve source winding."""
|
|
validate_output(database, output)
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
scene = {'source': database.path.name, 'sha256': hashlib.sha256(database.data).hexdigest(),
|
|
'coordinates': 'Original OMT world coordinates, unchanged',
|
|
'vertices': [], 'uvs': [], 'normals': [], 'indices': [], 'meshes': [],
|
|
'materials': [], 'textures': [], 'warnings': []}
|
|
for entry in database.entries:
|
|
payload = database.payload(entry)
|
|
if entry.type == '3DSh':
|
|
shape = read_shape(payload)
|
|
for poly in shape['polygons']:
|
|
first_index = len(scene['indices'])
|
|
first_vertex = len(scene['vertices'])
|
|
for corner in poly['corners']:
|
|
scene['vertices'].append(shape['vertices'][corner['vertex']])
|
|
scene['uvs'].append(corner['uv'])
|
|
ni = corner['normal']
|
|
scene['normals'].append(shape['normals'][ni] if ni < len(shape['normals']) else poly['normal'])
|
|
for i in range(1, len(poly['corners'])-1):
|
|
scene['indices'].extend((first_vertex, first_vertex+i, first_vertex+i+1))
|
|
if len(poly['corners']) > 3:
|
|
scene['warnings'].append(f'{entry.id}: polygon with {len(poly["corners"])} corners fan triangulated; concavity not validated')
|
|
material = poly['material']['id'] if poly['material'] else -1
|
|
if scene['meshes'] and scene['meshes'][-1]['shapeId'] == entry.id and scene['meshes'][-1]['material'] == material:
|
|
scene['meshes'][-1]['indexCount'] += len(scene['indices'])-first_index
|
|
else:
|
|
scene['meshes'].append({'name': entry.name, 'shapeId': entry.id,
|
|
'firstIndex': first_index, 'indexCount': len(scene['indices'])-first_index, 'material': material})
|
|
elif entry.type == '3DMa':
|
|
material = read_material(payload)
|
|
scene['materials'].append({'id': entry.id, 'name': entry.name, **material})
|
|
elif entry.type == 'Canv':
|
|
canvas = read_canvas(payload)
|
|
base = f'texture_{entry.id:08x}'
|
|
(output / (base + '.rgba')).write_bytes(canvas.pop('rgba'))
|
|
rgba = (output / (base + '.rgba')).read_bytes()
|
|
write_png(output / (base + '.png'), canvas['width'], canvas['height'], rgba)
|
|
scene['textures'].append({'id': entry.id, 'name': entry.name,
|
|
'path': base + '.rgba', 'png': base + '.png', **canvas})
|
|
(output / 'scene.json').write_text(json.dumps(scene, separators=(',', ':')) + '\n', encoding='utf-8')
|
|
return scene
|
|
|
|
|
|
def write_obj(shape: dict, path: Path):
|
|
"""Preserve original positions/UVs/polygon normals in a standard interchange file."""
|
|
lines = ['# Original OMT coordinates; no transform or guessed smoothing applied']
|
|
lines.extend('v ' + ' '.join(format(v, '.9g') for v in p) for p in shape['vertices'])
|
|
for poly in shape['polygons']:
|
|
lines.extend('vt ' + ' '.join(format(v, '.9g') for v in c['uv']) for c in poly['corners'])
|
|
lines.extend('vn ' + ' '.join(format(v, '.9g') for v in p['normal']) for p in shape['polygons'])
|
|
uv_index = 1
|
|
for normal_index, poly in enumerate(shape['polygons'], 1):
|
|
if poly['material']: lines.append(f"usemtl material_{poly['material']['id']}")
|
|
corners = []
|
|
for corner in poly['corners']:
|
|
corners.append(f"{corner['vertex']+1}/{uv_index}/{normal_index}")
|
|
uv_index += 1
|
|
if len(corners) >= 3: lines.append('f ' + ' '.join(corners))
|
|
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
|
|
|
|
|
def extract(database: Database, output: Path, raw: bool = False):
|
|
output = output.resolve()
|
|
validate_output(database, output)
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
report = {'source': str(database.path), 'sha256': hashlib.sha256(database.data).hexdigest(),
|
|
'table_offset': database.table_offset, 'entries': []}
|
|
for entry in database.entries:
|
|
info = asdict(entry)
|
|
payload = database.payload(entry)
|
|
base = f'{safe_name(entry.type)}_{entry.id:08x}_{safe_name(entry.name)}'
|
|
if raw and entry.type != '0HDR':
|
|
(output / (base + '.bin')).write_bytes(payload)
|
|
if entry.type == '3DSh':
|
|
try:
|
|
shape = read_shape(payload)
|
|
write_obj(shape, output / (base + '.obj'))
|
|
(output / (base + '.json')).write_text(json.dumps(shape), encoding='utf-8')
|
|
info.update(vertices=len(shape['vertices']), polygons=len(shape['polygons']))
|
|
except FormatError as ex: info['unsupported'] = str(ex)
|
|
if entry.type == 'Wave':
|
|
start = payload.find(b'RIFF')
|
|
if start >= 0 and payload[start+8:start+12] == b'WAVE':
|
|
size = struct.unpack_from('<I', payload, start+4)[0] + 8
|
|
if start + size <= len(payload):
|
|
target = output / (base + '.wav')
|
|
target.write_bytes(payload[start:start+size])
|
|
try:
|
|
with wave.open(str(target)) as wav:
|
|
info['audio'] = {'channels': wav.getnchannels(), 'rate': wav.getframerate(),
|
|
'sample_width': wav.getsampwidth(), 'frames': wav.getnframes()}
|
|
except wave.Error:
|
|
info['audio'] = {'encoding': 'non-PCM RIFF preserved unchanged'}
|
|
else: info['unsupported'] = 'RIFF size outside resource'
|
|
report['entries'].append(info)
|
|
(output / 'manifest.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
return report
|
|
|
|
|
|
def validate_output(database: Database, output: Path):
|
|
source_directory = database.path.parent.resolve()
|
|
target = output.resolve()
|
|
if target.is_relative_to(source_directory) or database.path.resolve().is_relative_to(target):
|
|
raise ValueError('Output must be separate from original input directory')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('database', type=Path)
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--raw', action='store_true', help='Also extract exact raw resources')
|
|
parser.add_argument('--scene', action='store_true', help='Export runtime scene.json with decoded textures')
|
|
args = parser.parse_args()
|
|
db = Database(args.database)
|
|
if args.output:
|
|
if args.scene:
|
|
scene = export_scene(db, args.output)
|
|
print(f"{len(scene['vertices'])} vertices, {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures; {args.output}")
|
|
return
|
|
report = extract(db, args.output, args.raw)
|
|
failures = [e for e in report['entries'] if 'unsupported' in e]
|
|
print(f'{len(db.entries)} resources; {len(failures)} unsupported conversions; {args.output}')
|
|
for fail in failures[:10]: print(f" {fail['type']}:{fail['id']} {fail['unsupported']}")
|
|
else:
|
|
for e in db.entries: print(f'{e.type} {e.id:10d} {e.offset:10d} {e.size:10d} {e.name}')
|
|
|
|
|
|
if __name__ == '__main__': main()
|