Recover OMT resources, morph topology, and original engine evidence

This commit is contained in:
UnixNight committed 2026-10-23 17:32:48 -04:00
1 parent d62e2ab4de
commit 3df20ac19a
12 files changed
+1648

No files matched your search

+140
View File
@@ -0,0 +1,140 @@
#!/usr/bin/env python3
"""Read-only PE32 import/export/RTTI evidence inventory, no third-party modules.
Addresses are preferred-base VAs, not relocation-adjusted runtime addresses.
Usage: python3 tools/binary/pe_inspect.py ../Original/OMT2.dll --symbols 'render|timer'
"""
import argparse
import hashlib
import json
from pathlib import Path
import re
import struct
class PE:
def __init__(self, path):
self.path = Path(path)
self.data = self.path.read_bytes()
if self.data[:2] != b'MZ':
raise ValueError('Missing MZ signature')
nt = self.u32(0x3c)
if self.data[nt:nt+4] != b'PE\0\0':
raise ValueError('Missing PE signature')
self.machine, count = struct.unpack_from('<HH', self.data, nt+4)
self.timestamp = self.u32(nt+8)
optsize = self.u16(nt+20)
opt = nt+24
if self.u16(opt) != 0x10b:
raise ValueError('This inspector supports PE32 only')
self.base = self.u32(opt+28)
self.entry = self.base+self.u32(opt+16)
self.header_size = self.u32(opt+60)
self.linker = list(self.data[opt+2:opt+4])
self.directories = [struct.unpack_from('<II', self.data, opt+96+8*i)
for i in range(min(16, self.u32(opt+92)))]
self.sections = []
for i in range(count):
off = opt+optsize+i*40
name = self.data[off:off+8].split(b'\0', 1)[0].decode('ascii')
vs, rva, size, raw = struct.unpack_from('<IIII', self.data, off+8)
self.sections.append(dict(name=name, virtual_size=vs, rva=rva,
raw_size=size, file_offset=raw))
def u16(self, off):
return struct.unpack_from('<H', self.data, off)[0]
def u32(self, off):
return struct.unpack_from('<I', self.data, off)[0]
def offset(self, rva):
if 0 <= rva < self.header_size:
return rva
for s in self.sections:
delta = rva-s['rva']
if 0 <= delta < s['raw_size']:
return s['file_offset']+delta
raise ValueError(f'RVA {rva:#x} has no file data')
def va(self, off):
if 0 <= off < self.header_size:
return self.base+off
for s in self.sections:
delta = off-s['file_offset']
if 0 <= delta < s['raw_size']:
return self.base+s['rva']+delta
raise ValueError(f'Offset {off:#x} has no virtual address')
def cstr(self, off):
end = self.data.index(0, off)
return self.data[off:end].decode('ascii', 'replace')
def exports(self):
rva, size = self.directories[0]
if not rva:
return []
off = self.offset(rva)
ordinal_base, count, names, functions, nameptrs, ordinals = struct.unpack_from(
'<IIIIII', self.data, off+16)
result = []
for i in range(names):
idx = self.u16(self.offset(ordinals)+2*i)
target = self.u32(self.offset(functions)+4*idx)
name = self.cstr(self.offset(self.u32(self.offset(nameptrs)+4*i)))
result.append(dict(name=name, ordinal=ordinal_base+idx,
va=f'0x{self.base+target:08x}', rva=target))
return result
def imports(self):
rva, _ = self.directories[1]
result = []
if not rva:
return result
off = self.offset(rva)
while any(self.data[off:off+20]):
original, _, _, name, first = struct.unpack_from('<IIIII', self.data, off)
dll = self.cstr(self.offset(name))
thunk = self.offset(original or first)
i = 0
while (v := self.u32(thunk+4*i)):
n = f'ordinal:{v & 0xffff}' if v & 0x80000000 else self.cstr(self.offset(v)+2)
result.append(dict(dll=dll, name=n, iat=f'0x{self.base+first+4*i:08x}'))
i += 1
off += 20
return result
def strings(self, pattern):
result = []
regex = re.compile(pattern, re.I)
for m in re.finditer(rb'[\x20-\x7e]{4,}', self.data):
value = m.group().decode('ascii')
if regex.search(value):
result.append(dict(value=value, offset=f'0x{m.start():x}',
va=f'0x{self.va(m.start()):08x}'))
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('binary', type=Path)
parser.add_argument('--symbols', help='Regex filter for imports and exports')
parser.add_argument('--strings', help='Regex filter for ASCII strings with VAs')
args = parser.parse_args()
pe = PE(args.binary)
imports, exports = pe.imports(), pe.exports()
if args.symbols:
regex = re.compile(args.symbols, re.I)
imports = [x for x in imports if regex.search(x['name'])]
exports = [x for x in exports if regex.search(x['name'])]
out = dict(path=str(args.binary), sha256=hashlib.sha256(pe.data).hexdigest(),
machine=hex(pe.machine), linker=pe.linker, timestamp=pe.timestamp,
image_base=hex(pe.base), entry_point=hex(pe.entry),
clr_directory=pe.directories[14], sections=pe.sections,
imports=imports, exports=exports)
if args.strings:
out['strings'] = pe.strings(args.strings)
print(json.dumps(out, indent=2))
if __name__ == '__main__':
main()
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Find MSVC x86 RTTI complete-object locators and candidate vtables.
Output is evidence, not a claim that every candidate is an actual vtable.
Usage: python3 tools/binary/rtti_map.py ../Original/Neutron.exe C3DJimmy
"""
import argparse
import json
import re
import struct
from pe_inspect import PE
def rtti_map(pe, class_name):
marker = ('.?AV'+class_name+'@@').encode('ascii')
at = pe.data.find(marker)
if at < 8:
raise ValueError(f'No type descriptor for {class_name}')
type_va = pe.va(at-8)
result = dict(class_name=class_name, type_descriptor=hex(type_va), vtables=[])
for m in re.finditer(re.escape(struct.pack('<I', type_va)), pe.data):
off = m.start()-12
if off < 0 or off % 4:
continue
sig, displacement, ctor_displacement, _, hierarchy = struct.unpack_from('<IIIII', pe.data, off)
if sig != 0 or displacement > 0x100000:
continue
locator_va = pe.va(off)
for ref in re.finditer(re.escape(struct.pack('<I', locator_va)), pe.data):
if ref.start() % 4:
continue
methods = []
for i in range(512):
pos = ref.start()+4+i*4
if pos+4 > len(pe.data):
break
va = pe.u32(pos)
try:
function_off = pe.offset(va-pe.base)
section = next(s for s in pe.sections if s['file_offset'] <= function_off < s['file_offset']+s['raw_size'])
if section['name'] != '.text':
break
except (ValueError, StopIteration):
break
methods.append(dict(slot=hex(i*4), va=hex(va)))
if methods:
result['vtables'].append(dict(va=hex(pe.va(ref.start()+4)),
complete_object_locator=hex(locator_va), object_displacement=hex(displacement),
hierarchy=hex(hierarchy), methods=methods))
return result
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('binary')
parser.add_argument('class_name')
args = parser.parse_args()
print(json.dumps(rtti_map(PE(args.binary), args.class_name), indent=2))
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Recover supplied 3DS Max ASE mesh animation samples into OMT coordinates."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from pathlib import Path
import re
from omt import FormatError
NUMBER = r'[-+\d.eE]+'
def blocks(text: str, tag: str):
for match in re.finditer(r'\*'+re.escape(tag)+r'\s*\{', text):
start, depth = match.end(), 1
cursor = start
while cursor < len(text) and depth:
if text[cursor] == '{': depth += 1
elif text[cursor] == '}': depth -= 1
cursor += 1
if depth: raise FormatError(f'Unclosed {tag} block')
yield match.start(), text[start:cursor-1]
def vector_table(text: str, tag: str):
values = {}
pattern = r'\*'+tag+r'\s+(\d+)\s+('+NUMBER+r')\s+('+NUMBER+r')\s+('+NUMBER+r')'
for match in re.finditer(pattern, text):
index = int(match[1])
if index in values: raise FormatError(f'Duplicate {tag} index {index}')
values[index] = tuple(float(match[i]) for i in (2,3,4))
if set(values) != set(range(len(values))): raise FormatError(f'Noncontiguous {tag} indices')
if any(not math.isfinite(v) for p in values.values() for v in p): raise FormatError('Nonfinite ASE value')
return [values[i] for i in range(len(values))]
def read_mesh(text: str) -> dict:
vertices = vector_table(text, 'MESH_VERTEX')
tvertices = vector_table(text, 'MESH_TVERT')
faces, tfaces = {}, {}
for match in re.finditer(r'\*MESH_FACE\s+(\d+):\s+A:\s*(\d+)\s+B:\s*(\d+)\s+C:\s*(\d+)', text):
faces[int(match[1])] = tuple(int(match[i]) for i in (2,3,4))
for match in re.finditer(r'\*MESH_TFACE\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)', text):
tfaces[int(match[1])] = tuple(int(match[i]) for i in (2,3,4))
for key, count in (('MESH_NUMVERTEX',len(vertices)),('MESH_NUMFACES',len(faces))):
match = re.search(r'\*'+key+r'\s+(\d+)', text)
if not match or int(match[1]) != count: raise FormatError(f'{key} does not match data')
if set(faces) != set(range(len(faces))): raise FormatError('Noncontiguous face indices')
if tfaces and set(tfaces) != set(faces): raise FormatError('Texture faces do not match faces')
if any(i>=len(vertices) for face in faces.values() for i in face): raise FormatError('ASE vertex outside table')
if any(i>=len(tvertices) for face in tfaces.values() for i in face): raise FormatError('ASE UV outside table')
time = re.search(r'\*TIMEVALUE\s+(-?\d+)',text)
return {'time': int(time[1]) if time else 0,
'vertices': [(x,z,y) for x,y,z in vertices],
'uvs': [(u,-v) for u,v,_ in tvertices],
'faces': [faces[i] for i in range(len(faces))],
'textureFaces': [tfaces[i] for i in range(len(tfaces))] if tfaces else []}
def read_ase(path: Path | str) -> dict:
path = Path(path)
data = path.read_bytes()
text = data.decode('latin1')
if not text.lstrip().startswith('*3DSMAX_ASCIIEXPORT'): raise FormatError('Expected ASE signature')
objects = []
for _, geom in blocks(text, 'GEOMOBJECT'):
name = re.search(r'\*NODE_NAME\s+"([^"]+)"', geom)
meshes = []
animations = list(blocks(geom, 'MESH_ANIMATION'))
if animations:
for _, animation in animations: meshes.extend(read_mesh(body) for _,body in blocks(animation,'MESH'))
else: meshes = [read_mesh(body) for _,body in blocks(geom,'MESH')]
objects.append({'name': name[1] if name else '', 'samples': meshes})
if not objects: raise FormatError('No geometry in ASE')
def setting(name):
match = re.search(r'\*'+name+r'\s+(\d+)',text)
if not match: raise FormatError(f'Missing ASE scene setting {name}')
return int(match[1])
return {'source':path.name, 'sha256':hashlib.sha256(data).hexdigest(),
'framesPerSecond':setting('SCENE_FRAMESPEED'),
'ticksPerFrame':setting('SCENE_TICKSPERFRAME'), 'objects':objects,
'coordinates':'(ASE x,z,y), texture (u,-v); matched OMT duplicate assets'}
def expand_mesh(mesh: dict) -> dict:
result = {'vertices':[], 'uvs':[], 'indices':[], 'normals':[]}
for face_index, face in enumerate(mesh['faces']):
a,b,c = [mesh['vertices'][i] for i in face]
ab,ac = [b[i]-a[i] for i in range(3)], [c[i]-a[i] for i in range(3)]
normal = [ab[1]*ac[2]-ab[2]*ac[1], ab[2]*ac[0]-ab[0]*ac[2], ab[0]*ac[1]-ab[1]*ac[0]]
length = math.sqrt(sum(v*v for v in normal))
normal = [v/length for v in normal] if length else [0.,0.,0.]
for corner,index in enumerate(face):
result['indices'].append(len(result['vertices']))
result['vertices'].append(mesh['vertices'][index])
result['uvs'].append(mesh['uvs'][mesh['textureFaces'][face_index][corner]] if mesh['textureFaces'] else [0.,0.])
result['normals'].append(normal)
return result
def export_ase(path: Path, output: Path, base_path: Path | None = None):
if output.resolve().is_relative_to(path.parent.resolve()) or path.resolve().is_relative_to(output.resolve()):
raise ValueError('Output must be separate from original inputs')
animation = read_ase(path)
base_animation = read_ase(base_path) if base_path else None
if base_animation and len(base_animation['objects']) != len(animation['objects']):
raise FormatError('Morph base and animation object counts differ')
output.mkdir(parents=True, exist_ok=True)
frames = []
for object_index,item in enumerate(animation['objects']):
for sample in item['samples']:
if base_animation:
base = base_animation['objects'][object_index]['samples'][0]
if len(base['vertices']) != len(sample['vertices']):
raise FormatError('Morph sample vertex count differs from base shape')
# Original OMedia3DMorphAnim supplies only per-index vectors to a
# base shape; its polygon/UV topology remains unchanged.
sample = {**sample, 'faces':base['faces'], 'textureFaces':base['textureFaces'],
'uvs':base['uvs']}
if not sample['textureFaces']:
raise FormatError('Mesh has no UVs; supply --base with the original textured morph base')
filename = f"frame_{len(frames):04d}.json"
mesh = expand_mesh(sample)
mesh['meshes'] = [{'name': item['name'],'firstIndex':0,'indexCount':len(mesh['indices']),'material':-1}]
(output/filename).write_text(json.dumps(mesh,separators=(',',':'))+'\n', encoding='utf-8')
frames.append({'node':item['name'],'time':sample['time'], 'path':filename})
manifest = {key:value for key,value in animation.items() if key != 'objects'}
manifest['frames'] = frames
if base_animation:
manifest['morphBase'] = {'source':base_animation['source'], 'sha256':base_animation['sha256'],
'mode':'Original base topology and UV, sample positions by vertex index'}
manifest['normalMode'] = 'Flat normals reconstructed from geometry; original smoothing not yet replicated'
(output/'animation.json').write_text(json.dumps(manifest,indent=2)+'\n', encoding='utf-8')
return manifest
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('ase',type=Path)
p.add_argument('--output',type=Path,required=True)
p.add_argument('--base',type=Path,help='Original textured shape whose topology a morph retains')
args = p.parse_args()
manifest = export_ase(args.ase,args.output,args.base)
print(f"{manifest['source']}: {len(manifest['frames'])} samples at {manifest['framesPerSecond']} fps")
if __name__ == '__main__': main()
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Validate every original OMT index, mesh, material, canvas and RIFF boundary."""
import argparse
from collections import Counter
import json
from pathlib import Path
import struct
from omt import Database, FormatError, read_canvas, read_material, read_shape
def audit(original: Path) -> dict:
counts, conversions, failures = Counter(), Counter(), []
archives = sorted(original.rglob('*.omt'))
for path in archives:
try:
db = Database(path)
except (OSError, FormatError) as ex:
failures.append({'source': str(path), 'error': str(ex)})
continue
for entry in db.entries:
counts[entry.type] += 1
function = {'3DSh': read_shape, '3DMa': read_material, 'Canv': read_canvas}.get(entry.type)
try:
payload = db.payload(entry)
if function:
function(payload)
conversions[entry.type] += 1
elif entry.type == 'Wave':
start = payload.find(b'RIFF')
if start < 0 or payload[start+8:start+12] != b'WAVE':
raise FormatError('No RIFF WAVE header')
size = struct.unpack_from('<I', payload, start+4)[0]+8
if size < 12 or start+size > len(payload): raise FormatError('RIFF size exceeds resource')
conversions['Wave'] += 1
except (FormatError, struct.error) as ex:
failures.append({'source': str(path), 'type': entry.type,
'id': entry.id, 'name': entry.name, 'error': str(ex)})
return {'archives': len(archives), 'counts': dict(counts),
'decoded': dict(conversions), 'failures': failures}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('original', type=Path)
parser.add_argument('--report', type=Path)
args = parser.parse_args()
result = audit(args.original)
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(result, indent=2)+'\n', encoding='utf-8')
print(json.dumps(result, indent=2))
raise SystemExit(bool(result['failures']))
if __name__ == '__main__': main()
+390
View File
@@ -0,0 +1,390 @@
#!/usr/bin/env python3
"""Read-only Open Media Toolkit 0MF2 database recovery. No third-party modules."""
from __future__ import annotations
import argparse
from dataclasses import asdict, dataclass
import hashlib
import json
import math
from pathlib import Path
import re
import struct
import wave
import zlib
class FormatError(ValueError):
"""Malformed or unsupported original resource."""
class Reader:
def __init__(self, data: bytes, position: int = 0):
self.data = data
self.position = position
def take(self, size: int) -> bytes:
if size < 0 or self.position < 0 or self.position + size > len(self.data):
raise FormatError(f"Read outside resource at 0x{self.position:x}, size {size}")
result = self.data[self.position:self.position + size]
self.position += size
return result
def unpack(self, fmt: str):
return struct.unpack(fmt, self.take(struct.calcsize(fmt)))
def u8(self): return self.unpack('>B')[0]
def u16(self): return self.unpack('>H')[0]
def u32(self): return self.unpack('>I')[0]
def f32(self): return self.unpack('>f')[0]
def boolean(self): return self.u16() != 0
def vec3(self): return self.unpack('>3f')
def string(self): return self.take(self.u8()).decode('latin1')
def count(self, minimum_bytes: int):
count = self.u32()
if count > (len(self.data) - self.position) // minimum_bytes:
raise FormatError(f"Impossible count {count} at 0x{self.position - 4:x}")
return count
def link(self):
if not self.boolean(): return None
return {'type': self.take(4).decode('latin1'), 'id': self.u32()}
def end(self):
if self.position != len(self.data):
raise FormatError(f"Unconsumed resource bytes: {len(self.data)-self.position}")
@dataclass(frozen=True)
class Entry:
type: str
id: int
offset: int
size: int
name: str
class Database:
def __init__(self, path: Path | str):
self.path = Path(path)
self.data = self.path.read_bytes()
reader = Reader(self.data)
if reader.take(4) != b'0MF2':
raise FormatError('Expected 0MF2 database signature')
self.table_offset = reader.u32()
if not 8 <= self.table_offset < len(self.data):
raise FormatError('Database table offset is outside file')
reader.position = self.table_offset
groups = reader.count(8)
self.entries = []
keys = set()
for _ in range(groups):
tag = reader.take(4).decode('latin1')
for _ in range(reader.count(13)):
identity, offset, size = reader.unpack('>III')
name = reader.string()
if offset < 8 or offset + size > len(self.data):
raise FormatError(f'Resource {tag}:{identity} outside file')
if (tag, identity) in keys:
raise FormatError(f'Duplicate resource {tag}:{identity}')
keys.add((tag, identity))
self.entries.append(Entry(tag, identity, offset, size, name))
self.table_end = reader.position
def payload(self, entry: Entry) -> bytes:
return self.data[entry.offset:entry.offset + entry.size]
def read_shape(data: bytes) -> dict:
"""Decode 3DSP versions 0..4 using OMT2.dll read_class evidence."""
r = Reader(data)
if r.take(4) != b'3DSP': raise FormatError('Expected 3DSP shape')
version = r.u32()
if version > 4: raise FormatError(f'Unsupported 3DSP version {version}')
bounds = r.unpack('>8f')
legacy = {'unused': r.f32(), 'flag': r.boolean()} if 0 < version < 3 else None
vertices = [r.vec3() for _ in range(r.count(12))]
polygons = []
for _ in range(r.count(1)):
if version <= 2:
flags = {'first': r.boolean(), 'second': r.boolean(), 'legacy': r.u16()}
identifier = r.u32()
extra = r.unpack('>2f') if version else None
legacy_flag = r.u16()
corners = r.u16()
if corners > (len(data)-r.position)//12: raise FormatError('Impossible polygon corner count')
points = []
for _ in range(corners):
index = r.u32()
uv = r.unpack('>2f')
normal_index = r.u32() if version > 1 else index
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
points.append({'vertex': index, 'uv': uv, 'normal': normal_index})
normal = r.vec3()
# Old polygon format has an explicit material-present bool around link.
material = r.link() if r.boolean() else None
polygons.append({'flags': flags, 'id': identifier, 'extra': extra,
'legacy_flag': legacy_flag, 'corners': points,
'normal': normal, 'material': material})
else:
flags = r.u16()
identifier = r.u32()
points = []
for _ in range(r.count(20)):
index, normal_index, colour_index = r.unpack('>3I')
uv = r.unpack('>2f')
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
points.append({'vertex': index, 'normal': normal_index,
'colour': colour_index, 'uv': uv})
polygons.append({'flags': flags, 'id': identifier, 'corners': points,
'normal': r.vec3(), 'material': r.link()})
colours = [r.unpack('>4f') for _ in range(r.count(16))]
normals = [r.vec3() for _ in range(r.count(12))]
additional = [r.unpack('>4f') for _ in range(r.count(16))] if version > 2 else []
r.end()
if not all(math.isfinite(v) for point in vertices for v in point):
raise FormatError('Nonfinite shape coordinate')
return {'version': version, 'bounds': bounds, 'legacy': legacy, 'vertices': vertices,
'polygons': polygons, 'colours': colours, 'normals': normals,
'additional': additional}
def safe_name(value: str) -> str:
return re.sub(r'[^a-zA-Z0-9._-]+', '_', value).strip('._')[:100] or 'unnamed'
def read_material(data: bytes) -> dict:
r = Reader(data)
version = r.u16()
if version != 1: raise FormatError(f'Unsupported material version {version}')
style = r.u16()
if style > 2: raise FormatError(f'Unsupported material style {style}')
shininess, legacy_flag = r.f32(), r.boolean()
colours = {}
if style == 0:
colours['diffuse'] = [v/65535 for v in r.unpack('>3H')]
else:
for name in ('emission', 'diffuse', 'specular', 'ambient'):
colours[name] = [v/65535 for v in r.unpack('>3H')]
legacy_boolean = r.boolean()
texture, texture_override_flag = None, False
if style == 2:
texture_override_flag = r.boolean()
if r.u16() == 1: texture = r.link()
trailing = r.u16()
r.end()
return {'version': version, 'style': style, 'shininess': shininess,
'legacyFlag': legacy_flag, 'colours': colours,
'legacyBoolean': legacy_boolean if style else False, 'textureOverrideFlag': texture_override_flag,
'texture': texture, 'trailing': trailing}
def unpack_row(encoded: bytes, expected_size: int, unit: int) -> bytes:
"""OMT PackBits: LE signed control of unit width, payload in source pixel order."""
r = Reader(encoded)
output = bytearray()
while len(output) < expected_size:
control_bytes = r.take(unit)
# 32-bit variant tests the low signed 16 bits, as original routine does.
control = int.from_bytes(control_bytes[:min(unit, 2)], 'little', signed=True)
if control == -128: continue
count = control + 1 if control >= 0 else 1 - control
if len(output) + count*unit > expected_size:
raise FormatError('RLE run exceeds decoded row')
output.extend(r.take(count*unit) if control >= 0 else r.take(unit)*count)
r.end()
return bytes(output)
def read_canvas(data: bytes) -> dict:
r = Reader(data)
magic = r.take(4)
if magic != b'OmCv': raise FormatError(f'Unsupported canvas signature {magic!r}')
flags = r.u32()
if r.take(4) != b'OmGW': raise FormatError('Missing OmGW pixel header')
width, height, depth = r.unpack('>3H')
packing, key_type, row_bytes = r.u8(), r.u8(), r.u16()
if not width or not height or width*height > 16777216: raise FormatError('Invalid canvas dimensions')
if packing not in (0, 1, 2): raise FormatError('Unknown canvas packing')
if depth not in (8, 16, 32): raise FormatError(f'Unsupported canvas depth {depth}')
has_key = r.boolean()
key = None
if key_type == 0:
rgb = r.unpack('>3H')
if has_key:
channels = [v*(31 if depth == 16 else 255)//65535 for v in rgb]
key = ((channels[0]<<10)|(channels[1]<<5)|channels[2]) if depth == 16 else ((channels[0]<<16)|(channels[1]<<8)|channels[2])
elif has_key: key = r.u32()
palette = None
if r.boolean():
signature = r.take(4)
if signature not in (b'OPa2', b'OmPa'): raise FormatError('Invalid canvas palette signature')
r.string() # Palette name; absent for these originals.
count = r.u32()
if count > 256: raise FormatError('Palette exceeds 256 colours')
palette = [tuple(v & 255 for v in r.unpack('>3H')) for _ in range(count)]
if signature == b'OPa2': r.take(4096) # Legacy palette matching accelerator.
if count <= 2: palette = [(255,255,255), (0,0,0)]
if depth == 8 and palette is None: raise FormatError('Indexed canvas has no palette')
size = r.u32()
compressed = Reader(r.take(size))
r.end()
if row_bytes < width*(depth//8): raise FormatError('Canvas row stride too small')
pixels = bytearray()
for _ in range(height):
row = unpack_row(compressed.take(compressed.u16()), row_bytes, 1 << packing)
for x in range(width):
v = int.from_bytes(row[x*(depth//8):(x+1)*(depth//8)], 'big')
if depth == 8:
if v >= len(palette): raise FormatError('Pixel index outside palette')
rgb = palette[v]
elif depth == 16:
rgb = (((v>>10)&31)<<3, ((v>>5)&31)<<3, (v&31)<<3)
else: rgb = ((v>>16)&255, (v>>8)&255, v&255)
pixels.extend((*rgb, 0 if key is not None and v == key else 255))
compressed.end()
return {'width': width, 'height': height, 'depth': depth, 'flags': flags,
'packing': packing, 'rowBytes': row_bytes, 'colourKey': key,
'rgba': bytes(pixels)}
def write_png(path: Path, width: int, height: int, rgba: bytes):
if len(rgba) != width*height*4: raise ValueError('Incorrect RGBA byte count')
def chunk(tag, payload):
return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag+payload))
scanlines = b''.join(b'\0' + rgba[y*width*4:(y+1)*width*4] for y in range(height))
path.write_bytes(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b''))
def export_scene(database: Database, output: Path):
"""Expanded original corner data; triangle fans preserve source winding."""
validate_output(database, output)
output.mkdir(parents=True, exist_ok=True)
scene = {'source': database.path.name, 'sha256': hashlib.sha256(database.data).hexdigest(),
'coordinates': 'Original OMT world coordinates, unchanged',
'vertices': [], 'uvs': [], 'normals': [], 'indices': [], 'meshes': [],
'materials': [], 'textures': [], 'warnings': []}
for entry in database.entries:
payload = database.payload(entry)
if entry.type == '3DSh':
shape = read_shape(payload)
for poly in shape['polygons']:
first_index = len(scene['indices'])
first_vertex = len(scene['vertices'])
for corner in poly['corners']:
scene['vertices'].append(shape['vertices'][corner['vertex']])
scene['uvs'].append(corner['uv'])
ni = corner['normal']
scene['normals'].append(shape['normals'][ni] if ni < len(shape['normals']) else poly['normal'])
for i in range(1, len(poly['corners'])-1):
scene['indices'].extend((first_vertex, first_vertex+i, first_vertex+i+1))
if len(poly['corners']) > 3:
scene['warnings'].append(f'{entry.id}: polygon with {len(poly["corners"])} corners fan triangulated; concavity not validated')
material = poly['material']['id'] if poly['material'] else -1
if scene['meshes'] and scene['meshes'][-1]['shapeId'] == entry.id and scene['meshes'][-1]['material'] == material:
scene['meshes'][-1]['indexCount'] += len(scene['indices'])-first_index
else:
scene['meshes'].append({'name': entry.name, 'shapeId': entry.id,
'firstIndex': first_index, 'indexCount': len(scene['indices'])-first_index, 'material': material})
elif entry.type == '3DMa':
material = read_material(payload)
scene['materials'].append({'id': entry.id, 'name': entry.name, **material})
elif entry.type == 'Canv':
canvas = read_canvas(payload)
base = f'texture_{entry.id:08x}'
(output / (base + '.rgba')).write_bytes(canvas.pop('rgba'))
rgba = (output / (base + '.rgba')).read_bytes()
write_png(output / (base + '.png'), canvas['width'], canvas['height'], rgba)
scene['textures'].append({'id': entry.id, 'name': entry.name,
'path': base + '.rgba', 'png': base + '.png', **canvas})
(output / 'scene.json').write_text(json.dumps(scene, separators=(',', ':')) + '\n', encoding='utf-8')
return scene
def write_obj(shape: dict, path: Path):
"""Preserve original positions/UVs/polygon normals in a standard interchange file."""
lines = ['# Original OMT coordinates; no transform or guessed smoothing applied']
lines.extend('v ' + ' '.join(format(v, '.9g') for v in p) for p in shape['vertices'])
for poly in shape['polygons']:
lines.extend('vt ' + ' '.join(format(v, '.9g') for v in c['uv']) for c in poly['corners'])
lines.extend('vn ' + ' '.join(format(v, '.9g') for v in p['normal']) for p in shape['polygons'])
uv_index = 1
for normal_index, poly in enumerate(shape['polygons'], 1):
if poly['material']: lines.append(f"usemtl material_{poly['material']['id']}")
corners = []
for corner in poly['corners']:
corners.append(f"{corner['vertex']+1}/{uv_index}/{normal_index}")
uv_index += 1
if len(corners) >= 3: lines.append('f ' + ' '.join(corners))
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
def extract(database: Database, output: Path, raw: bool = False):
output = output.resolve()
validate_output(database, output)
output.mkdir(parents=True, exist_ok=True)
report = {'source': str(database.path), 'sha256': hashlib.sha256(database.data).hexdigest(),
'table_offset': database.table_offset, 'entries': []}
for entry in database.entries:
info = asdict(entry)
payload = database.payload(entry)
base = f'{safe_name(entry.type)}_{entry.id:08x}_{safe_name(entry.name)}'
if raw and entry.type != '0HDR':
(output / (base + '.bin')).write_bytes(payload)
if entry.type == '3DSh':
try:
shape = read_shape(payload)
write_obj(shape, output / (base + '.obj'))
(output / (base + '.json')).write_text(json.dumps(shape), encoding='utf-8')
info.update(vertices=len(shape['vertices']), polygons=len(shape['polygons']))
except FormatError as ex: info['unsupported'] = str(ex)
if entry.type == 'Wave':
start = payload.find(b'RIFF')
if start >= 0 and payload[start+8:start+12] == b'WAVE':
size = struct.unpack_from('<I', payload, start+4)[0] + 8
if start + size <= len(payload):
target = output / (base + '.wav')
target.write_bytes(payload[start:start+size])
try:
with wave.open(str(target)) as wav:
info['audio'] = {'channels': wav.getnchannels(), 'rate': wav.getframerate(),
'sample_width': wav.getsampwidth(), 'frames': wav.getnframes()}
except wave.Error:
info['audio'] = {'encoding': 'non-PCM RIFF preserved unchanged'}
else: info['unsupported'] = 'RIFF size outside resource'
report['entries'].append(info)
(output / 'manifest.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
return report
def validate_output(database: Database, output: Path):
source_directory = database.path.parent.resolve()
target = output.resolve()
if target.is_relative_to(source_directory) or database.path.resolve().is_relative_to(target):
raise ValueError('Output must be separate from original input directory')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('database', type=Path)
parser.add_argument('--output', type=Path)
parser.add_argument('--raw', action='store_true', help='Also extract exact raw resources')
parser.add_argument('--scene', action='store_true', help='Export runtime scene.json with decoded textures')
args = parser.parse_args()
db = Database(args.database)
if args.output:
if args.scene:
scene = export_scene(db, args.output)
print(f"{len(scene['vertices'])} vertices, {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures; {args.output}")
return
report = extract(db, args.output, args.raw)
failures = [e for e in report['entries'] if 'unsupported' in e]
print(f'{len(db.entries)} resources; {len(failures)} unsupported conversions; {args.output}')
for fail in failures[:10]: print(f" {fail['type']}:{fail['id']} {fail['unsupported']}")
else:
for e in db.entries: print(f'{e.type} {e.id:10d} {e.offset:10d} {e.size:10d} {e.name}')
if __name__ == '__main__': main()
+68
View File
@@ -0,0 +1,68 @@
"""Small bounded decoder for the supplied non-interlaced 8-bit PNG textures."""
from pathlib import Path
import struct
import zlib
from omt import FormatError, Reader
def read_png(path: Path) -> dict:
r = Reader(path.read_bytes())
if r.take(8) != b'\x89PNG\r\n\x1a\n': raise FormatError('Invalid PNG signature')
header, palette, transparency = None, None, b''
compressed = bytearray()
ended = False
while r.position < len(r.data):
size = r.u32()
tag, data = r.take(4), r.take(size)
if r.u32() != zlib.crc32(tag+data): raise FormatError('PNG checksum mismatch')
if tag == b'IHDR': header = struct.unpack('>IIBBBBB',data)
elif tag == b'PLTE': palette = data
elif tag == b'tRNS': transparency = data
elif tag == b'IDAT': compressed.extend(data)
elif tag == b'IEND': ended = True; break
elif tag[0] & 32 == 0: raise FormatError(f'Unsupported critical PNG chunk {tag!r}')
if not ended or not header: raise FormatError('Incomplete PNG')
r.end()
width,height,depth,colour,compression,filter_method,interlace = header
if depth != 8 or compression or filter_method or interlace: raise FormatError('Only non-interlaced 8-bit PNG is supported')
channels = {0:1,2:3,3:1,4:2,6:4}.get(colour)
if not channels or not width or not height or width*height > 16777216: raise FormatError('Invalid PNG dimensions/colour')
stride = width*channels
expected = (stride+1)*height
decoder = zlib.decompressobj()
raw = decoder.decompress(bytes(compressed),expected+1)
if len(raw)!=expected or not decoder.eof or decoder.unused_data or decoder.unconsumed_tail:
raise FormatError('Invalid PNG decoded length')
rgba, previous = bytearray(), bytearray(stride)
for y in range(height):
mode = raw[y*(stride+1)]
scan = bytearray(raw[y*(stride+1)+1:(y+1)*(stride+1)])
if mode > 4: raise FormatError('Invalid PNG scanline filter')
for x in range(stride):
a = scan[x-channels] if x>=channels else 0
b = previous[x]
c = previous[x-channels] if x>=channels else 0
if mode==1: predict = a
elif mode==2: predict = b
elif mode==3: predict = (a+b)//2
elif mode==4:
p=a+b-c;pa,pb,pc=abs(p-a),abs(p-b),abs(p-c)
predict = a if pa<=pb and pa<=pc else b if pb<=pc else c
else: predict=0
scan[x] = (scan[x]+predict)&255
for x in range(width):
pixel = scan[x*channels:(x+1)*channels]
if colour==3:
index=pixel[0]
if palette is None or index*3+3>len(palette): raise FormatError('PNG pixel outside palette')
rgba.extend((*palette[index*3:index*3+3],transparency[index] if index<len(transparency) else 255))
elif colour==6: rgba.extend(pixel)
elif colour==2:
alpha=0 if transparency and tuple(pixel)==struct.unpack('>3H',transparency) else 255
rgba.extend((*pixel,alpha))
else:
alpha=pixel[1] if colour==4 else 0 if transparency and pixel[0]==struct.unpack('>H',transparency)[0] else 255
rgba.extend((pixel[0],pixel[0],pixel[0],alpha))
previous=scan
return {'width':width,'height':height,'rgba':bytes(rgba)}
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Prepare local scene/audio assets without modifying the original installation."""
import argparse
import json
from pathlib import Path
from omt import Database, export_scene, extract
from ase import export_ase
from png import read_png
def find_archive(original: Path, name: str) -> Path:
matches = [p for p in (original / 'omt').iterdir()
if p.name.casefold() == (name if name.casefold().endswith('.omt') else name + '.omt').casefold()]
if len(matches) != 1:
raise ValueError(f'Expected exactly one archive for {name}, found {len(matches)}')
return matches[0]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--original', type=Path, default=Path('../Original'))
parser.add_argument('--output', type=Path, default=Path('artifacts/extracted'))
parser.add_argument('--scenes', nargs='+', default=['level1b', 'VR01'])
parser.add_argument('--audio', nargs='*', default=['sounds', 'musicjimmyshouse', 'musicneighborhood'])
parser.add_argument('--animations', nargs='*', default=['jimstop', 'jimrun', 'jimjump'])
args = parser.parse_args()
if args.output.resolve().is_relative_to(args.original.resolve()):
parser.error('Output must be outside the original installation')
for name in args.scenes:
source = find_archive(args.original, name)
scene = export_scene(Database(source), args.output / source.stem)
print(f"{source.name}: {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures")
for name in args.audio:
source = find_archive(args.original, name)
report = extract(Database(source), args.output / source.stem)
count = sum(1 for e in report['entries'] if 'audio' in e)
print(f'{source.name}: {count} audio resources')
if args.animations:
texture_source = next(p for p in (args.original/'png').iterdir() if p.name.casefold()=='jimycarl.png')
texture = read_png(texture_source)
target = args.output/'animations'
target.mkdir(parents=True,exist_ok=True)
(target/'jimycarl.rgba').write_bytes(texture.pop('rgba'))
base = next(p for p in (args.original/'ASE').iterdir() if p.name.casefold()=='jimstop.ase')
for name in args.animations:
matches = [p for p in (args.original/'ASE').iterdir() if p.stem.casefold()==name.casefold() and p.suffix.casefold()=='.ase']
if len(matches)!=1: raise ValueError(f'Expected one ASE for {name}')
manifest = export_ase(matches[0],target/name,base)
manifest['texture'] = {'path':'../jimycarl.rgba', **texture}
(target/name/'animation.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
print(f"{matches[0].name}: {len(manifest['frames'])} animation samples")
if __name__ == '__main__': main()