Recover OMT resources, morph topology, and original engine evidence
This commit is contained in:
1 parent
d62e2ab4de
commit
3df20ac19a
12 files changed
+1648
No files matched your search
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recover supplied 3DS Max ASE mesh animation samples into OMT coordinates."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from omt import FormatError
|
||||
|
||||
NUMBER = r'[-+\d.eE]+'
|
||||
|
||||
|
||||
def blocks(text: str, tag: str):
|
||||
for match in re.finditer(r'\*'+re.escape(tag)+r'\s*\{', text):
|
||||
start, depth = match.end(), 1
|
||||
cursor = start
|
||||
while cursor < len(text) and depth:
|
||||
if text[cursor] == '{': depth += 1
|
||||
elif text[cursor] == '}': depth -= 1
|
||||
cursor += 1
|
||||
if depth: raise FormatError(f'Unclosed {tag} block')
|
||||
yield match.start(), text[start:cursor-1]
|
||||
|
||||
|
||||
def vector_table(text: str, tag: str):
|
||||
values = {}
|
||||
pattern = r'\*'+tag+r'\s+(\d+)\s+('+NUMBER+r')\s+('+NUMBER+r')\s+('+NUMBER+r')'
|
||||
for match in re.finditer(pattern, text):
|
||||
index = int(match[1])
|
||||
if index in values: raise FormatError(f'Duplicate {tag} index {index}')
|
||||
values[index] = tuple(float(match[i]) for i in (2,3,4))
|
||||
if set(values) != set(range(len(values))): raise FormatError(f'Noncontiguous {tag} indices')
|
||||
if any(not math.isfinite(v) for p in values.values() for v in p): raise FormatError('Nonfinite ASE value')
|
||||
return [values[i] for i in range(len(values))]
|
||||
|
||||
|
||||
def read_mesh(text: str) -> dict:
|
||||
vertices = vector_table(text, 'MESH_VERTEX')
|
||||
tvertices = vector_table(text, 'MESH_TVERT')
|
||||
faces, tfaces = {}, {}
|
||||
for match in re.finditer(r'\*MESH_FACE\s+(\d+):\s+A:\s*(\d+)\s+B:\s*(\d+)\s+C:\s*(\d+)', text):
|
||||
faces[int(match[1])] = tuple(int(match[i]) for i in (2,3,4))
|
||||
for match in re.finditer(r'\*MESH_TFACE\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)', text):
|
||||
tfaces[int(match[1])] = tuple(int(match[i]) for i in (2,3,4))
|
||||
for key, count in (('MESH_NUMVERTEX',len(vertices)),('MESH_NUMFACES',len(faces))):
|
||||
match = re.search(r'\*'+key+r'\s+(\d+)', text)
|
||||
if not match or int(match[1]) != count: raise FormatError(f'{key} does not match data')
|
||||
if set(faces) != set(range(len(faces))): raise FormatError('Noncontiguous face indices')
|
||||
if tfaces and set(tfaces) != set(faces): raise FormatError('Texture faces do not match faces')
|
||||
if any(i>=len(vertices) for face in faces.values() for i in face): raise FormatError('ASE vertex outside table')
|
||||
if any(i>=len(tvertices) for face in tfaces.values() for i in face): raise FormatError('ASE UV outside table')
|
||||
time = re.search(r'\*TIMEVALUE\s+(-?\d+)',text)
|
||||
return {'time': int(time[1]) if time else 0,
|
||||
'vertices': [(x,z,y) for x,y,z in vertices],
|
||||
'uvs': [(u,-v) for u,v,_ in tvertices],
|
||||
'faces': [faces[i] for i in range(len(faces))],
|
||||
'textureFaces': [tfaces[i] for i in range(len(tfaces))] if tfaces else []}
|
||||
|
||||
|
||||
def read_ase(path: Path | str) -> dict:
|
||||
path = Path(path)
|
||||
data = path.read_bytes()
|
||||
text = data.decode('latin1')
|
||||
if not text.lstrip().startswith('*3DSMAX_ASCIIEXPORT'): raise FormatError('Expected ASE signature')
|
||||
objects = []
|
||||
for _, geom in blocks(text, 'GEOMOBJECT'):
|
||||
name = re.search(r'\*NODE_NAME\s+"([^"]+)"', geom)
|
||||
meshes = []
|
||||
animations = list(blocks(geom, 'MESH_ANIMATION'))
|
||||
if animations:
|
||||
for _, animation in animations: meshes.extend(read_mesh(body) for _,body in blocks(animation,'MESH'))
|
||||
else: meshes = [read_mesh(body) for _,body in blocks(geom,'MESH')]
|
||||
objects.append({'name': name[1] if name else '', 'samples': meshes})
|
||||
if not objects: raise FormatError('No geometry in ASE')
|
||||
def setting(name):
|
||||
match = re.search(r'\*'+name+r'\s+(\d+)',text)
|
||||
if not match: raise FormatError(f'Missing ASE scene setting {name}')
|
||||
return int(match[1])
|
||||
return {'source':path.name, 'sha256':hashlib.sha256(data).hexdigest(),
|
||||
'framesPerSecond':setting('SCENE_FRAMESPEED'),
|
||||
'ticksPerFrame':setting('SCENE_TICKSPERFRAME'), 'objects':objects,
|
||||
'coordinates':'(ASE x,z,y), texture (u,-v); matched OMT duplicate assets'}
|
||||
|
||||
|
||||
def expand_mesh(mesh: dict) -> dict:
|
||||
result = {'vertices':[], 'uvs':[], 'indices':[], 'normals':[]}
|
||||
for face_index, face in enumerate(mesh['faces']):
|
||||
a,b,c = [mesh['vertices'][i] for i in face]
|
||||
ab,ac = [b[i]-a[i] for i in range(3)], [c[i]-a[i] for i in range(3)]
|
||||
normal = [ab[1]*ac[2]-ab[2]*ac[1], ab[2]*ac[0]-ab[0]*ac[2], ab[0]*ac[1]-ab[1]*ac[0]]
|
||||
length = math.sqrt(sum(v*v for v in normal))
|
||||
normal = [v/length for v in normal] if length else [0.,0.,0.]
|
||||
for corner,index in enumerate(face):
|
||||
result['indices'].append(len(result['vertices']))
|
||||
result['vertices'].append(mesh['vertices'][index])
|
||||
result['uvs'].append(mesh['uvs'][mesh['textureFaces'][face_index][corner]] if mesh['textureFaces'] else [0.,0.])
|
||||
result['normals'].append(normal)
|
||||
return result
|
||||
|
||||
|
||||
def export_ase(path: Path, output: Path, base_path: Path | None = None):
|
||||
if output.resolve().is_relative_to(path.parent.resolve()) or path.resolve().is_relative_to(output.resolve()):
|
||||
raise ValueError('Output must be separate from original inputs')
|
||||
animation = read_ase(path)
|
||||
base_animation = read_ase(base_path) if base_path else None
|
||||
if base_animation and len(base_animation['objects']) != len(animation['objects']):
|
||||
raise FormatError('Morph base and animation object counts differ')
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
frames = []
|
||||
for object_index,item in enumerate(animation['objects']):
|
||||
for sample in item['samples']:
|
||||
if base_animation:
|
||||
base = base_animation['objects'][object_index]['samples'][0]
|
||||
if len(base['vertices']) != len(sample['vertices']):
|
||||
raise FormatError('Morph sample vertex count differs from base shape')
|
||||
# Original OMedia3DMorphAnim supplies only per-index vectors to a
|
||||
# base shape; its polygon/UV topology remains unchanged.
|
||||
sample = {**sample, 'faces':base['faces'], 'textureFaces':base['textureFaces'],
|
||||
'uvs':base['uvs']}
|
||||
if not sample['textureFaces']:
|
||||
raise FormatError('Mesh has no UVs; supply --base with the original textured morph base')
|
||||
filename = f"frame_{len(frames):04d}.json"
|
||||
mesh = expand_mesh(sample)
|
||||
mesh['meshes'] = [{'name': item['name'],'firstIndex':0,'indexCount':len(mesh['indices']),'material':-1}]
|
||||
(output/filename).write_text(json.dumps(mesh,separators=(',',':'))+'\n', encoding='utf-8')
|
||||
frames.append({'node':item['name'],'time':sample['time'], 'path':filename})
|
||||
manifest = {key:value for key,value in animation.items() if key != 'objects'}
|
||||
manifest['frames'] = frames
|
||||
if base_animation:
|
||||
manifest['morphBase'] = {'source':base_animation['source'], 'sha256':base_animation['sha256'],
|
||||
'mode':'Original base topology and UV, sample positions by vertex index'}
|
||||
manifest['normalMode'] = 'Flat normals reconstructed from geometry; original smoothing not yet replicated'
|
||||
(output/'animation.json').write_text(json.dumps(manifest,indent=2)+'\n', encoding='utf-8')
|
||||
return manifest
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('ase',type=Path)
|
||||
p.add_argument('--output',type=Path,required=True)
|
||||
p.add_argument('--base',type=Path,help='Original textured shape whose topology a morph retains')
|
||||
args = p.parse_args()
|
||||
manifest = export_ase(args.ase,args.output,args.base)
|
||||
print(f"{manifest['source']}: {len(manifest['frames'])} samples at {manifest['framesPerSecond']} fps")
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate every original OMT index, mesh, material, canvas and RIFF boundary."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
from omt import Database, FormatError, read_canvas, read_material, read_shape
|
||||
|
||||
|
||||
def audit(original: Path) -> dict:
|
||||
counts, conversions, failures = Counter(), Counter(), []
|
||||
archives = sorted(original.rglob('*.omt'))
|
||||
for path in archives:
|
||||
try:
|
||||
db = Database(path)
|
||||
except (OSError, FormatError) as ex:
|
||||
failures.append({'source': str(path), 'error': str(ex)})
|
||||
continue
|
||||
for entry in db.entries:
|
||||
counts[entry.type] += 1
|
||||
function = {'3DSh': read_shape, '3DMa': read_material, 'Canv': read_canvas}.get(entry.type)
|
||||
try:
|
||||
payload = db.payload(entry)
|
||||
if function:
|
||||
function(payload)
|
||||
conversions[entry.type] += 1
|
||||
elif entry.type == 'Wave':
|
||||
start = payload.find(b'RIFF')
|
||||
if start < 0 or payload[start+8:start+12] != b'WAVE':
|
||||
raise FormatError('No RIFF WAVE header')
|
||||
size = struct.unpack_from('<I', payload, start+4)[0]+8
|
||||
if size < 12 or start+size > len(payload): raise FormatError('RIFF size exceeds resource')
|
||||
conversions['Wave'] += 1
|
||||
except (FormatError, struct.error) as ex:
|
||||
failures.append({'source': str(path), 'type': entry.type,
|
||||
'id': entry.id, 'name': entry.name, 'error': str(ex)})
|
||||
return {'archives': len(archives), 'counts': dict(counts),
|
||||
'decoded': dict(conversions), 'failures': failures}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('original', type=Path)
|
||||
parser.add_argument('--report', type=Path)
|
||||
args = parser.parse_args()
|
||||
result = audit(args.original)
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(result, indent=2)+'\n', encoding='utf-8')
|
||||
print(json.dumps(result, indent=2))
|
||||
raise SystemExit(bool(result['failures']))
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,390 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only Open Media Toolkit 0MF2 database recovery. No third-party modules."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import asdict, dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import wave
|
||||
import zlib
|
||||
|
||||
|
||||
class FormatError(ValueError):
|
||||
"""Malformed or unsupported original resource."""
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, data: bytes, position: int = 0):
|
||||
self.data = data
|
||||
self.position = position
|
||||
|
||||
def take(self, size: int) -> bytes:
|
||||
if size < 0 or self.position < 0 or self.position + size > len(self.data):
|
||||
raise FormatError(f"Read outside resource at 0x{self.position:x}, size {size}")
|
||||
result = self.data[self.position:self.position + size]
|
||||
self.position += size
|
||||
return result
|
||||
|
||||
def unpack(self, fmt: str):
|
||||
return struct.unpack(fmt, self.take(struct.calcsize(fmt)))
|
||||
|
||||
def u8(self): return self.unpack('>B')[0]
|
||||
def u16(self): return self.unpack('>H')[0]
|
||||
def u32(self): return self.unpack('>I')[0]
|
||||
def f32(self): return self.unpack('>f')[0]
|
||||
def boolean(self): return self.u16() != 0
|
||||
def vec3(self): return self.unpack('>3f')
|
||||
def string(self): return self.take(self.u8()).decode('latin1')
|
||||
|
||||
def count(self, minimum_bytes: int):
|
||||
count = self.u32()
|
||||
if count > (len(self.data) - self.position) // minimum_bytes:
|
||||
raise FormatError(f"Impossible count {count} at 0x{self.position - 4:x}")
|
||||
return count
|
||||
|
||||
def link(self):
|
||||
if not self.boolean(): return None
|
||||
return {'type': self.take(4).decode('latin1'), 'id': self.u32()}
|
||||
|
||||
def end(self):
|
||||
if self.position != len(self.data):
|
||||
raise FormatError(f"Unconsumed resource bytes: {len(self.data)-self.position}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Entry:
|
||||
type: str
|
||||
id: int
|
||||
offset: int
|
||||
size: int
|
||||
name: str
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, path: Path | str):
|
||||
self.path = Path(path)
|
||||
self.data = self.path.read_bytes()
|
||||
reader = Reader(self.data)
|
||||
if reader.take(4) != b'0MF2':
|
||||
raise FormatError('Expected 0MF2 database signature')
|
||||
self.table_offset = reader.u32()
|
||||
if not 8 <= self.table_offset < len(self.data):
|
||||
raise FormatError('Database table offset is outside file')
|
||||
reader.position = self.table_offset
|
||||
groups = reader.count(8)
|
||||
self.entries = []
|
||||
keys = set()
|
||||
for _ in range(groups):
|
||||
tag = reader.take(4).decode('latin1')
|
||||
for _ in range(reader.count(13)):
|
||||
identity, offset, size = reader.unpack('>III')
|
||||
name = reader.string()
|
||||
if offset < 8 or offset + size > len(self.data):
|
||||
raise FormatError(f'Resource {tag}:{identity} outside file')
|
||||
if (tag, identity) in keys:
|
||||
raise FormatError(f'Duplicate resource {tag}:{identity}')
|
||||
keys.add((tag, identity))
|
||||
self.entries.append(Entry(tag, identity, offset, size, name))
|
||||
self.table_end = reader.position
|
||||
|
||||
def payload(self, entry: Entry) -> bytes:
|
||||
return self.data[entry.offset:entry.offset + entry.size]
|
||||
|
||||
|
||||
def read_shape(data: bytes) -> dict:
|
||||
"""Decode 3DSP versions 0..4 using OMT2.dll read_class evidence."""
|
||||
r = Reader(data)
|
||||
if r.take(4) != b'3DSP': raise FormatError('Expected 3DSP shape')
|
||||
version = r.u32()
|
||||
if version > 4: raise FormatError(f'Unsupported 3DSP version {version}')
|
||||
bounds = r.unpack('>8f')
|
||||
legacy = {'unused': r.f32(), 'flag': r.boolean()} if 0 < version < 3 else None
|
||||
vertices = [r.vec3() for _ in range(r.count(12))]
|
||||
polygons = []
|
||||
for _ in range(r.count(1)):
|
||||
if version <= 2:
|
||||
flags = {'first': r.boolean(), 'second': r.boolean(), 'legacy': r.u16()}
|
||||
identifier = r.u32()
|
||||
extra = r.unpack('>2f') if version else None
|
||||
legacy_flag = r.u16()
|
||||
corners = r.u16()
|
||||
if corners > (len(data)-r.position)//12: raise FormatError('Impossible polygon corner count')
|
||||
points = []
|
||||
for _ in range(corners):
|
||||
index = r.u32()
|
||||
uv = r.unpack('>2f')
|
||||
normal_index = r.u32() if version > 1 else index
|
||||
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
|
||||
points.append({'vertex': index, 'uv': uv, 'normal': normal_index})
|
||||
normal = r.vec3()
|
||||
# Old polygon format has an explicit material-present bool around link.
|
||||
material = r.link() if r.boolean() else None
|
||||
polygons.append({'flags': flags, 'id': identifier, 'extra': extra,
|
||||
'legacy_flag': legacy_flag, 'corners': points,
|
||||
'normal': normal, 'material': material})
|
||||
else:
|
||||
flags = r.u16()
|
||||
identifier = r.u32()
|
||||
points = []
|
||||
for _ in range(r.count(20)):
|
||||
index, normal_index, colour_index = r.unpack('>3I')
|
||||
uv = r.unpack('>2f')
|
||||
if index >= len(vertices): raise FormatError(f'Invalid vertex index {index}')
|
||||
points.append({'vertex': index, 'normal': normal_index,
|
||||
'colour': colour_index, 'uv': uv})
|
||||
polygons.append({'flags': flags, 'id': identifier, 'corners': points,
|
||||
'normal': r.vec3(), 'material': r.link()})
|
||||
colours = [r.unpack('>4f') for _ in range(r.count(16))]
|
||||
normals = [r.vec3() for _ in range(r.count(12))]
|
||||
additional = [r.unpack('>4f') for _ in range(r.count(16))] if version > 2 else []
|
||||
r.end()
|
||||
if not all(math.isfinite(v) for point in vertices for v in point):
|
||||
raise FormatError('Nonfinite shape coordinate')
|
||||
return {'version': version, 'bounds': bounds, 'legacy': legacy, 'vertices': vertices,
|
||||
'polygons': polygons, 'colours': colours, 'normals': normals,
|
||||
'additional': additional}
|
||||
|
||||
|
||||
def safe_name(value: str) -> str:
|
||||
return re.sub(r'[^a-zA-Z0-9._-]+', '_', value).strip('._')[:100] or 'unnamed'
|
||||
|
||||
|
||||
def read_material(data: bytes) -> dict:
|
||||
r = Reader(data)
|
||||
version = r.u16()
|
||||
if version != 1: raise FormatError(f'Unsupported material version {version}')
|
||||
style = r.u16()
|
||||
if style > 2: raise FormatError(f'Unsupported material style {style}')
|
||||
shininess, legacy_flag = r.f32(), r.boolean()
|
||||
colours = {}
|
||||
if style == 0:
|
||||
colours['diffuse'] = [v/65535 for v in r.unpack('>3H')]
|
||||
else:
|
||||
for name in ('emission', 'diffuse', 'specular', 'ambient'):
|
||||
colours[name] = [v/65535 for v in r.unpack('>3H')]
|
||||
legacy_boolean = r.boolean()
|
||||
texture, texture_override_flag = None, False
|
||||
if style == 2:
|
||||
texture_override_flag = r.boolean()
|
||||
if r.u16() == 1: texture = r.link()
|
||||
trailing = r.u16()
|
||||
r.end()
|
||||
return {'version': version, 'style': style, 'shininess': shininess,
|
||||
'legacyFlag': legacy_flag, 'colours': colours,
|
||||
'legacyBoolean': legacy_boolean if style else False, 'textureOverrideFlag': texture_override_flag,
|
||||
'texture': texture, 'trailing': trailing}
|
||||
|
||||
|
||||
def unpack_row(encoded: bytes, expected_size: int, unit: int) -> bytes:
|
||||
"""OMT PackBits: LE signed control of unit width, payload in source pixel order."""
|
||||
r = Reader(encoded)
|
||||
output = bytearray()
|
||||
while len(output) < expected_size:
|
||||
control_bytes = r.take(unit)
|
||||
# 32-bit variant tests the low signed 16 bits, as original routine does.
|
||||
control = int.from_bytes(control_bytes[:min(unit, 2)], 'little', signed=True)
|
||||
if control == -128: continue
|
||||
count = control + 1 if control >= 0 else 1 - control
|
||||
if len(output) + count*unit > expected_size:
|
||||
raise FormatError('RLE run exceeds decoded row')
|
||||
output.extend(r.take(count*unit) if control >= 0 else r.take(unit)*count)
|
||||
r.end()
|
||||
return bytes(output)
|
||||
|
||||
|
||||
def read_canvas(data: bytes) -> dict:
|
||||
r = Reader(data)
|
||||
magic = r.take(4)
|
||||
if magic != b'OmCv': raise FormatError(f'Unsupported canvas signature {magic!r}')
|
||||
flags = r.u32()
|
||||
if r.take(4) != b'OmGW': raise FormatError('Missing OmGW pixel header')
|
||||
width, height, depth = r.unpack('>3H')
|
||||
packing, key_type, row_bytes = r.u8(), r.u8(), r.u16()
|
||||
if not width or not height or width*height > 16777216: raise FormatError('Invalid canvas dimensions')
|
||||
if packing not in (0, 1, 2): raise FormatError('Unknown canvas packing')
|
||||
if depth not in (8, 16, 32): raise FormatError(f'Unsupported canvas depth {depth}')
|
||||
has_key = r.boolean()
|
||||
key = None
|
||||
if key_type == 0:
|
||||
rgb = r.unpack('>3H')
|
||||
if has_key:
|
||||
channels = [v*(31 if depth == 16 else 255)//65535 for v in rgb]
|
||||
key = ((channels[0]<<10)|(channels[1]<<5)|channels[2]) if depth == 16 else ((channels[0]<<16)|(channels[1]<<8)|channels[2])
|
||||
elif has_key: key = r.u32()
|
||||
palette = None
|
||||
if r.boolean():
|
||||
signature = r.take(4)
|
||||
if signature not in (b'OPa2', b'OmPa'): raise FormatError('Invalid canvas palette signature')
|
||||
r.string() # Palette name; absent for these originals.
|
||||
count = r.u32()
|
||||
if count > 256: raise FormatError('Palette exceeds 256 colours')
|
||||
palette = [tuple(v & 255 for v in r.unpack('>3H')) for _ in range(count)]
|
||||
if signature == b'OPa2': r.take(4096) # Legacy palette matching accelerator.
|
||||
if count <= 2: palette = [(255,255,255), (0,0,0)]
|
||||
if depth == 8 and palette is None: raise FormatError('Indexed canvas has no palette')
|
||||
size = r.u32()
|
||||
compressed = Reader(r.take(size))
|
||||
r.end()
|
||||
if row_bytes < width*(depth//8): raise FormatError('Canvas row stride too small')
|
||||
pixels = bytearray()
|
||||
for _ in range(height):
|
||||
row = unpack_row(compressed.take(compressed.u16()), row_bytes, 1 << packing)
|
||||
for x in range(width):
|
||||
v = int.from_bytes(row[x*(depth//8):(x+1)*(depth//8)], 'big')
|
||||
if depth == 8:
|
||||
if v >= len(palette): raise FormatError('Pixel index outside palette')
|
||||
rgb = palette[v]
|
||||
elif depth == 16:
|
||||
rgb = (((v>>10)&31)<<3, ((v>>5)&31)<<3, (v&31)<<3)
|
||||
else: rgb = ((v>>16)&255, (v>>8)&255, v&255)
|
||||
pixels.extend((*rgb, 0 if key is not None and v == key else 255))
|
||||
compressed.end()
|
||||
return {'width': width, 'height': height, 'depth': depth, 'flags': flags,
|
||||
'packing': packing, 'rowBytes': row_bytes, 'colourKey': key,
|
||||
'rgba': bytes(pixels)}
|
||||
|
||||
|
||||
def write_png(path: Path, width: int, height: int, rgba: bytes):
|
||||
if len(rgba) != width*height*4: raise ValueError('Incorrect RGBA byte count')
|
||||
def chunk(tag, payload):
|
||||
return struct.pack('>I', len(payload)) + tag + payload + struct.pack('>I', zlib.crc32(tag+payload))
|
||||
scanlines = b''.join(b'\0' + rgba[y*width*4:(y+1)*width*4] for y in range(height))
|
||||
path.write_bytes(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def export_scene(database: Database, output: Path):
|
||||
"""Expanded original corner data; triangle fans preserve source winding."""
|
||||
validate_output(database, output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
scene = {'source': database.path.name, 'sha256': hashlib.sha256(database.data).hexdigest(),
|
||||
'coordinates': 'Original OMT world coordinates, unchanged',
|
||||
'vertices': [], 'uvs': [], 'normals': [], 'indices': [], 'meshes': [],
|
||||
'materials': [], 'textures': [], 'warnings': []}
|
||||
for entry in database.entries:
|
||||
payload = database.payload(entry)
|
||||
if entry.type == '3DSh':
|
||||
shape = read_shape(payload)
|
||||
for poly in shape['polygons']:
|
||||
first_index = len(scene['indices'])
|
||||
first_vertex = len(scene['vertices'])
|
||||
for corner in poly['corners']:
|
||||
scene['vertices'].append(shape['vertices'][corner['vertex']])
|
||||
scene['uvs'].append(corner['uv'])
|
||||
ni = corner['normal']
|
||||
scene['normals'].append(shape['normals'][ni] if ni < len(shape['normals']) else poly['normal'])
|
||||
for i in range(1, len(poly['corners'])-1):
|
||||
scene['indices'].extend((first_vertex, first_vertex+i, first_vertex+i+1))
|
||||
if len(poly['corners']) > 3:
|
||||
scene['warnings'].append(f'{entry.id}: polygon with {len(poly["corners"])} corners fan triangulated; concavity not validated')
|
||||
material = poly['material']['id'] if poly['material'] else -1
|
||||
if scene['meshes'] and scene['meshes'][-1]['shapeId'] == entry.id and scene['meshes'][-1]['material'] == material:
|
||||
scene['meshes'][-1]['indexCount'] += len(scene['indices'])-first_index
|
||||
else:
|
||||
scene['meshes'].append({'name': entry.name, 'shapeId': entry.id,
|
||||
'firstIndex': first_index, 'indexCount': len(scene['indices'])-first_index, 'material': material})
|
||||
elif entry.type == '3DMa':
|
||||
material = read_material(payload)
|
||||
scene['materials'].append({'id': entry.id, 'name': entry.name, **material})
|
||||
elif entry.type == 'Canv':
|
||||
canvas = read_canvas(payload)
|
||||
base = f'texture_{entry.id:08x}'
|
||||
(output / (base + '.rgba')).write_bytes(canvas.pop('rgba'))
|
||||
rgba = (output / (base + '.rgba')).read_bytes()
|
||||
write_png(output / (base + '.png'), canvas['width'], canvas['height'], rgba)
|
||||
scene['textures'].append({'id': entry.id, 'name': entry.name,
|
||||
'path': base + '.rgba', 'png': base + '.png', **canvas})
|
||||
(output / 'scene.json').write_text(json.dumps(scene, separators=(',', ':')) + '\n', encoding='utf-8')
|
||||
return scene
|
||||
|
||||
|
||||
def write_obj(shape: dict, path: Path):
|
||||
"""Preserve original positions/UVs/polygon normals in a standard interchange file."""
|
||||
lines = ['# Original OMT coordinates; no transform or guessed smoothing applied']
|
||||
lines.extend('v ' + ' '.join(format(v, '.9g') for v in p) for p in shape['vertices'])
|
||||
for poly in shape['polygons']:
|
||||
lines.extend('vt ' + ' '.join(format(v, '.9g') for v in c['uv']) for c in poly['corners'])
|
||||
lines.extend('vn ' + ' '.join(format(v, '.9g') for v in p['normal']) for p in shape['polygons'])
|
||||
uv_index = 1
|
||||
for normal_index, poly in enumerate(shape['polygons'], 1):
|
||||
if poly['material']: lines.append(f"usemtl material_{poly['material']['id']}")
|
||||
corners = []
|
||||
for corner in poly['corners']:
|
||||
corners.append(f"{corner['vertex']+1}/{uv_index}/{normal_index}")
|
||||
uv_index += 1
|
||||
if len(corners) >= 3: lines.append('f ' + ' '.join(corners))
|
||||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def extract(database: Database, output: Path, raw: bool = False):
|
||||
output = output.resolve()
|
||||
validate_output(database, output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
report = {'source': str(database.path), 'sha256': hashlib.sha256(database.data).hexdigest(),
|
||||
'table_offset': database.table_offset, 'entries': []}
|
||||
for entry in database.entries:
|
||||
info = asdict(entry)
|
||||
payload = database.payload(entry)
|
||||
base = f'{safe_name(entry.type)}_{entry.id:08x}_{safe_name(entry.name)}'
|
||||
if raw and entry.type != '0HDR':
|
||||
(output / (base + '.bin')).write_bytes(payload)
|
||||
if entry.type == '3DSh':
|
||||
try:
|
||||
shape = read_shape(payload)
|
||||
write_obj(shape, output / (base + '.obj'))
|
||||
(output / (base + '.json')).write_text(json.dumps(shape), encoding='utf-8')
|
||||
info.update(vertices=len(shape['vertices']), polygons=len(shape['polygons']))
|
||||
except FormatError as ex: info['unsupported'] = str(ex)
|
||||
if entry.type == 'Wave':
|
||||
start = payload.find(b'RIFF')
|
||||
if start >= 0 and payload[start+8:start+12] == b'WAVE':
|
||||
size = struct.unpack_from('<I', payload, start+4)[0] + 8
|
||||
if start + size <= len(payload):
|
||||
target = output / (base + '.wav')
|
||||
target.write_bytes(payload[start:start+size])
|
||||
try:
|
||||
with wave.open(str(target)) as wav:
|
||||
info['audio'] = {'channels': wav.getnchannels(), 'rate': wav.getframerate(),
|
||||
'sample_width': wav.getsampwidth(), 'frames': wav.getnframes()}
|
||||
except wave.Error:
|
||||
info['audio'] = {'encoding': 'non-PCM RIFF preserved unchanged'}
|
||||
else: info['unsupported'] = 'RIFF size outside resource'
|
||||
report['entries'].append(info)
|
||||
(output / 'manifest.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
return report
|
||||
|
||||
|
||||
def validate_output(database: Database, output: Path):
|
||||
source_directory = database.path.parent.resolve()
|
||||
target = output.resolve()
|
||||
if target.is_relative_to(source_directory) or database.path.resolve().is_relative_to(target):
|
||||
raise ValueError('Output must be separate from original input directory')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('database', type=Path)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--raw', action='store_true', help='Also extract exact raw resources')
|
||||
parser.add_argument('--scene', action='store_true', help='Export runtime scene.json with decoded textures')
|
||||
args = parser.parse_args()
|
||||
db = Database(args.database)
|
||||
if args.output:
|
||||
if args.scene:
|
||||
scene = export_scene(db, args.output)
|
||||
print(f"{len(scene['vertices'])} vertices, {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures; {args.output}")
|
||||
return
|
||||
report = extract(db, args.output, args.raw)
|
||||
failures = [e for e in report['entries'] if 'unsupported' in e]
|
||||
print(f'{len(db.entries)} resources; {len(failures)} unsupported conversions; {args.output}')
|
||||
for fail in failures[:10]: print(f" {fail['type']}:{fail['id']} {fail['unsupported']}")
|
||||
else:
|
||||
for e in db.entries: print(f'{e.type} {e.id:10d} {e.offset:10d} {e.size:10d} {e.name}')
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Small bounded decoder for the supplied non-interlaced 8-bit PNG textures."""
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from omt import FormatError, Reader
|
||||
|
||||
|
||||
def read_png(path: Path) -> dict:
|
||||
r = Reader(path.read_bytes())
|
||||
if r.take(8) != b'\x89PNG\r\n\x1a\n': raise FormatError('Invalid PNG signature')
|
||||
header, palette, transparency = None, None, b''
|
||||
compressed = bytearray()
|
||||
ended = False
|
||||
while r.position < len(r.data):
|
||||
size = r.u32()
|
||||
tag, data = r.take(4), r.take(size)
|
||||
if r.u32() != zlib.crc32(tag+data): raise FormatError('PNG checksum mismatch')
|
||||
if tag == b'IHDR': header = struct.unpack('>IIBBBBB',data)
|
||||
elif tag == b'PLTE': palette = data
|
||||
elif tag == b'tRNS': transparency = data
|
||||
elif tag == b'IDAT': compressed.extend(data)
|
||||
elif tag == b'IEND': ended = True; break
|
||||
elif tag[0] & 32 == 0: raise FormatError(f'Unsupported critical PNG chunk {tag!r}')
|
||||
if not ended or not header: raise FormatError('Incomplete PNG')
|
||||
r.end()
|
||||
width,height,depth,colour,compression,filter_method,interlace = header
|
||||
if depth != 8 or compression or filter_method or interlace: raise FormatError('Only non-interlaced 8-bit PNG is supported')
|
||||
channels = {0:1,2:3,3:1,4:2,6:4}.get(colour)
|
||||
if not channels or not width or not height or width*height > 16777216: raise FormatError('Invalid PNG dimensions/colour')
|
||||
stride = width*channels
|
||||
expected = (stride+1)*height
|
||||
decoder = zlib.decompressobj()
|
||||
raw = decoder.decompress(bytes(compressed),expected+1)
|
||||
if len(raw)!=expected or not decoder.eof or decoder.unused_data or decoder.unconsumed_tail:
|
||||
raise FormatError('Invalid PNG decoded length')
|
||||
rgba, previous = bytearray(), bytearray(stride)
|
||||
for y in range(height):
|
||||
mode = raw[y*(stride+1)]
|
||||
scan = bytearray(raw[y*(stride+1)+1:(y+1)*(stride+1)])
|
||||
if mode > 4: raise FormatError('Invalid PNG scanline filter')
|
||||
for x in range(stride):
|
||||
a = scan[x-channels] if x>=channels else 0
|
||||
b = previous[x]
|
||||
c = previous[x-channels] if x>=channels else 0
|
||||
if mode==1: predict = a
|
||||
elif mode==2: predict = b
|
||||
elif mode==3: predict = (a+b)//2
|
||||
elif mode==4:
|
||||
p=a+b-c;pa,pb,pc=abs(p-a),abs(p-b),abs(p-c)
|
||||
predict = a if pa<=pb and pa<=pc else b if pb<=pc else c
|
||||
else: predict=0
|
||||
scan[x] = (scan[x]+predict)&255
|
||||
for x in range(width):
|
||||
pixel = scan[x*channels:(x+1)*channels]
|
||||
if colour==3:
|
||||
index=pixel[0]
|
||||
if palette is None or index*3+3>len(palette): raise FormatError('PNG pixel outside palette')
|
||||
rgba.extend((*palette[index*3:index*3+3],transparency[index] if index<len(transparency) else 255))
|
||||
elif colour==6: rgba.extend(pixel)
|
||||
elif colour==2:
|
||||
alpha=0 if transparency and tuple(pixel)==struct.unpack('>3H',transparency) else 255
|
||||
rgba.extend((*pixel,alpha))
|
||||
else:
|
||||
alpha=pixel[1] if colour==4 else 0 if transparency and pixel[0]==struct.unpack('>H',transparency)[0] else 255
|
||||
rgba.extend((pixel[0],pixel[0],pixel[0],alpha))
|
||||
previous=scan
|
||||
return {'width':width,'height':height,'rgba':bytes(rgba)}
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare local scene/audio assets without modifying the original installation."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from omt import Database, export_scene, extract
|
||||
from ase import export_ase
|
||||
from png import read_png
|
||||
|
||||
|
||||
def find_archive(original: Path, name: str) -> Path:
|
||||
matches = [p for p in (original / 'omt').iterdir()
|
||||
if p.name.casefold() == (name if name.casefold().endswith('.omt') else name + '.omt').casefold()]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f'Expected exactly one archive for {name}, found {len(matches)}')
|
||||
return matches[0]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--original', type=Path, default=Path('../Original'))
|
||||
parser.add_argument('--output', type=Path, default=Path('artifacts/extracted'))
|
||||
parser.add_argument('--scenes', nargs='+', default=['level1b', 'VR01'])
|
||||
parser.add_argument('--audio', nargs='*', default=['sounds', 'musicjimmyshouse', 'musicneighborhood'])
|
||||
parser.add_argument('--animations', nargs='*', default=['jimstop', 'jimrun', 'jimjump'])
|
||||
args = parser.parse_args()
|
||||
if args.output.resolve().is_relative_to(args.original.resolve()):
|
||||
parser.error('Output must be outside the original installation')
|
||||
for name in args.scenes:
|
||||
source = find_archive(args.original, name)
|
||||
scene = export_scene(Database(source), args.output / source.stem)
|
||||
print(f"{source.name}: {len(scene['indices'])//3} triangles, {len(scene['textures'])} textures")
|
||||
for name in args.audio:
|
||||
source = find_archive(args.original, name)
|
||||
report = extract(Database(source), args.output / source.stem)
|
||||
count = sum(1 for e in report['entries'] if 'audio' in e)
|
||||
print(f'{source.name}: {count} audio resources')
|
||||
if args.animations:
|
||||
texture_source = next(p for p in (args.original/'png').iterdir() if p.name.casefold()=='jimycarl.png')
|
||||
texture = read_png(texture_source)
|
||||
target = args.output/'animations'
|
||||
target.mkdir(parents=True,exist_ok=True)
|
||||
(target/'jimycarl.rgba').write_bytes(texture.pop('rgba'))
|
||||
base = next(p for p in (args.original/'ASE').iterdir() if p.name.casefold()=='jimstop.ase')
|
||||
for name in args.animations:
|
||||
matches = [p for p in (args.original/'ASE').iterdir() if p.stem.casefold()==name.casefold() and p.suffix.casefold()=='.ase']
|
||||
if len(matches)!=1: raise ValueError(f'Expected one ASE for {name}')
|
||||
manifest = export_ase(matches[0],target/name,base)
|
||||
manifest['texture'] = {'path':'../jimycarl.rgba', **texture}
|
||||
(target/name/'animation.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
|
||||
print(f"{matches[0].name}: {len(manifest['frames'])} animation samples")
|
||||
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
Reference in new issue
Block a user