diff options
author | Treeki <treeki@gmail.com> | 2011-03-12 23:17:12 +0100 |
---|---|---|
committer | Treeki <treeki@gmail.com> | 2011-03-12 23:17:12 +0100 |
commit | 7d4e4c0b34a613dd3c0220475ae4e448197522c1 (patch) | |
tree | 4f5cee367de3fdef4f9a7c84af59ffe76a2bb1c3 /tools/kamek.py | |
download | kamek-7d4e4c0b34a613dd3c0220475ae4e448197522c1.tar.gz kamek-7d4e4c0b34a613dd3c0220475ae4e448197522c1.zip |
initial commit. now I can start playing with stuff!
Diffstat (limited to 'tools/kamek.py')
-rw-r--r-- | tools/kamek.py | 548 |
1 files changed, 548 insertions, 0 deletions
diff --git a/tools/kamek.py b/tools/kamek.py new file mode 100644 index 0000000..f67f049 --- /dev/null +++ b/tools/kamek.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python + +# Kamek - build tool for custom C++ code in New Super Mario Bros. Wii +# All rights reserved (c) Treeki 2010 +# Some function definitions by megazig + +# Requires PyYAML + +version_str = 'Kamek 0.1 by Treeki' + +import binascii +import os +import os.path +import shutil +import struct +import subprocess +import sys +import tempfile +import yaml + +import hooks + +u32 = struct.Struct('>I') + +verbose = True +use_rels = True +use_mw = False +use_wine = False +mw_path = '' +gcc_path = '' +gcc_type = 'powerpc-eabi' +show_cmd = False +delete_temp = True +override_config_file = None +only_build = None + + +def parse_cmd_options(): + global use_rels, use_mw, use_wine, show_cmd, delete_temp, only_build + global override_config_file, gcc_type, gcc_path, mw_path + + if '--no-rels' in sys.argv: + use_rels = False + + if '--use-mw' in sys.argv: + use_mw = True + + if '--use-wine' in sys.argv: + use_wine = True + + if '--show-cmd' in sys.argv: + show_cmd = True + + if '--keep-temp' in sys.argv: + delete_temp = False + + + + only_build = [] + + for arg in sys.argv: + if arg.startswith('--configs='): + override_config_file = arg[10:] + + if arg.startswith('--build='): + only_build.append(arg[8:]) + + if arg.startswith('--gcc-type='): + gcc_type = arg[11:] + + if arg.startswith('--gcc-path='): + gcc_path = arg[11:] + '/' + + if arg.startswith('--mw-path='): + mw_path = arg[10:] + '/' + + if len(only_build) == 0: + only_build = None + + +def print_debug(s): + if verbose: print '* '+str(s) + + +def read_configs(filename): + with open(filename, 'r') as f: + data = f.read() + + return yaml.safe_load(data) + + +current_unique_id = 0 +def generate_unique_id(): + # this is used for temporary filenames, to ensure that .o files + # do not overwrite each other + global current_unique_id + current_unique_id += 1 + return current_unique_id + + +def align_addr_up(addr, align): + align -= 1 + return (addr + align) & ~align + + +def generate_riiv_mempatch(offset, data): + return '<memory offset="0x%08X" value="%s" />' % (offset, binascii.hexlify(data)) + + +def generate_ocarina_patch(destOffset, data): + out = [] + count = len(data) + + sourceOffset = 0 + destOffset -= 0x80000000 + for i in xrange(count >> 2): + out.append('%08X %s' % (destOffset | 0x4000000, binascii.hexlify(data[sourceOffset:sourceOffset+4]))) + sourceOffset += 4 + destOffset += 4 + + # take care + remainder = count % 4 + if remainder == 3: + out.append('%08X 0000%s' % (destOffset | 0x2000000, binascii.hexlify(data[sourceOffset:sourceOffset+2]))) + out.append('%08X 000000%s' % (destOffset, binascii.hexlify(data[sourceOffset+2]))) + elif remainder == 2: + out.append('%08X 0000%s' % (destOffset | 0x2000000, binascii.hexlify(data[sourceOffset:sourceOffset+2]))) + elif remainder == 1: + out.append('%08X 000000%s' % (destOffset, binascii.hexlify(data[sourceOffset]))) + + return '\n'.join(out) + + +def generate_kamek_patches(patchlist): + kamekpatch = '' + for patch in patchlist: + if len(patch[1]) > 4: + # block patch + kamekpatch += u32.pack(align_addr_up(len(patch[1]), 4) / 4) + kamekpatch += u32.pack(patch[0]) + kamekpatch += patch[1] + # align it + if len(patch[1]) % 4 != 0: + kamekpatch += '\0' * (4 - (len(patch[1]) % 4)) + else: + # single patch + kamekpatch += u32.pack(patch[0]) + kamekpatch += patch[1] + + kamekpatch += u32.pack(0xFFFFFFFF) + return kamekpatch + + + +class KamekModule(object): + _requiredFields = ['source_files'] + + + def __init__(self, filename): + # load the module data + self.modulePath = os.path.normpath(filename) + self.moduleName = os.path.basename(self.modulePath) + self.moduleDir = os.path.dirname(self.modulePath) + + with open(self.modulePath, 'r') as f: + self.rawData = f.read() + + self.data = yaml.safe_load(self.rawData) + if not isinstance(self.data, dict): + raise ValueError, 'the module file %s is an invalid format (it should be a YAML mapping)' % self.moduleName + + # verify it + for field in self._requiredFields: + if field not in self.data: + raise ValueError, 'Missing field in the module file %s: %s' % (self.moduleName, field) + + + +class KamekBuilder(object): + def __init__(self, project, configs): + self.project = project + self.configs = configs + + + def build(self): + print_debug('Starting build') + + self._prepare_dirs() + + for config in self.configs: + if only_build != None and config['short_name'] not in only_build: + continue + + self._set_config(config) + + self._configTempDir = tempfile.mkdtemp() + print_debug('Temp files for this configuration are in: '+self._configTempDir) + + self._builtCodeAddr = 0x80001800 + if 'code_address' in self.project.data: + self._builtCodeAddr = self.project.data['code_address'] + + self._patches = [] + self._rel_patches = [] + self._hooks = [] + + # hook setup + self._hook_contexts = {} + for name, hookType in hooks.HookTypes.iteritems(): + if hookType.has_context: + self._hook_contexts[hookType] = hookType.context_type() + + self._create_hooks() + self._compile_modules() + self._link() + self._read_symbol_map() + + for hook in self._hooks: + hook.create_patches() + + self._create_patch() + + if delete_temp: + shutil.rmtree(self._configTempDir) + + + def _prepare_dirs(self): + self._outDir = self.project.makeRelativePath(self.project.data['output_dir']) + print_debug('Project will be built in: '+self._outDir) + + if not os.path.isdir(self._outDir): + os.makedirs(self._outDir) + print_debug('Created that directory') + + + def _set_config(self, config): + self._config = config + print_debug('---') + print_debug('Building for configuration: '+config['friendly_name']) + + self._config_short_name = config['short_name'] + self._rel_area = (config['rel_area_start'], config['rel_area_end']) + + + def _create_hooks(self): + print_debug('---') + print_debug('Creating hooks') + + for m in self.project.modules: + if 'hooks' in m.data: + for hookData in m.data['hooks']: + assert 'name' in hookData and 'type' in hookData + + print_debug('Hook: %s : %s' % (m.moduleName, hookData['name'])) + + if hookData['type'] in hooks.HookTypes: + hookType = hooks.HookTypes[hookData['type']] + hook = hookType(self, m, hookData) + self._hooks.append(hook) + else: + raise ValueError, 'Unknown hook type: %s' % hookData['type'] + + + def _compile_modules(self): + print_debug('---') + print_debug('Compiling modules') + + if use_mw: + # metrowerks setup + cc_command = ['%smwcceppc.exe' % mw_path, '-I.', '-I-', '-I.', '-nostdinc', '-Cpp_exceptions', 'off', '-Os', '-proc', 'gekko', '-fp', 'hard', '-enum', 'int', '-sdata', '0', '-sdata2', '0', '-g'] + as_command = ['%smwasmeppc.exe' % mw_path, '-I.', '-I-', '-I.', '-nostdinc', '-proc', 'gekko', '-d', '__MWERKS__'] + + for d in self._config['defines']: + cc_command.append('-d') + cc_command.append(d) + as_command.append('-d') + as_command.append(d) + + for i in self._config['include_dirs']: + cc_command.append('-I%s' % i) + #cc_command.append(i) + as_command.append('-I%s' % i) + #as_command.append(i) + + if use_wine: + cc_command.insert(0, 'wine') + as_command.insert(0, 'wine') + + else: + # gcc setup + cc_command = ['%s%s-g++' % (gcc_path, gcc_type), '-nodefaultlibs', '-I.', '-fno-builtin', '-Os', '-fno-exceptions', '-fno-rtti', '-mno-sdata'] + as_command = cc_command + + for d in self._config['defines']: + cc_command.append('-D%s' % d) + + for i in self._config['include_dirs']: + cc_command.append('-I%s' % i) + + + self._moduleFiles = [] + for m in self.project.modules: + for normal_sourcefile in m.data['source_files']: + print_debug('Compiling %s : %s' % (m.moduleName, normal_sourcefile)) + + objfile = os.path.join(self._configTempDir, '%d.o' % generate_unique_id()) + sourcefile = os.path.join(m.moduleDir, normal_sourcefile) + + # todo: better extension detection + if sourcefile.endswith('.s') or sourcefile.endswith('.S'): + command = as_command + else: + command = cc_command + + new_command = command + ['-c', '-o', objfile, sourcefile] + + if 'cc_args' in m.data: + new_command += m.data['cc_args'] + + if show_cmd: + print_debug(new_command) + + errorVal = subprocess.call(new_command) + if errorVal != 0: + print 'BUILD FAILED!' + print 'compiler returned %d - an error occurred while compiling %s' % (errorVal, sourcefile) + sys.exit(1) + + self._moduleFiles.append(objfile) + + print_debug('Compilation complete') + + + def _link(self): + print_debug('---') + print_debug('Linking project') + + self._mapFile = '%s/%s_linkmap.map' % (self._outDir, self._config_short_name) + self._outFile = '%s/%s_out.bin' % (self._outDir, self._config_short_name) + + ld_command = ['%s%s-ld' % (gcc_path, gcc_type), '-L.'] + ld_command.append('-o') + ld_command.append(self._outFile) + ld_command.append('-Ttext') + ld_command.append('0x%08X' % self._builtCodeAddr) + ld_command.append('-T') + ld_command.append(self._config['linker_script']) + ld_command.append('-Map') + ld_command.append(self._mapFile) + ld_command.append('--no-demangle') # for debugging + #ld_command.append('--verbose') + ld_command += self._moduleFiles + + if show_cmd: + print_debug(ld_command) + + errorVal = subprocess.call(ld_command) + if errorVal != 0: + print 'BUILD FAILED!' + print 'ld returned %d' % errorVal + sys.exit(1) + + print_debug('Linked successfully') + + + def _read_symbol_map(self): + print_debug('---') + print_debug('Reading symbol map') + + self._symbols = [] + + file = open(self._mapFile, 'r') + + for line in file: + if '__text_start' in line: + self._textSegStart = int(line.split()[0],0) + break + + # now read the individual symbols + # this is probably a bad method to parse it, but whatever + for line in file: + if '__text_end' in line: + self._textSegEnd = int(line.split()[0],0) + break + + if not line.startswith(' '): continue + + sym = line.split() + sym[0] = int(sym[0],0) + self._symbols.append(sym) + + # we've found __text_end, so now we should be at the output section + currentEndAddress = self._textSegEnd + + for line in file: + if line[0] == '.': + # probably a segment + data = line.split() + if len(data) < 3: continue + + segAddr = int(data[1],0) + segSize = int(data[2],0) + + if segAddr+segSize > currentEndAddress: + currentEndAddress = segAddr+segSize + + self._codeStart = self._textSegStart + self._codeEnd = currentEndAddress + + file.close() + print_debug('Read, %d symbol(s) parsed' % len(self._symbols)) + + + # next up, run it through c++filt + print_debug('Running c++filt') + p = subprocess.Popen(gcc_type + '-c++filt', stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + symbolNameList = [sym[1] for sym in self._symbols] + filtResult = p.communicate('\n'.join(symbolNameList)) + filteredSymbols = filtResult[0].split('\n') + + for sym, filt in zip(self._symbols, filteredSymbols): + sym.append(filt) + + print_debug('Done. All symbols complete.') + print_debug('Generated code is at 0x%08X .. 0x%08X' % (self._codeStart, self._codeEnd - 4)) + + + def _find_func_by_symbol(self, find_symbol): + for sym in self._symbols: + if sym[2] == find_symbol: + return sym[0] + + raise ValueError, 'Cannot find function: %s' % find_symbol + + + def _add_patch(self, offset, data): + if offset >= self._rel_area[0] and offset <= self._rel_area[1] and use_rels: + self._rel_patches.append((offset, data)) + else: + self._patches.append((offset, data)) + + + def _create_patch(self): + print_debug('---') + print_debug('Creating patch') + + # convert the .rel patches to KamekPatcher format + if len(self._rel_patches) > 0: + kamekpatch = generate_kamek_patches(self._rel_patches) + #self._patches.append((0x817F4800, kamekpatch)) + self._patches.append((0x80002F60, kamekpatch)) + + # add the outfile as a patch + file = open(self._outFile, 'rb') + patch = (self._codeStart, file.read()) + file.close() + + self._patches.append(patch) + + # generate a Riivolution patch + riiv = open('%s/%s_riiv.xml' % (self._outDir, self._config['short_name']), 'w') + for patch in self._patches: + riiv.write(generate_riiv_mempatch(*patch) + '\n') + + riiv.close() + + # generate an Ocarina patch + ocarina = open('%s/%s_ocarina.txt' % (self._outDir, self._config['short_name']), 'w') + for patch in self._patches: + ocarina.write(generate_ocarina_patch(*patch) + '\n') + + ocarina.close() + + # generate a KamekPatcher patch + kpatch = open('%s/%s_loader.bin' % (self._outDir, self._config['short_name']), 'wb') + kpatch.write(generate_kamek_patches(self._patches)) + kpatch.close() + + print_debug('Patches generated') + + + +class KamekProject(object): + _requiredFields = ['output_dir', 'modules'] + + + def __init__(self, filename): + # load the project data + self.projectPath = os.path.abspath(filename) + self.projectName = os.path.basename(self.projectPath) + self.projectDir = os.path.dirname(self.projectPath) + + with open(self.projectPath, 'r') as f: + self.rawData = f.read() + + self.data = yaml.safe_load(self.rawData) + if not isinstance(self.data, dict): + raise ValueError, 'the project file is an invalid format (it should be a YAML mapping)' + + # verify it + for field in self._requiredFields: + if field not in self.data: + raise ValueError, 'Missing field in the project file: %s' % field + + # load each module + self.modules = [] + for moduleName in self.data['modules']: + modulePath = self.makeRelativePath(moduleName) + self.modules.append(KamekModule(modulePath)) + + + def makeRelativePath(self, path): + return os.path.normpath(os.path.join(self.projectDir, path)) + + + def build(self): + # compile everything in the project + builder = KamekBuilder(self, self.configs) + builder.build() + + + +def main(): + print version_str + print + + if len(sys.argv) < 2: + print 'No input file specified' + sys.exit() + + parse_cmd_options() + + project = KamekProject(os.path.normpath(sys.argv[1])) + + if override_config_file: + project.configs = read_configs(override_config_file) + else: + project.configs = read_configs('kamek_configs.yaml') + + project.build() + + + +if __name__ == '__main__': + main() + + |