diff --git a/.gitignore b/.gitignore index 7b04a01..2a6d039 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ conker/ctx.c tools/decomp-permuter tools/gfxdis +venv/ +.venv/ diff --git a/Makefile b/Makefile index c1b1bb6..e621ac5 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,10 @@ endif ### Targets +## REVERT ME - CI only uses the publish Docker container, so this is a workaround to install requirements. DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS +pipinstall: + python3 -m pip install -r requirements.txt + default: all all: dirs $(TARGET).z64 $(VERIFY) @@ -55,7 +59,7 @@ really-clean: clean check: .baserom.$(VERSION).ok -extract: check $(GAME_DIR)/$(BASENAME).$(VERSION).bin +extract: pipinstall check $(GAME_DIR)/$(BASENAME).$(VERSION).bin decompress: $(EXTRACT_DIR)/00000000.bin @@ -86,16 +90,16 @@ $(GAME_DIR)/$(BASENAME).$(VERSION).bin: $(BIN_DIR)/game.$(VERSION).bin # game code is not compressed in ECTS ROM ifeq ($(VERSION),ects) $(BIN_DIR)/game.$(VERSION).bin: $(BASENAME).$(VERSION).yaml - $(PYTHON) tools/n64splat/split.py $< + $(PYTHON) -m splat split $< else $(BIN_DIR)/game.$(VERSION).bin: $(BIN_DIR)/game/rzip/data/0000.bin cat $(BIN_DIR)/game/rzip/code/0*.bin $(BIN_DIR)/game/rzip/data/0000.bin > $@ $(BIN_DIR)/game/rzip/data/0000.bin: $(BIN_DIR)/game.$(VERSION).rzip.bin - $(PYTHON) tools/n64splat/split.py game.$(VERSION).rzip.yaml --modes bin rzip + $(PYTHON) -m splat split game.$(VERSION).rzip.yaml --modes bin rzip $(BIN_DIR)/game.$(VERSION).rzip.bin: $(BASENAME).$(VERSION).yaml - $(PYTHON) tools/n64splat/split.py $< + $(PYTHON) -m splat split $< endif .baserom.$(VERSION).ok: baserom.$(VERSION).z64 diff --git a/conker.us.yaml b/conker.us.yaml index 6b3eb45..04a65e2 100644 --- a/conker.us.yaml +++ b/conker.us.yaml @@ -1,10 +1,11 @@ +sha1: 4cbadd3c4e0729dec46af64ad018050eada4f47a options: basename: conker target_path: baserom.us.z64 + platform: n64 base_path: . compiler: IDO extensions_path: tools/splat_ext - sha1: 4cbadd3c4e0729dec46af64ad018050eada4f47a segments: - name: header.us type: bin diff --git a/conker/Makefile b/conker/Makefile index 6f6aaf5..c4185b0 100644 --- a/conker/Makefile +++ b/conker/Makefile @@ -169,7 +169,11 @@ really-clean: clean rm -f *auto.txt rm -rf conker.us.bin conker.eu.bin conker.ects.bin conker.debug.bin -extract: $(BUILD_DIR)/splat +## REVERT ME - CI only uses the publish Docker container, so this is a workaround to install requirements. DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS +pipinstall: + python3 -m pip install -r ../requirements.txt + +extract: pipinstall $(BUILD_DIR)/splat # sanity check base bin check: $(BASENAME).$(VERSION).ok @@ -201,7 +205,7 @@ $(GLOBAL_ASM_O_FILES): $(BUILD_DIR)/%.c.o: %.c include/variables.h include/struc $(PYTHON) $(ASM_PROCESSOR_DIR)/asm_processor.py $(OPT_FLAGS) $< > $(BUILD_DIR)/$< $(CC) -c -32 $(CFLAGS) $(OPT_FLAGS) $(LOOP_UNROLL) $(MIPSBIT) -o $@ $(BUILD_DIR)/$< $(PYTHON) $(ASM_PROCESSOR_DIR)/asm_processor.py $(OPT_FLAGS) $< --post-process $@ \ - --assembler "$(AS) $(ASFLAGS)" --asm-prelude $(ASM_PROCESSOR_DIR)/prelude.s + --assembler "$(AS) $(ASFLAGS)" --asm-prelude $(ASM_PROCESSOR_DIR)/prelude.inc endif $(BUILD_DIR)/%.c.o: %.c @@ -218,7 +222,7 @@ $(TARGET).bin: $(TARGET).elf # extract $(BUILD_DIR)/splat: check $(BASENAME).$(VERSION).yaml - $(PYTHON) ../tools/n64splat/split.py $(BASENAME).$(VERSION).yaml + $(PYTHON) -m splat split $(BASENAME).$(VERSION).yaml %.ok: %.bin @echo "$$(cat $(BASENAME).$(VERSION).sha1) $<" | sha1sum --check diff --git a/conker/conker.debug.yaml b/conker/conker.debug.yaml index 5b10d33..8d667f8 100644 --- a/conker/conker.debug.yaml +++ b/conker/conker.debug.yaml @@ -4,6 +4,7 @@ options: base_path: . find_file_boundaries: yes compiler: IDO + platform: n64 cpp_args: - "-Iinclude" - "-Iinclude/2.0L" diff --git a/conker/conker.ects.yaml b/conker/conker.ects.yaml index c45446f..7ffa6d2 100644 --- a/conker/conker.ects.yaml +++ b/conker/conker.ects.yaml @@ -4,6 +4,7 @@ options: base_path: . find_file_boundaries: yes compiler: IDO + platform: n64 cpp_args: - "-Iinclude" - "-Iinclude/2.0L" diff --git a/conker/conker.eu.yaml b/conker/conker.eu.yaml index 4c3d29e..3585c9d 100644 --- a/conker/conker.eu.yaml +++ b/conker/conker.eu.yaml @@ -4,6 +4,7 @@ options: base_path: . find_file_boundaries: yes compiler: IDO + platform: n64 cpp_args: - "-Iinclude" - "-Iinclude/2.0L" diff --git a/conker/conker.us.yaml b/conker/conker.us.yaml index fc43fe0..e4d7c1e 100644 --- a/conker/conker.us.yaml +++ b/conker/conker.us.yaml @@ -1,16 +1,18 @@ +sha1: 842e3d348e3c8ae0039e2ab367ad492f9b5266d8 options: basename: conker target_path: conker.us.bin base_path: . # find_file_boundaries: yes compiler: IDO + platform: n64 generated_c_preamble: | #include #include "functions.h" #include "variables.h" - create_detected_syms: yes - undefined_syms_path: undefined_syms.us.txt - symbol_addrs_path: symbol_addrs.us.txt + symbol_addrs_path: symbol_addrs.us.txt + symbol_name_format: $VRAM + symbol_name_format_no_rom: $VRAM segments: - name: header type: header @@ -47,7 +49,7 @@ segments: - [0x50A0, c, init_50A0] - [0x5570, asm, libultra/io/motor] # - [0x57E0, asm] no file split here! - - [0x5AB0, c, init_5AB0] + - [0x5AB0, asm] # - [0x6240, c] # - [0x71D0, c] - [0x8120, asm, libultra/os/maptlbrdb] diff --git a/conker/src/init_5AB0.c b/conker/src/init_5AB0.c deleted file mode 100644 index d8da486..0000000 --- a/conker/src/init_5AB0.c +++ /dev/null @@ -1,64 +0,0 @@ -#include -#include "functions.h" -#include "variables.h" - - -// entrypoint, map tlb and jump to func_1050 -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10005AB0.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10005B04.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10005BE0.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10005C2C.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_100061F8.s") - -// is this whole file handwritten? -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10006240.s") -// void func_10006240(void) { -// func_1000625C(); -// } - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_1000625C.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_1000632C.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10006380.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10006424.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10006828.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_1000692C.s") -// s32 func_1000692C(s32 arg291, s32 arg29D) { -// func_10006E00(&D_8003BE94, &D_8003C858, 7, 5); -// return 0; -// } - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_1000696C.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10006E00.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_1000709C.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_100077B8.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_100079D8.s") - -// thread related -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007A24.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/__osCleanupThread.s") - -// handwritten -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007C74.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007CC4.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007D28.s") - -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007DA0.s") - -// contains "EXITED OK" etc text -#pragma GLOBAL_ASM("asm/nonmatchings/init_5AB0/func_10007DAC.s") diff --git a/conker/symbol_addrs.us.txt b/conker/symbol_addrs.us.txt index 9cb249c..d7401d6 100644 --- a/conker/symbol_addrs.us.txt +++ b/conker/symbol_addrs.us.txt @@ -118,11 +118,7 @@ __osSpDeviceBusy = 0x10023e80; __osSetTimerIntr = 0x100240B4; __osInsertTimer = 0x10024128; __osViInit = 0x100242b0; -__osGetActiveQueue = 0x10024400; -__osGetCurrFaultedThread = 0x10024400; -__osViGetCurrentContext = 0x10024400; -__osViGetNextContext = 0x10024400; -osPiGetDeviceType = 0x10024400; +__osGetActiveQueue = 0x10024400; // allow_duplicated:True __osViSwapContext = 0x10024410; osGetCount = 0x10024770; __osDequeueThread = 0x10024780; @@ -219,7 +215,6 @@ corrupted_init2 = 0x151F1974; corrupted2 = 0x151F1B28; osPfsInit2 = 0x151EF450; osContStartReadData2 = 0x151EFAA0; -__osPackReadData2 = 0x151EFC0C; osContGetReadData2 = 0x151EFB64; __osPackReadData2 = 0x151EFC0C; guMtxXFML = 0x151F0080; diff --git a/conker/undefined_funcs.us.txt b/conker/undefined_funcs.us.txt index 60c5e85..59a343e 100644 --- a/conker/undefined_funcs.us.txt +++ b/conker/undefined_funcs.us.txt @@ -1,2 +1,3 @@ __osDevMgrMain = 0x10002E50; __n_CSPHandleMetaMsg = 0x10015044; +func_160036F0 = 0x160036F0; diff --git a/conker/undefined_syms.us.txt b/conker/undefined_syms.us.txt index 2cf239d..3eb0501 100644 --- a/conker/undefined_syms.us.txt +++ b/conker/undefined_syms.us.txt @@ -48,7 +48,5 @@ D_8002BA44 = 0x8002BA44; D_8002C080 = 0x8002C080; D_8002C0A0 = 0x8002C0A0; D_8002C410 = 0x8002C410; -D_8002C460 = 0x8002C460; -D_8002C4CC = 0x8002C4CC; D_8002C520 = 0x8002C520; D_80095A30 = 0x80095A30; diff --git a/game.debug.rzip.yaml b/game.debug.rzip.yaml index 7131c47..690c805 100644 --- a/game.debug.rzip.yaml +++ b/game.debug.rzip.yaml @@ -3,6 +3,7 @@ options: target_path: assets/game.debug.rzip.bin base_path: . asset_path: assets/game + platform: n64 extensions_path: tools/splat_ext segments: - name: code diff --git a/game.eu.rzip.yaml b/game.eu.rzip.yaml index 4dacdb2..c09a5a7 100644 --- a/game.eu.rzip.yaml +++ b/game.eu.rzip.yaml @@ -2,6 +2,7 @@ options: target_path: assets/game.eu.rzip.bin base_path: . asset_path: assets/game + platform: n64 extensions_path: tools/splat_ext segments: - name: code diff --git a/game.us.rzip.yaml b/game.us.rzip.yaml index 62327cc..1d3be71 100644 --- a/game.us.rzip.yaml +++ b/game.us.rzip.yaml @@ -1,7 +1,9 @@ options: + basename: conker target_path: assets/game.us.rzip.bin base_path: . asset_path: assets/game + platform: n64 extensions_path: tools/splat_ext segments: - name: code diff --git a/requirements.txt b/requirements.txt index 9d84f06..32e4e74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ python-ranges pyyaml toml watchdog +splat64[mips]>=0.34,<0.35 diff --git a/tools/asm-processor b/tools/asm-processor index f511734..42e7cca 160000 --- a/tools/asm-processor +++ b/tools/asm-processor @@ -1 +1 @@ -Subproject commit f511734d56ebb152c2b0c5aab212a0fea2588513 +Subproject commit 42e7ccaf1883279c7fe2f9d338ff3383175aa97f diff --git a/tools/n64splat b/tools/n64splat index e6f5a25..3376e8c 160000 --- a/tools/n64splat +++ b/tools/n64splat @@ -1 +1 @@ -Subproject commit e6f5a25f67282eeafa6bfd70c8fb5816021cd93a +Subproject commit 3376e8c1b542d68016d74c685a9b3c64d4e488a1 diff --git a/tools/splat_ext/rareunzip.py b/tools/splat_ext/rareunzip.py deleted file mode 120000 index 0dfc8ea..0000000 --- a/tools/splat_ext/rareunzip.py +++ /dev/null @@ -1 +0,0 @@ -../rareunzip.py \ No newline at end of file diff --git a/tools/splat_ext/rareunzip.py b/tools/splat_ext/rareunzip.py new file mode 100644 index 0000000..2ef5e2a --- /dev/null +++ b/tools/splat_ext/rareunzip.py @@ -0,0 +1,24 @@ +import sys +import zlib + + +def runzip_with_leftovers(data): + d = zlib.decompressobj(wbits=-15) # raw deflate bytestream + res = d.decompress(data[4:]) # drop 4 byte length header + return (res, d.unused_data) + +def runzip(data): + res, leftovers = runzip_with_leftovers(data) + return res + +def main(): + with open(sys.argv[1], "rb") as f: + with open(sys.argv[2], "wb") as o: + o.write(runzip(f.read())) + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print("usage %s infile outfile" % sys.argv[0]) + else: + main() diff --git a/tools/splat_ext/rzip.py b/tools/splat_ext/rzip.py index 5bf84a7..ef056a4 100644 --- a/tools/splat_ext/rzip.py +++ b/tools/splat_ext/rzip.py @@ -3,21 +3,23 @@ from pathlib import Path -from segtypes.n64.segment import N64Segment -from util import options +from splat.util import options +from splat.segtypes.segment import Segment +import zlib -import sys -if options.get_extensions_path() not in sys.path: - sys.path.append('tools/splat_ext') -import rareunzip +# TODO: This is from rareunzip.py, but I can't get the relative import to work with splat. +# Figure out a way to import this function instead of copying it here. +def runzip_with_leftovers(data): + d = zlib.decompressobj(wbits=-15) # raw deflate bytestream + res = d.decompress(data[4:]) # drop 4 byte length header + return (res, d.unused_data) # Rare zip format: # 4 byte uncompressed length followed by deflate level 9 raw payload -class N64SegRzip(N64Segment): - def __init__(self, rom_start, rom_end, type, name, vram_start, extract, given_subalign, given_is_overlay, given_dir, args, yaml): - super().__init__(rom_start, rom_end, type, name, vram_start, extract, given_subalign, given_is_overlay, given_dir, args, yaml) +class N64SegRzip(Segment): + def __init__(self, rom_start, rom_end, type, name, vram_start, args, yaml): + super().__init__(rom_start, rom_end, type, name, vram_start, args=args, yaml=yaml) self.has_subsegments = "subsegments" in yaml - self.yaml = yaml self.xor = yaml.get("xor", None) def get_default_name(self, addr): @@ -92,7 +94,7 @@ def parse_subsegments(self): ret = [] for i, split_file in enumerate(self.yaml["subsegments"]): - if type(split_file) is dict: + if isinstance(split_file, dict): start = split_file["start"] end = split_file["end"] name = None if "name" not in split_file else split_file["name"] @@ -115,7 +117,7 @@ def out_path(self) -> Path: return self.out_dir() / f"{self.name}.bin" def out_dir(self) -> Path: - return options.get_asset_path() / "rzip" / self.name + return options.opts.asset_path / "rzip" / self.name def split(self, rom_bytes): if self.has_subsegments: @@ -155,7 +157,7 @@ def split(self, rom_bytes): extension = "mp3" else: try: - result, padding = rareunzip.runzip_with_leftovers(data) + result, padding = runzip_with_leftovers(data) except Exception as e: print("Failed to decompress file", split_file, e) # update total