(DRAFT PR) Update tools

.gitignore

@@ -36,3 +36,5 @@ conker/ctx.c
 
 tools/decomp-permuter
 tools/gfxdis
+venv/
+.venv/

Makefile

@@ -38,6 +38,10 @@ endif
 
 ### Targets
 
+## REVERT ME - CI only uses the publish Docker container, so this is a workaround to install requirements. DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS
+pipinstall:
+	python3 -m pip install -r requirements.txt
+
 default: all
 
 all: dirs $(TARGET).z64 $(VERIFY)
@@ -55,7 +59,7 @@ really-clean: clean
 
 check: .baserom.$(VERSION).ok
 
-extract: check $(GAME_DIR)/$(BASENAME).$(VERSION).bin
+extract: pipinstall check $(GAME_DIR)/$(BASENAME).$(VERSION).bin
 
 decompress: $(EXTRACT_DIR)/00000000.bin
 
@@ -86,16 +90,16 @@ $(GAME_DIR)/$(BASENAME).$(VERSION).bin: $(BIN_DIR)/game.$(VERSION).bin
 # game code is not compressed in ECTS ROM
 ifeq ($(VERSION),ects)
 $(BIN_DIR)/game.$(VERSION).bin: $(BASENAME).$(VERSION).yaml
-	$(PYTHON) tools/n64splat/split.py $<
+	$(PYTHON) -m splat split $<
 else
 $(BIN_DIR)/game.$(VERSION).bin: $(BIN_DIR)/game/rzip/data/0000.bin
 	cat $(BIN_DIR)/game/rzip/code/0*.bin $(BIN_DIR)/game/rzip/data/0000.bin > $@
 
 $(BIN_DIR)/game/rzip/data/0000.bin: $(BIN_DIR)/game.$(VERSION).rzip.bin
-	$(PYTHON) tools/n64splat/split.py game.$(VERSION).rzip.yaml --modes bin rzip
+	$(PYTHON) -m splat split game.$(VERSION).rzip.yaml --modes bin rzip
 
 $(BIN_DIR)/game.$(VERSION).rzip.bin: $(BASENAME).$(VERSION).yaml
-	$(PYTHON) tools/n64splat/split.py $<
+	$(PYTHON) -m splat split $<
 endif
 
 .baserom.$(VERSION).ok: baserom.$(VERSION).z64

conker.us.yaml

@@ -1,10 +1,11 @@
+sha1: 4cbadd3c4e0729dec46af64ad018050eada4f47a
 options:
   basename: conker
   target_path: baserom.us.z64
+  platform: n64
   base_path: .
   compiler: IDO
   extensions_path: tools/splat_ext
-  sha1: 4cbadd3c4e0729dec46af64ad018050eada4f47a
 segments:
   - name:  header.us
     type:  bin

conker/Makefile

@@ -169,7 +169,11 @@ really-clean: clean
 	rm -f *auto.txt
 	rm -rf conker.us.bin conker.eu.bin conker.ects.bin conker.debug.bin
 
-extract: $(BUILD_DIR)/splat
+## REVERT ME - CI only uses the publish Docker container, so this is a workaround to install requirements. DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS DO NOT MERGE THIS
+pipinstall:
+	python3 -m pip install -r ../requirements.txt
+
+extract: pipinstall $(BUILD_DIR)/splat
 
 # sanity check base bin
 check: $(BASENAME).$(VERSION).ok
@@ -201,7 +205,7 @@ $(GLOBAL_ASM_O_FILES): $(BUILD_DIR)/%.c.o: %.c include/variables.h include/struc
 	$(PYTHON) $(ASM_PROCESSOR_DIR)/asm_processor.py $(OPT_FLAGS) $< > $(BUILD_DIR)/$<
 	$(CC) -c -32 $(CFLAGS) $(OPT_FLAGS) $(LOOP_UNROLL) $(MIPSBIT) -o $@ $(BUILD_DIR)/$<
 	$(PYTHON) $(ASM_PROCESSOR_DIR)/asm_processor.py $(OPT_FLAGS) $< --post-process $@ \
-		--assembler "$(AS) $(ASFLAGS)" --asm-prelude $(ASM_PROCESSOR_DIR)/prelude.s
+		--assembler "$(AS) $(ASFLAGS)" --asm-prelude $(ASM_PROCESSOR_DIR)/prelude.inc
 endif
 
 $(BUILD_DIR)/%.c.o: %.c
@@ -218,7 +222,7 @@ $(TARGET).bin: $(TARGET).elf
 
 # extract
 $(BUILD_DIR)/splat: check $(BASENAME).$(VERSION).yaml
-	$(PYTHON) ../tools/n64splat/split.py $(BASENAME).$(VERSION).yaml
+	$(PYTHON) -m splat split $(BASENAME).$(VERSION).yaml
 
 %.ok: %.bin
 	@echo "$$(cat $(BASENAME).$(VERSION).sha1)  $<" | sha1sum --check

conker/conker.debug.yaml

@@ -4,6 +4,7 @@ options:
   base_path: .
   find_file_boundaries: yes
   compiler: IDO
+  platform: n64
   cpp_args:
   - "-Iinclude"
   - "-Iinclude/2.0L"

conker/conker.ects.yaml

@@ -4,6 +4,7 @@ options:
   base_path: .
   find_file_boundaries: yes
   compiler: IDO
+  platform: n64
   cpp_args:
   - "-Iinclude"
   - "-Iinclude/2.0L"

conker/conker.eu.yaml

@@ -4,6 +4,7 @@ options:
   base_path: .
   find_file_boundaries: yes
   compiler: IDO
+  platform: n64
   cpp_args:
   - "-Iinclude"
   - "-Iinclude/2.0L"

conker/conker.us.yaml

@@ -1,16 +1,18 @@
+sha1: 842e3d348e3c8ae0039e2ab367ad492f9b5266d8
 options:
   basename: conker
   target_path: conker.us.bin
   base_path: .
   # find_file_boundaries: yes
   compiler: IDO
+  platform: n64
   generated_c_preamble: |
     #include <ultra64.h>
     #include "functions.h"
     #include "variables.h"
-  create_detected_syms: yes
-  undefined_syms_path: undefined_syms.us.txt
-  symbol_addrs_path:   symbol_addrs.us.txt
+  symbol_addrs_path: symbol_addrs.us.txt
+  symbol_name_format: $VRAM
+  symbol_name_format_no_rom: $VRAM
 segments:
   - name:  header
     type:  header
@@ -47,7 +49,7 @@ segments:
     - [0x50A0, c, init_50A0]
     - [0x5570, asm, libultra/io/motor]
   # - [0x57E0, asm] no file split here!
-    - [0x5AB0, c, init_5AB0]
+    - [0x5AB0, asm]
     # - [0x6240, c]
     # - [0x71D0, c]
     - [0x8120, asm, libultra/os/maptlbrdb]

conker/symbol_addrs.us.txt

@@ -118,11 +118,7 @@ __osSpDeviceBusy                = 0x10023e80;
 __osSetTimerIntr                = 0x100240B4;
 __osInsertTimer                 = 0x10024128;
 __osViInit                      = 0x100242b0;
-__osGetActiveQueue              = 0x10024400;
-__osGetCurrFaultedThread        = 0x10024400;
-__osViGetCurrentContext         = 0x10024400;
-__osViGetNextContext            = 0x10024400;
-osPiGetDeviceType               = 0x10024400;
+__osGetActiveQueue              = 0x10024400; // allow_duplicated:True
 __osViSwapContext               = 0x10024410;
 osGetCount                      = 0x10024770;
 __osDequeueThread               = 0x10024780;
@@ -219,7 +215,6 @@ corrupted_init2                 = 0x151F1974;
 corrupted2                      = 0x151F1B28;
 osPfsInit2                      = 0x151EF450;
 osContStartReadData2            = 0x151EFAA0;
-__osPackReadData2               = 0x151EFC0C;
 osContGetReadData2              = 0x151EFB64;
 __osPackReadData2               = 0x151EFC0C;
 guMtxXFML                       = 0x151F0080;

conker/undefined_funcs.us.txt

@@ -1,2 +1,3 @@
 __osDevMgrMain                  = 0x10002E50;
 __n_CSPHandleMetaMsg            = 0x10015044;
+func_160036F0 = 0x160036F0;

conker/undefined_syms.us.txt

@@ -48,7 +48,5 @@ D_8002BA44 = 0x8002BA44;
 D_8002C080 = 0x8002C080;
 D_8002C0A0 = 0x8002C0A0;
 D_8002C410 = 0x8002C410;
-D_8002C460 = 0x8002C460;
-D_8002C4CC = 0x8002C4CC;
 D_8002C520 = 0x8002C520;
 D_80095A30 = 0x80095A30;

game.debug.rzip.yaml

@@ -3,6 +3,7 @@ options:
   target_path: assets/game.debug.rzip.bin
   base_path: .
   asset_path: assets/game
+  platform: n64
   extensions_path: tools/splat_ext
 segments:
   - name:  code

game.eu.rzip.yaml

@@ -2,6 +2,7 @@ options:
   target_path: assets/game.eu.rzip.bin
   base_path: .
   asset_path: assets/game
+  platform: n64
   extensions_path: tools/splat_ext
 segments:
   - name:  code

game.us.rzip.yaml

@@ -1,7 +1,9 @@
 options:
+  basename: conker
   target_path: assets/game.us.rzip.bin
   base_path: .
   asset_path: assets/game
+  platform: n64
   extensions_path: tools/splat_ext
 segments:
   - name:  code

requirements.txt

@@ -12,3 +12,4 @@ python-ranges
 pyyaml
 toml
 watchdog
+splat64[mips]>=0.34,<0.35

tools/splat_ext/rareunzip.py

@@ -0,0 +1,24 @@
+import sys
+import zlib
+
+
+def runzip_with_leftovers(data):
+    d = zlib.decompressobj(wbits=-15) # raw deflate bytestream
+    res = d.decompress(data[4:])      # drop 4 byte length header
+    return (res, d.unused_data)
+
+def runzip(data):
+    res, leftovers = runzip_with_leftovers(data)
+    return res
+
+def main():
+    with open(sys.argv[1], "rb") as f:
+        with open(sys.argv[2], "wb") as o:
+            o.write(runzip(f.read()))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print("usage %s infile outfile" % sys.argv[0])
+    else:
+        main()

tools/splat_ext/rzip.py

@@ -3,21 +3,23 @@
 
 from pathlib import Path
 
-from segtypes.n64.segment import N64Segment
-from util import options
+from splat.util import options
+from splat.segtypes.segment import Segment
+import zlib
 
-import sys
-if options.get_extensions_path() not in sys.path:
-    sys.path.append('tools/splat_ext')
-import rareunzip
+# TODO: This is from rareunzip.py, but I can't get the relative import to work with splat.
+#       Figure out a way to import this function instead of copying it here.
+def runzip_with_leftovers(data):
+    d = zlib.decompressobj(wbits=-15) # raw deflate bytestream
+    res = d.decompress(data[4:])      # drop 4 byte length header
+    return (res, d.unused_data)
 
 # Rare zip format:
 # 4 byte uncompressed length followed by deflate level 9 raw payload
-class N64SegRzip(N64Segment):
-    def __init__(self, rom_start, rom_end, type, name, vram_start, extract, given_subalign, given_is_overlay, given_dir, args, yaml):
-        super().__init__(rom_start, rom_end, type, name, vram_start, extract, given_subalign, given_is_overlay, given_dir, args, yaml)
+class N64SegRzip(Segment):
+    def __init__(self, rom_start, rom_end, type, name, vram_start, args, yaml):
+        super().__init__(rom_start, rom_end, type, name, vram_start, args=args, yaml=yaml)
         self.has_subsegments = "subsegments" in yaml
-        self.yaml = yaml
         self.xor = yaml.get("xor", None)
 
     def get_default_name(self, addr):
@@ -92,7 +94,7 @@ def parse_subsegments(self):
 
         ret = []
         for i, split_file in enumerate(self.yaml["subsegments"]):
-            if type(split_file) is dict:
+            if isinstance(split_file, dict):
                 start = split_file["start"]
                 end = split_file["end"]
                 name = None if "name" not in split_file else split_file["name"]
@@ -115,7 +117,7 @@ def out_path(self) -> Path:
         return self.out_dir() / f"{self.name}.bin"
 
     def out_dir(self) -> Path:
-        return options.get_asset_path() / "rzip" / self.name
+        return options.opts.asset_path / "rzip" / self.name
 
     def split(self, rom_bytes):
         if self.has_subsegments:
@@ -155,7 +157,7 @@ def split(self, rom_bytes):
                     extension = "mp3"
             else:
                 try:
-                    result, padding = rareunzip.runzip_with_leftovers(data)
+                    result, padding = runzip_with_leftovers(data)
                 except Exception as e:
                     print("Failed to decompress file", split_file, e)
             # update total