diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d168646e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Force LF on everyfile +* text eol=lf diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 5fa2fb67..8bef31a1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -22,6 +22,7 @@ jobs: pip install mypy pip install -r requirements.txt pip install types-PyYAML + pip install -e . - name: mypy splat lib run: mypy --show-column-numbers --hide-error-context src/splat diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7f0e1d06..70406666 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -6,22 +6,55 @@ on: jobs: unit_tests: - runs-on: ubuntu-latest + name: unit_tests on ${{ matrix.os.name }} + runs-on: ${{ matrix.os.runner }} + strategy: + fail-fast: false + matrix: + os: [ + { + name: linux, + runner: ubuntu-latest, + python_venv: .venv/bin/python3, + }, + { + name: macos, + runner: macos-latest, + python_venv: .venv/bin/python3, + }, + { + name: windows, + runner: windows-latest, + python_venv: .venv/Scripts/python3, + }, + ] + steps: - name: Checkout uses: actions/checkout@v4 - name: Install dependencies - run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip wget + if: matrix.os.name == 'linux' + run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip python3-venv wget + + - name: Setup Python venv + run: | + python3 -m venv .venv - name: Install Python dependencies - run: python3 -m pip install -U -r requirements.txt + run: | + ${{ matrix.os.python_venv }} -m pip install -U -r requirements.txt + ${{ matrix.os.python_venv }} -m pip install -e . - - name: Build basic_app + - name: Build `basic_app` on ${{ matrix.os.name }} + if: matrix.os.name == 'linux' + # Linux CI checks if any of the test C code has changed without updating the generated binary run: | make -C test/basic_app clean make -C test/basic_app download_kmc make -C test/basic_app all + git diff --exit-code test/basic_app/build/basic_app.bin - name: Run the test - run: python3 test.py + run: | + ${{ matrix.os.python_venv }} test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f984274c..037e15ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # splat Release Notes +### 0.32.4 + +- Fix splat on Windows not using forward slashes on generated paths. +- Setup CI to be run on Windows and Macos too. + ### 0.32.3 - Fix "unrecognized YAML option" error if disassemble_all is provided via CLI and as a YAML option. diff --git a/README.md b/README.md index 173fda12..738250e4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin If you use a `requirements.txt` file in your repository, then you can add this library with the following line: ```txt -splat64[mips]>=0.32.3,<1.0.0 +splat64[mips]>=0.32.4,<1.0.0 ``` ### Optional dependencies diff --git a/pyproject.toml b/pyproject.toml index eed84dad..85703ccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "splat64" # Should be synced with src/splat/__init__.py -version = "0.32.3" +version = "0.32.4" description = "A binary splitting tool to assist with decompilation and modding projects" readme = "README.md" license = {file = "LICENSE"} diff --git a/src/splat/__init__.py b/src/splat/__init__.py index 46c1a9ed..76ff8112 100644 --- a/src/splat/__init__.py +++ b/src/splat/__init__.py @@ -1,7 +1,7 @@ __package_name__ = __name__ # Should be synced with pyproject.toml -__version__ = "0.32.3" +__version__ = "0.32.4" __author__ = "ethteck" from . import util as util diff --git a/src/splat/segtypes/common/c.py b/src/splat/segtypes/common/c.py index 5e1d08ec..a072478a 100644 --- a/src/splat/segtypes/common/c.py +++ b/src/splat/segtypes/common/c.py @@ -451,7 +451,7 @@ def create_c_file( c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir) c_path.parent.mkdir(parents=True, exist_ok=True) - with c_path.open("w") as f: + with c_path.open("w", newline=options.opts.c_newline) as f: f.write("\n".join(c_lines)) log.write(f"Wrote {self.name} to {c_path}") @@ -475,12 +475,12 @@ def create_asm_dependencies_file( dep_path = build_path / c_path.with_suffix(".asmproc.d") dep_path.parent.mkdir(parents=True, exist_ok=True) - with dep_path.open("w") as f: + with dep_path.open("w", newline=options.opts.c_newline) as f: if options.opts.use_o_as_suffix: o_path = build_path / c_path.with_suffix(".o") else: o_path = build_path / c_path.with_suffix(c_path.suffix + ".o") - f.write(f"{o_path}:") + f.write(f"{o_path.as_posix()}:") depend_list = [] for entry in symbols_entries: if entry.function is not None: @@ -491,7 +491,7 @@ def create_asm_dependencies_file( outpath.parent.mkdir(parents=True, exist_ok=True) depend_list.append(outpath) - f.write(f" \\\n {outpath}") + f.write(f" \\\n {outpath.as_posix()}") else: for rodata_sym in entry.rodataSyms: rodata_name = rodata_sym.getName() @@ -501,9 +501,9 @@ def create_asm_dependencies_file( outpath.parent.mkdir(parents=True, exist_ok=True) depend_list.append(outpath) - f.write(f" \\\n {outpath}") + f.write(f" \\\n {outpath.as_posix()}") f.write("\n") for depend_file in depend_list: - f.write(f"{depend_file}:\n") + f.write(f"{depend_file.as_posix()}:\n") diff --git a/src/splat/segtypes/linker_entry.py b/src/splat/segtypes/linker_entry.py index 492071ff..4a4974b8 100644 --- a/src/splat/segtypes/linker_entry.py +++ b/src/splat/segtypes/linker_entry.py @@ -51,7 +51,7 @@ def write_file_if_different(path: Path, new_content: str): if old_content != new_content: path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w") as f: + with path.open("w", newline=options.opts.c_newline) as f: f.write(new_content) @@ -522,18 +522,18 @@ def save_symbol_header(self): ) def save_dependencies_file(self, output_path: Path, target_elf_path: Path): - output = f"{target_elf_path}:" + output = f"{target_elf_path.as_posix()}:" for entry in self.dependencies_entries: if entry.object_path is None: continue - output += f" \\\n {entry.object_path}" + output += f" \\\n {entry.object_path.as_posix()}" output += "\n" for entry in self.dependencies_entries: if entry.object_path is None: continue - output += f"{entry.object_path}:\n" + output += f"{entry.object_path.as_posix()}:\n" write_file_if_different(output_path, output) def _writeln(self, line: str): @@ -561,7 +561,7 @@ def _write_symbol(self, symbol: str, value: Union[str, int]): self.header_symbols.add(symbol) def _write_object_path_section(self, object_path: Path, section: str): - self._writeln(f"{object_path}({section});") + self._writeln(f"{object_path.as_posix()}({section});") def _begin_segment( self, segment: Segment, seg_name: str, noload: bool, is_first: bool diff --git a/test.py b/test.py index d584652e..303e391c 100755 --- a/test.py +++ b/test.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 -from spimdisasm.common import FileSectionType - -from src.splat.scripts.split import * -import unittest -import io +import difflib import filecmp +import io import pathlib -from src.splat.util import symbols, options import spimdisasm +import unittest + +from src.splat import __version__ +from src.splat.scripts.split import * +from src.splat.util import symbols, options from src.splat.segtypes.common.rodata import CommonSegRodata from src.splat.segtypes.common.code import CommonSegCode from src.splat.segtypes.common.c import CommonSegC from src.splat.segtypes.common.bss import CommonSegBss -from src.splat import __version__ -import difflib class Testing(unittest.TestCase): @@ -22,28 +21,32 @@ def compare_files(self, test_path, ref_path): with io.open(test_path) as test_f, io.open(ref_path) as ref_f: self.assertListEqual(list(test_f), list(ref_f)) - def get_same_files(self, dcmp, out): + def get_same_files(self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]): for name in dcmp.same_files: out.append((name, dcmp.left, dcmp.right)) for sub_dcmp in dcmp.subdirs.values(): self.get_same_files(sub_dcmp, out) - def get_diff_files(self, dcmp, out): + def get_diff_files(self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]): for name in dcmp.diff_files: out.append((name, dcmp.left, dcmp.right)) for sub_dcmp in dcmp.subdirs.values(): self.get_diff_files(sub_dcmp, out) - def get_left_only_files(self, dcmp, out): + def get_left_only_files( + self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]] + ): for name in dcmp.left_only: out.append((name, dcmp.left, dcmp.right)) for sub_dcmp in dcmp.subdirs.values(): self.get_left_only_files(sub_dcmp, out) - def get_right_only_files(self, dcmp, out): + def get_right_only_files( + self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]] + ): for name in dcmp.right_only: out.append((name, dcmp.left, dcmp.right)) @@ -188,7 +191,7 @@ def test_add_symbol_to_spim_section(self): vram=0x40000000, filename="test", words=[], - sectionType=FileSectionType.Text, + sectionType=spimdisasm.common.FileSectionType.Text, segmentVromStart=0x0, overlayCategory=None, ) diff --git a/test/basic_app/.gitignore b/test/basic_app/.gitignore index df3d40d8..7703c914 100644 --- a/test/basic_app/.gitignore +++ b/test/basic_app/.gitignore @@ -1,3 +1,4 @@ -build/ -split/ -gcc-2.7.2/ +/build/ +/split/ +/gcc-2.7.2/ +!/expected/ diff --git a/test/basic_app/build/basic_app.bin b/test/basic_app/build/basic_app.bin new file mode 100755 index 00000000..6bc57623 Binary files /dev/null and b/test/basic_app/build/basic_app.bin differ diff --git a/test/basic_app/expected/.splache b/test/basic_app/expected/.splache index 4bf620c1..b54ddfbd 100644 Binary files a/test/basic_app/expected/.splache and b/test/basic_app/expected/.splache differ diff --git a/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s b/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s index 324fd1b2..f0a6022d 100644 --- a/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s +++ b/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s @@ -1,6 +1,3 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - .section .rodata .align 3 glabel jtbl_80000518 diff --git a/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s b/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s index c6c33477..cbb4eade 100644 --- a/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s +++ b/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s @@ -1,6 +1,3 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - glabel func_800004A0 /* 10A0 800004A0 27BDFFE8 */ addiu $sp, $sp, -0x18 /* 10A4 800004A4 AFBF0014 */ sw $ra, 0x14($sp) diff --git a/test/basic_app/expected/basic_app.d b/test/basic_app/expected/basic_app.d new file mode 100644 index 00000000..6e5b2a01 --- /dev/null +++ b/test/basic_app/expected/basic_app.d @@ -0,0 +1,21 @@ +test/basic_app/split/build/basic_app_target.elf: \ + build/asm/header.o \ + build/assets/dummy_ipl3.o \ + build/src/main.o \ + build/asm/handwritten.o \ + build/asm/data/main.data.o \ + build/asm/handwritten.o \ + build/src/main.o \ + build/asm/handwritten.o \ + build/asm/data/main.bss.o \ + build/asm/handwritten.o +build/asm/header.o: +build/assets/dummy_ipl3.o: +build/src/main.o: +build/asm/handwritten.o: +build/asm/data/main.data.o: +build/asm/handwritten.o: +build/src/main.o: +build/asm/handwritten.o: +build/asm/data/main.bss.o: +build/asm/handwritten.o: diff --git a/test/basic_app/expected/basic_app.ld b/test/basic_app/expected/basic_app.ld index 402aa02e..56689bcc 100644 --- a/test/basic_app/expected/basic_app.ld +++ b/test/basic_app/expected/basic_app.ld @@ -7,7 +7,7 @@ SECTIONS { FILL(0x00000000); header_DATA_START = .; - build/split/asm/header.o(.data); + build/asm/header.o(.data); header_DATA_END = .; header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START); } @@ -21,7 +21,7 @@ SECTIONS { FILL(0x00000000); dummy_ipl3_DATA_START = .; - build/split/assets/dummy_ipl3.o(.data); + build/assets/dummy_ipl3.o(.data); . = ALIGN(., 16); dummy_ipl3_DATA_END = .; dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START); @@ -38,20 +38,20 @@ SECTIONS { FILL(0x00000000); boot_TEXT_START = .; - build/split/src/main.o(.text); - build/split/asm/handwritten.o(.text); + build/src/main.o(.text); + build/asm/handwritten.o(.text); . = ALIGN(., 16); boot_TEXT_END = .; boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START); boot_DATA_START = .; - build/split/asm/data/main.data.o(.data); - build/split/asm/handwritten.o(.data); + build/asm/data/main.data.o(.data); + build/asm/handwritten.o(.data); . = ALIGN(., 16); boot_DATA_END = .; boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START); boot_RODATA_START = .; - build/split/src/main.o(.rodata); - build/split/asm/handwritten.o(.rodata); + build/src/main.o(.rodata); + build/asm/handwritten.o(.rodata); . = ALIGN(., 16); boot_RODATA_END = .; boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START); @@ -61,8 +61,8 @@ SECTIONS { FILL(0x00000000); boot_BSS_START = .; - build/split/asm/data/main.bss.o(.bss); - build/split/asm/handwritten.o(.bss); + build/asm/data/main.bss.o(.bss); + build/asm/handwritten.o(.bss); . = ALIGN(., 16); boot_BSS_END = .; boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START); diff --git a/test/basic_app/expected/build/test/basic_app/split/src/main.asmproc.d b/test/basic_app/expected/build/test/basic_app/split/src/main.asmproc.d new file mode 100644 index 00000000..4ec280ec --- /dev/null +++ b/test/basic_app/expected/build/test/basic_app/split/src/main.asmproc.d @@ -0,0 +1,7 @@ +test/basic_app/split/build/test/basic_app/split/src/main.o: \ + test/basic_app/split/asm/nonmatchings/main/D_80000510.s \ + test/basic_app/split/asm/nonmatchings/main/func_80000400.s \ + test/basic_app/split/asm/nonmatchings/main/func_800004A0.s +test/basic_app/split/asm/nonmatchings/main/D_80000510.s: +test/basic_app/split/asm/nonmatchings/main/func_80000400.s: +test/basic_app/split/asm/nonmatchings/main/func_800004A0.s: diff --git a/test/basic_app/splat.yaml b/test/basic_app/splat.yaml index dcb164dd..ba5b749b 100644 --- a/test/basic_app/splat.yaml +++ b/test/basic_app/splat.yaml @@ -1,20 +1,22 @@ options: + base_path: split platform: n64 - compiler: GCC + compiler: KMC basename: basic_app - base_path: . build_path: build - target_path: build/basic_app.bin - asm_path: split/asm - src_path: split/src - ld_script_path: split/basic_app.ld - cache_path: split/.splache - symbol_addrs_path: split/generated.symbols.txt - undefined_funcs_auto_path: split/undefined_funcs_auto.txt - undefined_syms_auto_path: split/undefined_syms_auto.txt - asset_path: split/assets - compiler: GCC + target_path: ../build/basic_app.bin + elf_path: build/basic_app_target.elf + asm_path: asm + src_path: src + ld_script_path: basic_app.ld + cache_path: .splache + symbol_addrs_path: generated.symbols.txt + undefined_funcs_auto_path: undefined_funcs_auto.txt + undefined_syms_auto_path: undefined_syms_auto.txt + asset_path: assets o_as_suffix: True + ld_dependencies: True + create_asm_dependencies: True segments: - name: header type: header diff --git a/test/test_gen_expected.sh b/test/test_gen_expected.sh index e0e27570..20a3e85b 100755 --- a/test/test_gen_expected.sh +++ b/test/test_gen_expected.sh @@ -7,7 +7,7 @@ export SPIMDISASM_ASM_GENERATED_BY="False" # Ensure we start from a clean state rm -rf test/basic_app/split -python3 ./split.py test/basic_app/splat.yaml --use-cache +python3 -m splat split test/basic_app/splat.yaml --use-cache rm -rf test/basic_app/expected cp -r test/basic_app/split test/basic_app/expected