diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 17a85ac..415fe58 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,7 +24,6 @@ env: PACKWIZ_COMMIT: 0bb89a4872d8dc2c45af251345ee780cab7ab9ad PACKWIZ_DIR: /tmp/packwiz_artifact PACKWIZ: /tmp/packwiz_artifact/packwiz - PACKWIZ_BOOTSTRAP_VERSION: "v0.0.3" jobs: build_test_deploy: @@ -43,7 +42,7 @@ jobs: - name: Cache Packwiz id: cache-packwiz - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.PACKWIZ_DIR }} key: packwiz-${{ env.PACKWIZ_COMMIT }} @@ -65,18 +64,33 @@ jobs: - name: Build pack run: python scripts/assemble_packwiz.py - # Test - # - name: Cache Packwiz Bootstrap - # uses: actions/cache@v3 - # with: - # path: run/packwiz-installer - # key: packwiz-bootstrap-${{ env.PACKWIZ_BOOTSTRAP_VERSION }} - # - uses: actions/setup-java@v4 - # with: - # distribution: 'temurin' # See 'Supported distributions' for available options - # java-version: '21' - # - name: Test pack - # run: python scripts/run_test.py + # Setup test-runner cache + - name: Generate cache key + run: python scripts/run_test.py + env: + GENERATE_DESIRED_CACHE_STATE_AND_EXIT: true + - name: Cache test-runner (static cache) + uses: actions/cache@v4 + with: + path: | + run/cache-dynamic + key: test-run-static-cache-${{ hashFiles('run/desired_cache_state_for_static_cache.json') }} # Try and get the desired state + restore-keys: test-run-static-cache- # But fall back if needed + - name: Cache test-runner (dynamic cache) + uses: actions/cache@v4 + with: + path: | + run/cache-dynamic + key: test-run-dynamic-cache-${{ hashFiles('generated/pack/pack.toml') }} # pack.toml contains hashes for all other files + restore-keys: test-run-dynamic-cache- + + # Run tests + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' # See 'Supported distributions' for available options + java-version: '21' + - name: Test pack + run: python scripts/run_test.py # Deploy diff --git a/.github/workflows/pull_platform.yml b/.github/workflows/pull_platform.yml index c3a36cd..a2a8cd8 100644 --- a/.github/workflows/pull_platform.yml +++ b/.github/workflows/pull_platform.yml @@ -28,7 +28,7 @@ jobs: - name: Cache Packwiz id: cache-packwiz - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.PACKWIZ_DIR }} key: packwiz-${{ env.PACKWIZ_COMMIT }} diff --git a/README.md b/README.md index ae8ebba..7ad3636 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ ## Repository setup Please create an (empty) `pack/index.toml`. -This will ensure packwiz commands work correctly in the `pack/` directory. -Scripts require only a recent version of python to run (>=3.11). No dependencies are required except when running tests. +This will ensure packwiz commands work correctly in the `pack/` directory. `tomli_w` is the only dependency needed to run the scripts. The `pack/` directory contains the bulk of the pack. The files in here can be updated using the [packwiz](https://github.com/packwiz/packwiz) utility. The final pack will also include all submissions (and their dependencies), which are pulled from ModFest's platform api. The `pack/` directory will always take priority and can be used to override submitted mods. Submissions can be excluded altogether by putting it in the `platform.ignore` file. diff --git a/scripts/assemble_packwiz.py b/scripts/assemble_packwiz.py index 50d8225..d0d57ca 100755 --- a/scripts/assemble_packwiz.py +++ b/scripts/assemble_packwiz.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -import shutil import json -import common import os -import subprocess import re +import shutil +import subprocess +from typing import Any, TypeAlias, TypedDict + +import common import tomli_w + def main(): repo_root = common.get_repo_root() submission_lock_file = repo_root / "submissions-lock.json" @@ -24,7 +27,7 @@ def main(): exclusions = list(filter(lambda l : len(l) > 0, [re.sub("#.*", "", l.strip()) for l in common.read_file(exclude_file).split("\n")])) - locked_data = json.loads(common.read_file(submission_lock_file)) + locked_data: SubmissionLockfileFormat = json.loads(common.read_file(submission_lock_file)) for platformid, moddata in locked_data.items(): if not "files" in moddata: raise RuntimeError(f"lock data for {platformid} is invalid. Does not contain file key") @@ -49,4 +52,10 @@ def main(): subprocess.run([packwiz, "refresh"]) if __name__ == "__main__": - main() \ No newline at end of file + main() + +# For type hints +class SubmissionLockfileEntry(TypedDict): + url: str + files: dict[str, Any] +SubmissionLockfileFormat: TypeAlias = dict[str, SubmissionLockfileEntry] \ No newline at end of file diff --git a/scripts/assemble_unsup.py b/scripts/assemble_unsup.py index b9614da..0f42165 100755 --- a/scripts/assemble_unsup.py +++ b/scripts/assemble_unsup.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import sys -import re -from zipfile import ZipFile -import zipfile import io import json +import re +import sys import urllib.request +import zipfile +from typing import Any +from zipfile import ZipFile -from common import Ansi import common +from common import Ansi + def main(): repo_root = common.get_repo_root() @@ -17,7 +19,7 @@ def main(): generated_dir = common.get_generated_dir() url = common.env("URL") - if url == None: + if url is None: print(f"{Ansi.ERROR}Please set the URL environment variable to the public url for this pack{Ansi.RESET}") sys.exit(1) if not url.endswith("pack.toml"): @@ -102,7 +104,7 @@ def create_unsup_patch(unsup_version): # Creates the mmc-pack.json file, which stores "dependency" information for prism/multimc # The most important thing is that it defines the minecraft version and launcher used def create_mmc_meta(packwiz_info, unsup_version): - meta = {} + meta: Any = {} meta["formatVersion"] = 1 components = [] @@ -144,7 +146,7 @@ def create_instance_config(packwiz_info, icon_name): # Creates the unsup config file, which tells unsup where # to download mods from -def create_unsup_ini(url, constants): +def create_unsup_ini(url: str, constants): colour_entries = [] for colour_key in unsup_colours: colour_value = common.get_colour(constants, "_unsup_"+colour_key) diff --git a/scripts/common.py b/scripts/common.py index 3459543..9cca548 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,12 +1,14 @@ +import hashlib import json import os +import re import shutil import time -from pathlib import Path -import hashlib import tomllib from dataclasses import dataclass -import re +from pathlib import Path +from typing import Any, Callable, TypedDict, TypeVar, Unpack, overload + class Ansi: BOLD = '\033[1m' @@ -17,42 +19,45 @@ class Ansi: ERROR = RED_FG+BOLD RESET = '\033[0m' -def check_packwiz(): +def check_packwiz() -> Path: + """Get the current packwiz executable""" packwiz = env("PACKWIZ", default="packwiz") if p := shutil.which(packwiz): - return p + return Path(p) else: raise RuntimeError(f"!!! Couldn't find packwiz (looked for '{packwiz}'). Please put packwiz on your path or set the PACKWIZ environment variable to a packwiz executable") -def check_java(): +def check_java() -> Path: + """Get the current java executable""" java = "java" if "JAVA_HOME" in os.environ: - java = Path(os.environ["JAVA_HOME"]) / "bin/java" - if not java.exists(): + java_p = Path(os.environ["JAVA_HOME"]) / "bin/java" + if not java_p.exists(): raise RuntimeError(f"!!! JAVA_HOME is invalid. {java} does not exist") - return java + return java_p else: - if java := shutil.which("java"): - return p + if resolved_java := shutil.which("java"): + return Path(resolved_java) else: raise RuntimeError(f"!!! Couldn't find java on path. Please add it or set JAVA_HOME") -def get_repo_root(): +def get_repo_root() -> Path: # This file should be located in /scripts/common.py, so the root # is one directory up from this one return Path(os.path.join(os.path.dirname(__file__), '..')) -def get_generated_dir(): +def get_generated_dir() -> Path: dir = env("OUTPUT_DIR", default=(get_repo_root() / "generated")) + dir = Path(dir) if not dir.exists(): dir.mkdir(exist_ok=True, parents=True) return dir -def read_file(path): +def read_file(path: os.PathLike) -> str: with open(path, "r") as f: return f.read() -def fix_packwiz_pack(pack_toml): +def fix_packwiz_pack(pack_toml: Path): data = tomllib.loads(read_file(pack_toml)) index = pack_toml.parent / data["index"]["file"] if not index.exists(): @@ -62,32 +67,42 @@ class JSONWithCommentsDecoder(json.JSONDecoder): def __init__(self, **kw): super().__init__(**kw) - def decode(self, s: str): + def decode(self, s, _w): s = '\n'.join(l if not l.lstrip().startswith('//') else '' for l in s.split('\n')) - return super().decode(s) + return super().decode(s, _w) -def jsonc_at_home(input): +def jsonc_at_home(input: str | bytes) -> Any: return json.loads(input, cls=JSONWithCommentsDecoder) -def hash(values: list[str]): +def hash(values: list[str]) -> str: hasher = hashlib.sha256() for value in values: hasher.update(value.encode("UTF-8")) return hasher.hexdigest() -def env(env: str, **kwargs): +# overloads for proper type checking +T = TypeVar('T') +@overload +def env(env: str, *, default: None = None) -> None | str: ... +@overload +def env(env: str, *, default: T) -> T | str: ... + +def env(env: str, *, default: Any = None) -> Any | str: if env in os.environ: return os.environ[env] else: - return kwargs.get("default") + return default + +class Constants(TypedDict): + colours: dict[str, str] -def get_colour(parsed_constants, key): +def get_colour(parsed_constants: Constants, key: str) -> str: """Given a parsed constants.jsonc, retrieves a colour by key. Returns a value in the form of #FFFFFF""" if not key.startswith("_"): raise RuntimeError("Scripts should only depend on colour keys starting with an underscore") def get_inner(k): v = parsed_constants["colours"].get(k) - if v == None: + if v is None: return None elif v.startswith("."): return get_inner(v[1:]) @@ -98,16 +113,29 @@ def get_inner(k): return get_inner(key) class Ratelimiter: - def __init__(self, time): + def __init__(self, time: float): # Time is given in seconds, convert to nanoseconds self.wait_time = time - self.last_action = 0 + self.last_action: float = 0 def limit(self): time.sleep(max(0, self.wait_time - (time.time() - self.last_action))) self.last_action = time.time() -def parse_packwiz(pack_toml_file): +@dataclass +class PackwizPackInfo: + name: str | None + author: str | None + pack_version: str | None + minecraft_version: str + loader: str + loader_version: str + + def safe_name(self) -> str: + assert self.name is not None + return re.sub("[^a-zA-Z0-9]+", "-", self.name) + +def parse_packwiz(pack_toml_file: Any) -> PackwizPackInfo: pack_toml = tomllib.loads(read_file(pack_toml_file)) version_data = pack_toml["versions"] @@ -136,16 +164,4 @@ def parse_packwiz(pack_toml_file): version_data["minecraft"], loader, loader_version - ) - -@dataclass -class PackwizPackInfo: - name: str - author: str - pack_version: str - minecraft_version: str - loader: str - loader_version: str - - def safe_name(self): - return re.sub("[^a-zA-Z0-9]+", "-", self.name) \ No newline at end of file + ) \ No newline at end of file diff --git a/scripts/pull_platform.py b/scripts/pull_platform.py index 75657df..d4a1209 100755 --- a/scripts/pull_platform.py +++ b/scripts/pull_platform.py @@ -1,16 +1,19 @@ #!/usr/bin/env python3 -import urllib.request import json -import subprocess -import tomllib -import tempfile import os -from pathlib import Path import shutil +import subprocess import sys +import tempfile +import tomllib +import urllib.request +from pathlib import Path + import common +from assemble_packwiz import SubmissionLockfileFormat from common import Ansi + def main(): modrinth_api = "https://api.modrinth.com/v2" repo_root = common.get_repo_root() @@ -37,7 +40,7 @@ def main(): # Update the lock file # Read the needed files and transform the submission data into a dict where the ids are keys - lock_data = json.loads(common.read_file(submission_lock_file)) if submission_lock_file.exists() else {} + lock_data: SubmissionLockfileFormat = json.loads(common.read_file(submission_lock_file)) if submission_lock_file.exists() else {} submissions_by_id = {s["id"]:s for s in submission_data} # Remove stale data @@ -49,14 +52,15 @@ def main(): platform_info = submissions_by_id[mod_id] lock_info = lock_data.get(mod_id) # Might be None # If the url changes we need to update the lock data. This is the only use of 'url' in the lock file - if lock_info == None or lock_info["url"] != platform_info["download"]: + if lock_info is None or lock_info["url"] != platform_info["download"]: print(f"Updating lock data for {mod_id}") lock_info = {} # Reset the lock info for this mod + assert lock_info is not None # mypy is quite stupid lock_info["url"] = platform_info["download"] # We steal packwiz's dependency resolution by making a quick packwiz dir - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = Path(tmpdir) + with tempfile.TemporaryDirectory() as tmpdir_name: + tmpdir = Path(tmpdir_name) # Run commands in the temporary directory os.chdir(tmpdir) diff --git a/scripts/run_test.py b/scripts/run_test.py index 6f5fb01..3aad64b 100755 --- a/scripts/run_test.py +++ b/scripts/run_test.py @@ -1,12 +1,22 @@ #!/usr/bin/env python3 -import tomllib +import json import os -import urllib.request +import shlex +import shutil import subprocess -from pathlib import Path import sys +import tempfile +import urllib.request +from pathlib import Path +from typing import Any, NewType, Optional +import assemble_packwiz import common +from common import Ansi + +FABRIC_INSTALLER_VERSION = "1.0.1" +PACKWIZ_BOOTSTRAP_VERSION = "v0.0.3" # https://github.com/packwiz/packwiz-installer-bootstrap +MC_TEST_INJECTOR_VERSION = "v1.0.0" # https://github.com/TheEpicBlock/mc-test-injector def main(): repo_root = common.get_repo_root() @@ -15,124 +25,289 @@ def main(): pack_toml_file = pack / "pack.toml" test_server_working = Path(common.env("WORK_DIR", default=(repo_root / "run"))) + # Run the pack assembly script + assemble_packwiz.main() + if not pack.exists(): print(f"{pack} does not exist") - raise Exception("Error, couldn't find pack. Please ensure that it was created (run assemble_packwiz.py)") + raise Exception("Error, couldn't find pack. assemble_packwiz.py might've failed") if not pack_toml_file.exists(): print(f"{pack_toml_file} does not exist") raise Exception("Pack is not a valid packwiz pack (pack.toml) doesn't exist") # Parse pack information - pack_toml = tomllib.loads(common.read_file(pack_toml_file)) + pack_info = common.parse_packwiz(pack_toml_file) - print("Testing modpack "+str(pack_toml.get("name"))+" "+str(pack_toml.get("version"))) + print(f"Testing modpack {pack_info.name} {pack_info.pack_version}") - version_data = pack_toml["versions"] - if not "minecraft" in version_data: - raise Exception("pack.toml doesn't define a minecraft version") - - mc_version = version_data["minecraft"] + mc_version = pack_info.minecraft_version + loader = pack_info.loader + loader_version = pack_info.loader_version - # detect loader - supported_loaders = ["fabric", "neoforge"] + print(f"Setting up a {loader} {loader_version} server for {mc_version}") - for v in version_data: - if v != "minecraft" and v not in supported_loaders: - raise Exception(f"pack is using unsupported software: {v}") + # Various run dirs and files + # This is the cache for things that don't change very often + static_cache_dir = test_server_working / "cache-static" + cache_state_file = static_cache_dir / "cache_state.json" # Info about the cache + cached_server_dir = static_cache_dir / "server" # Dir containing the server jar and libraries + cached_packwiz_dir = static_cache_dir / "packwiz" # Dir containing packwiz installer and packwiz bootstrap + cached_injector_dir = static_cache_dir / "mc-test-injector" # Dir where mc-test-injector will be downloaded to - loaders = {k:v for k, v in version_data.items() if k in supported_loaders} - if len(loaders) >= 2: - raise Exception("pack is using multiple loaders, unsure which one to use: ["+", ".join(loaders.keys())+"]") - if len(loaders) == 0: - raise Exception("pack does not seem to define a loader") - - loader = list(loaders.keys())[0] - loader_version = list(loaders.values())[0] + # This is the cache that does change quite often (eg, whenever the mods change) + # These are all managed by external programs, so they don't need a state file + dynamic_cache_dir = test_server_working / "cache-dynamic" + cached_pack_dir = dynamic_cache_dir / "pack" # Dir containing an instance of the pack + runtime_cache = dynamic_cache_dir / "runtime" # Dirs which are known to contain caches maintained by the server (e.g .fabric) + exec_dir = test_server_working / "exec" # Where the server will end up running - print(f"Setting up a {loader} ({loader_version}) server") + cached_server_dir.mkdir(exist_ok=True, parents=True) + cached_pack_dir.mkdir(exist_ok=True, parents=True) + cached_packwiz_dir.mkdir(exist_ok=True, parents=True) + cached_injector_dir.mkdir(exist_ok=True, parents=True) + runtime_cache.mkdir(exist_ok=True, parents=True) + exec_dir.mkdir(exist_ok=True, parents=True) - # Set up modloader - server_hash = common.hash([mc_version, loader, loader_version]) - minecraft_dir = test_server_working / "loader" / server_hash - game_dir = test_server_working / "game" - minecraft_cached = minecraft_dir.exists() - server_jar = None - if loader == "fabric": - server_jar = minecraft_dir / "server.jar" - if not minecraft_cached: - print("Download fabric server jar") - minecraft_dir.mkdir(exist_ok=True, parents=True) - fabric_installer = "1.0.1" - urllib.request.urlretrieve(f"https://meta.fabricmc.net/v2/versions/loader/{mc_version}/{loader_version}/{fabric_installer}/server/jar", server_jar) - elif loader == "neoforge": - installer_file = minecraft_dir / f"installer.jar" - if not minecraft_cached: - print("! Running neoforge installer") - minecraft_dir.mkdir(exist_ok=True, parents=True) - urllib.request.urlretrieve(f"https://maven.neoforged.net/releases/net/neoforged/neoforge/{loader_version}/neoforge-{loader_version}-installer.jar", installer_file) - subprocess.run(["java", "-jar", installer_file, "--install-server", minecraft_dir]) - print("! Neoforge installer ran") - # TODO - # server_run_cmd = [minecraft_dir / ("run.bat" if os.name == "nt" else "run.sh")] + # Generate the desired cache state so we can compare it + desired_cache_state = { + "server": common.hash([mc_version, loader, loader_version]), + "pw_bootstrap": PACKWIZ_BOOTSTRAP_VERSION, + "mc-test-injector": MC_TEST_INJECTOR_VERSION + } + if common.env("GENERATE_DESIRED_CACHE_STATE_AND_EXIT") == "true": + save_cache_state(desired_cache_state, test_server_working / "desired_cache_state_for_static_cache.json") + sys.exit() + return + + # Read the file describing the state of the current cache + if cache_state_file.exists(): + try: + cached_state = json.loads(common.read_file(cache_state_file)) + except Exception: + print(f"Failed to load cache state, ignoring it") + cached_state = {} else: - raise RuntimeError(f"{loader} not handled") + cached_state = {} - # Accept eula - if loader == "fabric": - eula = game_dir / "eula.txt" - elif loader == "neoforge": - eula = minecraft_dir / "eula.txt" - if not eula.exists(): - eula.parent.mkdir(exist_ok=True, parents=True) - eula.touch() - with open(eula, "w") as f: - f.write("eula=true") - - # Set up the packwiz and game dir - packwiz_dir = test_server_working / "packwiz-installer" - packwiz_bootstrap = packwiz_dir / "packwiz-bootstrap.jar" - if not packwiz_bootstrap.exists(): - print("Installing packwiz bootstrap") - bootstrap_version = common.env("PACKWIZ_BOOTSTRAP_VERSION", default="v0.0.3") - packwiz_dir.mkdir(exist_ok=True) - urllib.request.urlretrieve(f"https://github.com/packwiz/packwiz-installer-bootstrap/releases/download/{bootstrap_version}/packwiz-installer-bootstrap.jar", packwiz_bootstrap) + # Make sure we have an install of the server files + server_hash = desired_cache_state["server"] + if server_hash != cached_state.get("server"): + print("Existing cached server files are stale. Deleting it.") + shutil.rmtree(cached_server_dir) + shutil.rmtree(runtime_cache) + cached_state["server"] = None + save_cache_state(cached_state, cache_state_file) # Don't forget to immediatly save any changes to the state + elif err := validate_server(loader, cached_server_dir): + print(f"{Ansi.WARN}Something is wrong with the cached server:{Ansi.RESET} {err}") + print("Removing cached server files") + shutil.rmtree(cached_server_dir) + shutil.rmtree(runtime_cache) + cached_state["server"] = None + save_cache_state(cached_state, cache_state_file) # Don't forget to immediatly save any changes to the state + + if cached_state.get("server") == None: + # Set up new server files + setup_server(java, mc_version, loader, loader_version, cached_server_dir) + # Update cache state to reflect the newly installed server files + cached_state["server"] = server_hash + save_cache_state(cached_state, cache_state_file) + else: + print(f"Cache hit: a {mc_version} server using {loader} {loader_version} is in the cache") + + # Make sure we have an install of packwiz + bootstrap_version = desired_cache_state["pw_bootstrap"] + if bootstrap_version != cached_state.get("pw_bootstrap"): + print("Installed packwiz bootstrap is stale. Deleting it.") + shutil.rmtree(cached_packwiz_dir) + cached_state["pw_bootstrap"] = None + save_cache_state(cached_state, cache_state_file) + elif err := validate_packwiz(cached_packwiz_dir): + print(f"{Ansi.WARN}Something is wrong with the cached packwiz installer or bootstrap:{Ansi.RESET} {err}") + shutil.rmtree(cached_packwiz_dir) + cached_state["pw_bootstrap"] = None + save_cache_state(cached_state, cache_state_file) + + if cached_state.get("pw_bootstrap") == None: + # Set up new server files + setup_packwiz_bootstrap(java, bootstrap_version, cached_packwiz_dir) + # Update cache state to reflect the newly installed packwiz + cached_state["pw_bootstrap"] = bootstrap_version + save_cache_state(cached_state, cache_state_file) + else: + print(f"Cache hit: packwiz bootstrap {bootstrap_version} is in the cache") + + # Make sure we have an install of mc test injector + injector_version = desired_cache_state["mc-test-injector"] + if injector_version != cached_state.get("mc-test-injector"): + print("Installed mc-test-injector is stale. Deleting it.") + shutil.rmtree(cached_injector_dir) + cached_state["mc-test-injector"] = None + save_cache_state(cached_state, cache_state_file) + elif err := validate_test_injector(cached_injector_dir): + print(f"{Ansi.WARN}Something is wrong with the cached mc-test-injector:{Ansi.RESET} {err}") + shutil.rmtree(cached_injector_dir) + cached_state["mc-test-injector"] = None + save_cache_state(cached_state, cache_state_file) + + if cached_state.get("mc-test-injector") == None: + # Set up new server files + setup_mc_test_injector(java, injector_version, cached_injector_dir) + # Update cache state to reflect the newly installed mc-test-injector + cached_state["mc-test-injector"] = injector_version + save_cache_state(cached_state, cache_state_file) + else: + print(f"Cache hit: mc-test-injector {injector_version} is in the cache") + + # Update the pack dir; + # it should have all the files in the pack downloaded + # packwiz should take care of keeping this synchronized + packwiz_bootstrap = cached_packwiz_dir / "packwiz_bootstrap.jar" + print(f"Invoking packwiz installer to synchronize {cached_pack_dir.relative_to(repo_root)}") subprocess.run([ java, "-jar", packwiz_bootstrap, "--no-gui", # Ensures bootstrap installs packwiz to `packwiz_dir` for caching reasons - "--bootstrap-main-jar", packwiz_dir / "packwiz-installer.jar", - "--pack-folder", game_dir, + "--bootstrap-main-jar", cached_packwiz_dir / "packwiz-installer.jar", + "--pack-folder", cached_pack_dir, + "-s", "server", # Tell packwiz to install only server files f"file://{pack_toml_file}" ]) - - # Setup the testing java agent - agent_jar = common.env("AGENT") - - # Run the server - worlds_dir = test_server_working / "worlds" - worlds_dir.mkdir(exist_ok=True) - server_run_cmd = [java] - if agent_jar: - agent_jar = Path(agent_jar).resolve() - server_run_cmd += [f"-javaagent:{agent_jar}"] - server_run_cmd += ["-jar", server_jar] - server_run_cmd += ["--nogui"] - server_run_cmd += ["--universe", worlds_dir] + # Symlink the cached server files and cached pack files + shutil.rmtree(exec_dir) + exec_dir.mkdir(parents=True) + for f in cached_server_dir.iterdir(): + os.symlink(f, exec_dir / (f.relative_to(cached_server_dir)), target_is_directory=f.is_dir()) + for f in cached_pack_dir.rglob("*"): + if f.is_file(): + # We do *not* symlink entire directories. Instead we symlink individual files. + # This is because NeoForge doesn't like it. + # Also, it helps prevents stuff from accidentally modifying our cache, so that's nice + dest = exec_dir / (f.relative_to(cached_pack_dir)) + dest.parent.mkdir(exist_ok=True, parents=True) + os.symlink(f, dest, target_is_directory=False) - if loader == "fabric": - # Will look for the mods here, but it'll also dump libraries here unfortunately - os.chdir(game_dir) - elif loader == "neoforge": - server_run_cmd += ["--gameDir", game_dir] - os.chdir(minecraft_dir) # Won't launch unless these match - print(f"Running minecraft with {server_run_cmd}") + dotfabric = runtime_cache / ".fabric" + dotfabric.mkdir(exist_ok=True, parents=True) + os.symlink(dotfabric, exec_dir / ".fabric", target_is_directory=True) + + dotconnector = runtime_cache / ".connector" + dotconnector.mkdir(exist_ok=True, parents=True) + os.symlink(dotconnector, exec_dir / "mods" / ".connector", target_is_directory=True) - result = subprocess.run(server_run_cmd, timeout=120) + # Accept eula + eula = exec_dir / "eula.txt" + if not eula.exists(): + eula.touch() + with open(eula, "w") as file: + file.write("eula=true") + + # Setup mc-test-injector + test_injector = cached_injector_dir / "McTestInjector.jar" + + # Clear any lingering crash reports + crashreport_dir = exec_dir / "crash-reports" + if crashreport_dir.exists(): + shutil.rmtree(crashreport_dir) + + # Run the server + test_injector = Path(test_injector).resolve() + java_args = [f"-javaagent:{test_injector}"] + mc_args = ["--nogui"] + + sys.stdout.flush() # Prevents python's output from appearing after mc's + os.chdir(exec_dir) + result = run_server(exec_dir, java, loader, java_args, mc_args, timeout=240) + if result.returncode != 0: print(f"! Minecraft returned status code {result.returncode}") sys.exit(1) + else: + print(f"Minecraft exited with status code 0") + + if crashreport_dir.exists() and len(list(crashreport_dir.iterdir())) > 0: + print(f"! Found files in the crash-reports directory. Marking test as failed") + sys.exit(2) + +def save_cache_state(state, file): + # This is nice to store, for if we ever make breaking changes + state["script_version"] = 1 + with open(file, "w") as f: + f.write(json.dumps(state, sort_keys=True)) + +def setup_server(java, mc_version, loader, loader_version, directory): + """Install the server files and libraries for a given version. The given directory should be empty""" + directory.mkdir(exist_ok=True, parents=True) + with tempfile.TemporaryDirectory() as installer_tmp: + installer = Path(installer_tmp) / "installer.jar" + + # Download and run the appropriate installer + if loader == "fabric": + print(f"Downloading {loader}-installer {FABRIC_INSTALLER_VERSION} to {installer}") + urllib.request.urlretrieve(f"https://maven.fabricmc.net/net/fabricmc/fabric-installer/{FABRIC_INSTALLER_VERSION}/fabric-installer-{FABRIC_INSTALLER_VERSION}.jar", installer) + subprocess.run([java, "-jar", installer, + "server", + "-dir", directory, + "-mcversion", mc_version, + "-loader", loader_version, + "-downloadMinecraft" # Makes fabric install the server jar as well + ]) + elif loader == "neoforge": + print(f"Downloading {loader} installer for {loader_version} to {installer}") + urllib.request.urlretrieve(f"https://maven.neoforged.net/releases/net/neoforged/neoforge/{loader_version}/neoforge-{loader_version}-installer.jar", installer) + # NeoForge installers are always meant for a certain neoforge and minecraft version + subprocess.run([java, "-jar", installer, "--install-server", directory]) + else: + raise RuntimeError(f"Unknown loader {loader}, can't install server files") + # Validate result + if err := validate_server(loader, directory): + raise RuntimeError(f"Failed to install server files: {err}") + +def validate_server(loader, server_dir) -> str | None: + if loader == "fabric" and not (server_dir / "fabric-server-launch.jar").exists(): + return "Fabric servers should have a fabric-server-launch.jar" + if loader == "neoforge" and not (server_dir / "user_jvm_args.txt").exists(): + # This is the behaviour as of NeoForge 21.1.64 + return "NeoForge should set up a user_jvm_args.txt file. Did the way neoforge servers are set up change?" + if not (server_dir / "libraries").exists(): + return "The server directory should have a libraries folder" + return None + +def setup_packwiz_bootstrap(java, bootstrap_version, directory): + print(f"Downloading packwiz bootstrap {bootstrap_version}") + directory.mkdir(exist_ok=True, parents=True) + urllib.request.urlretrieve(f"https://github.com/packwiz/packwiz-installer-bootstrap/releases/download/{bootstrap_version}/packwiz-installer-bootstrap.jar", directory / "packwiz_bootstrap.jar") + +def validate_packwiz(packwiz_dir) -> str | None: + if not (packwiz_dir / "packwiz_bootstrap.jar").exists(): + return "packwiz_bootstrap.jar should exist" + return None + +def setup_mc_test_injector(java, injector_version, directory): + print(f"Downloading mc-test-injector {injector_version}") + directory.mkdir(exist_ok=True, parents=True) + unprefixed = injector_version + if unprefixed.startswith("v"): + unprefixed = unprefixed[1:] + urllib.request.urlretrieve(f"https://github.com/TheEpicBlock/mc-test-injector/releases/download/{injector_version}/McTestInjector-{unprefixed}.jar", directory / "McTestInjector.jar") + +def validate_test_injector(packwiz_dir) -> str | None: + if not (packwiz_dir / "McTestInjector.jar").exists(): + return "McTestInjector.jar should exist" + return None + +def run_server(exec_dir, java, loader, java_args, mc_args, **kwargs) -> subprocess.CompletedProcess[Any]: + if loader == "fabric": + return subprocess.run([java] + java_args + ["-jar", exec_dir / "fabric-server-launch.jar"] + mc_args, **kwargs) + elif loader == "neoforge": + env = {} + if len(java_args) > 0: + env["JDK_JAVA_OPTIONS"] = " ".join(java_args) + bash_file = "run.bat" if os.name == "nt" else "run.sh" + return subprocess.run([exec_dir / bash_file] + mc_args, env=env, **kwargs) + else: + raise RuntimeError(f"Unknown loader {loader}, can't run server") if __name__ == "__main__": main()