Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache sha1 file #76

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions meta/common/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import os
import os.path
import datetime
import hashlib
import sys
from urllib.parse import urlparse
from typing import Any, Optional

Expand Down Expand Up @@ -86,3 +88,35 @@ def default_session():
sess.headers.update({"User-Agent": "PrismLauncherMeta/1.0"})

return sess


def remove_files(file_paths):
for file_path in file_paths:
try:
if os.path.isfile(file_path):
os.remove(file_path)
except Exception as e:
print(e)


def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)


def filehash(filename, hashtype, blocksize=65536):
hashtype = hashtype()
with open(filename, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
hashtype.update(block)
return hashtype.hexdigest()


def get_file_sha1_from_file(file_name, sha1_file):
if os.path.isfile(sha1_file):
with open(sha1_file, "r") as file:
return file.read()

new_sha1 = filehash(file_name, hashlib.sha1)
with open(sha1_file, "w") as file:
file.write(new_sha1)
return
7 changes: 1 addition & 6 deletions meta/run/generate_forge.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import os
import re
import sys
from packaging import version as pversion
from operator import attrgetter
from typing import Collection

from meta.common import ensure_component_dir, launcher_path, upstream_path
from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
from meta.common.forge import (
FORGE_COMPONENT,
INSTALLER_MANIFEST_DIR,
Expand Down Expand Up @@ -44,10 +43,6 @@
ensure_component_dir(FORGE_COMPONENT)


def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)


# Construct a set of libraries out of a Minecraft version file, for filtering.
mc_version_cache = {}

Expand Down
7 changes: 1 addition & 6 deletions meta/run/generate_neoforge.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
from copy import deepcopy
import os
import re
import sys
from operator import attrgetter
from typing import Collection

from meta.common import ensure_component_dir, launcher_path, upstream_path
from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
from meta.common.neoforge import (
NEOFORGE_COMPONENT,
INSTALLER_MANIFEST_DIR,
Expand Down Expand Up @@ -38,10 +37,6 @@
ensure_component_dir(NEOFORGE_COMPONENT)


def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)


def version_from_build_system_installer(
installer: MojangVersion,
profile: NeoForgeInstallerProfileV2,
Expand Down
65 changes: 41 additions & 24 deletions meta/run/update_forge.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import json
import os
import re
import sys
import zipfile
from contextlib import suppress
from datetime import datetime
Expand All @@ -16,7 +15,15 @@

from pydantic import ValidationError

from meta.common import upstream_path, ensure_upstream_dir, default_session
from meta.common import (
upstream_path,
ensure_upstream_dir,
default_session,
remove_files,
eprint,
filehash,
get_file_sha1_from_file,
)
from meta.common.forge import (
JARS_DIR,
INSTALLER_INFO_DIR,
Expand All @@ -38,6 +45,7 @@
InstallerInfo,
ForgeLegacyInfo,
)
from meta.common.http import download_binary_file
from meta.model.mojang import MojangVersion

UPSTREAM_DIR = upstream_path()
Expand All @@ -53,18 +61,6 @@
sess = default_session()


def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)


def filehash(filename, hashtype, blocksize=65536):
hashtype = hashtype()
with open(filename, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
hashtype.update(block)
return hashtype.hexdigest()


def get_single_forge_files_manifest(longversion):
print(f"Getting Forge manifest for {longversion}")
path_thing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion
Expand Down Expand Up @@ -292,6 +288,24 @@ def main():
UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version
)

new_sha1 = None
sha1_file = jar_path + ".sha1"
if not os.path.isfile(jar_path):
remove_files([profile_path, installer_info_path])
else:
fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
try:
rfile = sess.get(version.url() + ".sha1")
rfile.raise_for_status()
new_sha1 = rfile.text.strip()
if fileSha1 != new_sha1:
remove_files(
[jar_path, profile_path, installer_info_path, sha1_file]
)
except Exception as e:
eprint("Failed to check sha1 %s" % version.url())
eprint("Error is %s" % e)

installer_refresh_required = not os.path.isfile(
profile_path
) or not os.path.isfile(installer_info_path)
Expand All @@ -300,11 +314,18 @@ def main():
# grab the installer if it's not there
if not os.path.isfile(jar_path):
eprint("Downloading %s" % version.url())
rfile = sess.get(version.url(), stream=True)
rfile.raise_for_status()
with open(jar_path, "wb") as f:
for chunk in rfile.iter_content(chunk_size=128):
f.write(chunk)
download_binary_file(sess, jar_path, version.url())
if new_sha1 is None:
try:
rfile = sess.get(version.url() + ".sha1")
rfile.raise_for_status()
new_sha1 = rfile.text.strip()
except Exception as e:
eprint("Failed to download new sha1 %s" % version.url())
eprint("Error is %s" % e)
if new_sha1 is not None: # this is in case the fetch failed
with open(sha1_file, "w") as file:
file.write(new_sha1)

eprint("Processing %s" % version.url())
# harvestables from the installer
Expand Down Expand Up @@ -369,11 +390,7 @@ def main():
if not os.path.isfile(LEGACYINFO_PATH):
# grab the jar/zip if it's not there
if not os.path.isfile(jar_path):
rfile = sess.get(version.url(), stream=True)
rfile.raise_for_status()
with open(jar_path, "wb") as f:
for chunk in rfile.iter_content(chunk_size=128):
f.write(chunk)
download_binary_file(sess, jar_path, version.url())
# find the latest timestamp in the zip file
tstamp = datetime.fromtimestamp(0)
with zipfile.ZipFile(jar_path) as jar:
Expand Down
59 changes: 40 additions & 19 deletions meta/run/update_neoforge.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import json
import os
import re
import sys
import zipfile
from contextlib import suppress
from datetime import datetime
Expand All @@ -17,7 +16,16 @@

from pydantic import ValidationError

from meta.common import upstream_path, ensure_upstream_dir, default_session
from meta.common import (
upstream_path,
ensure_upstream_dir,
default_session,
remove_files,
eprint,
filehash,
get_file_sha1_from_file,
)
from meta.common.http import download_binary_file
from meta.common.neoforge import (
JARS_DIR,
INSTALLER_INFO_DIR,
Expand Down Expand Up @@ -47,18 +55,6 @@
sess = default_session()


def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)


def filehash(filename, hashtype, blocksize=65536):
hashtype = hashtype()
with open(filename, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
hashtype.update(block)
return hashtype.hexdigest()


def find_nth(haystack, needle, n):
start = haystack.find(needle)
while start >= 0 and n > 1:
Expand Down Expand Up @@ -243,6 +239,24 @@ def main():
UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
)

new_sha1 = None
sha1_file = jar_path + ".sha1"
if not os.path.isfile(jar_path):
remove_files([profile_path, installer_info_path])
else:
fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
try:
rfile = sess.get(version.url() + ".sha1")
rfile.raise_for_status()
new_sha1 = rfile.text.strip()
if fileSha1 != new_sha1:
remove_files(
[jar_path, profile_path, installer_info_path, sha1_file]
)
except Exception as e:
eprint("Failed to check sha1 %s" % version.url())
eprint("Error is %s" % e)

installer_refresh_required = not os.path.isfile(
profile_path
) or not os.path.isfile(installer_info_path)
Expand All @@ -252,16 +266,23 @@ def main():
if not os.path.isfile(jar_path):
eprint("Downloading %s" % version.url())
try:
rfile = sess.get(version.url(), stream=True)
rfile.raise_for_status()
Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
with open(jar_path, "wb") as f:
for chunk in rfile.iter_content(chunk_size=128):
f.write(chunk)
download_binary_file(sess, jar_path, version.url())
except Exception as e:
eprint("Failed to download %s" % version.url())
eprint("Error is %s" % e)
continue
if new_sha1 is None:
try:
rfile = sess.get(version.url() + ".sha1")
rfile.raise_for_status()
new_sha1 = rfile.text.strip()
except Exception as e:
eprint("Failed to download new sha1 %s" % version.url())
eprint("Error is %s" % e)
if new_sha1 is not None: # this is in case the fetch failed
with open(sha1_file, "w") as file:
file.write(new_sha1)

eprint("Processing %s" % version.url())
# harvestables from the installer
Expand Down
2 changes: 2 additions & 0 deletions update.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ function launcher_git() {
currentDate=$(date -I)

upstream_git reset --hard HEAD || exit 1
upstream_git pull

python -m meta.run.update_mojang || fail_in
python -m meta.run.update_forge || fail_in
Expand All @@ -57,6 +58,7 @@ if [ "${DEPLOY_TO_GIT}" = true ]; then
fi

launcher_git reset --hard HEAD || exit 1
launcher_git pull

python -m meta.run.generate_mojang || fail_out
python -m meta.run.generate_forge || fail_out
Expand Down