From 535614dcc863108fa5af1568e35d6284b8889bc2 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Fri, 5 Jul 2024 09:32:52 +0300 Subject: [PATCH 1/3] check sha1 for forge/neoforge maven Signed-off-by: Trial97 --- meta/common/__init__.py | 9 +++++++++ meta/run/update_forge.py | 20 +++++++++++++++++++- meta/run/update_neoforge.py | 20 +++++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index d7ee49a..16a4f50 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -86,3 +86,12 @@ def default_session(): sess.headers.update({"User-Agent": "PrismLauncherMeta/1.0"}) return sess + + +def remove_files(file_paths): + for file_path in file_paths: + try: + if os.path.isfile(file_path): + os.remove(file_path) + except Exception as e: + print(e) diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py index 8fc8920..5a86727 100755 --- a/meta/run/update_forge.py +++ b/meta/run/update_forge.py @@ -16,7 +16,12 @@ from pydantic import ValidationError -from meta.common import upstream_path, ensure_upstream_dir, default_session +from meta.common import ( + upstream_path, + ensure_upstream_dir, + default_session, + remove_files, +) from meta.common.forge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -292,6 +297,19 @@ def main(): UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version ) + if not os.path.isfile(jar_path): + remove_files([profile_path, installer_info_path]) + else: + fileSha1 = filehash(jar_path, hashlib.sha1) + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + if fileSha1 != rfile.text.strip(): + remove_files([jar_path, profile_path, installer_info_path]) + except Exception as e: + eprint("Failed to check sha1 %s" % version.url()) + eprint("Error is %s" % e) + installer_refresh_required = not os.path.isfile( profile_path ) or not os.path.isfile(installer_info_path) diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py index 60b94ec..c6eb0f3 100644 --- a/meta/run/update_neoforge.py +++ b/meta/run/update_neoforge.py @@ -17,7 +17,12 @@ from pydantic import ValidationError -from meta.common import upstream_path, ensure_upstream_dir, default_session +from meta.common import ( + upstream_path, + ensure_upstream_dir, + default_session, + remove_files, +) from meta.common.neoforge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -243,6 +248,19 @@ def main(): UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version ) + if not os.path.isfile(jar_path): + remove_files([profile_path, installer_info_path]) + else: + fileSha1 = filehash(jar_path, hashlib.sha1) + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + if fileSha1 != rfile.text.strip(): + remove_files([jar_path, profile_path, installer_info_path]) + except Exception as e: + eprint("Failed to check sha1 %s" % version.url()) + eprint("Error is %s" % e) + installer_refresh_required = not os.path.isfile( profile_path ) or not os.path.isfile(installer_info_path) From 09c5517983f36d1ad7a7c4c982649a30fa660a84 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Mon, 16 Dec 2024 21:00:29 +0200 Subject: [PATCH 2/3] Chache sha1 file Signed-off-by: Trial97 --- meta/common/__init__.py | 25 +++++++++++++++++ meta/run/generate_forge.py | 7 +---- meta/run/generate_neoforge.py | 7 +---- meta/run/update_forge.py | 51 +++++++++++++++++------------------ meta/run/update_neoforge.py | 45 ++++++++++++++++--------------- 5 files changed, 76 insertions(+), 59 deletions(-) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 16a4f50..8e42933 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -1,6 +1,8 @@ import os import os.path import datetime +import hashlib +import sys from urllib.parse import urlparse from typing import Any, Optional @@ -95,3 +97,26 @@ def remove_files(file_paths): os.remove(file_path) except Exception as e: print(e) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def filehash(filename, hashtype, blocksize=65536): + hashtype = hashtype() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(blocksize), b""): + hashtype.update(block) + return hashtype.hexdigest() + + +def get_file_sha1_from_file(file_name, sha1_file): + if os.path.isfile(sha1_file): + with open(sha1_file, "r") as file: + return file.read() + + new_sha1 = filehash(file_name, hashlib.sha1) + with open(sha1_file, "w") as file: + file.write(new_sha1) + return diff --git a/meta/run/generate_forge.py b/meta/run/generate_forge.py index 95b5ac3..a01f503 100755 --- a/meta/run/generate_forge.py +++ b/meta/run/generate_forge.py @@ -1,11 +1,10 @@ import os import re -import sys from packaging import version as pversion from operator import attrgetter from typing import Collection -from meta.common import ensure_component_dir, launcher_path, upstream_path +from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint from meta.common.forge import ( FORGE_COMPONENT, INSTALLER_MANIFEST_DIR, @@ -44,10 +43,6 @@ ensure_component_dir(FORGE_COMPONENT) -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - # Construct a set of libraries out of a Minecraft version file, for filtering. mc_version_cache = {} diff --git a/meta/run/generate_neoforge.py b/meta/run/generate_neoforge.py index ee1e269..ff58bed 100644 --- a/meta/run/generate_neoforge.py +++ b/meta/run/generate_neoforge.py @@ -1,11 +1,10 @@ from copy import deepcopy import os import re -import sys from operator import attrgetter from typing import Collection -from meta.common import ensure_component_dir, launcher_path, upstream_path +from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint from meta.common.neoforge import ( NEOFORGE_COMPONENT, INSTALLER_MANIFEST_DIR, @@ -38,10 +37,6 @@ ensure_component_dir(NEOFORGE_COMPONENT) -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - def version_from_build_system_installer( installer: MojangVersion, profile: NeoForgeInstallerProfileV2, diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py index 5a86727..30c701f 100755 --- a/meta/run/update_forge.py +++ b/meta/run/update_forge.py @@ -7,7 +7,6 @@ import json import os import re -import sys import zipfile from contextlib import suppress from datetime import datetime @@ -21,6 +20,9 @@ ensure_upstream_dir, default_session, remove_files, + eprint, + filehash, + get_file_sha1_from_file, ) from meta.common.forge import ( JARS_DIR, @@ -43,6 +45,7 @@ InstallerInfo, ForgeLegacyInfo, ) +from meta.common.http import download_binary_file from meta.model.mojang import MojangVersion UPSTREAM_DIR = upstream_path() @@ -58,18 +61,6 @@ sess = default_session() -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def filehash(filename, hashtype, blocksize=65536): - hashtype = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - hashtype.update(block) - return hashtype.hexdigest() - - def get_single_forge_files_manifest(longversion): print(f"Getting Forge manifest for {longversion}") path_thing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion @@ -297,15 +288,20 @@ def main(): UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version ) + new_sha1 = None + sha1_file = jar_path + ".sha1" if not os.path.isfile(jar_path): remove_files([profile_path, installer_info_path]) else: - fileSha1 = filehash(jar_path, hashlib.sha1) + fileSha1 = get_file_sha1_from_file(jar_path, sha1_file) try: rfile = sess.get(version.url() + ".sha1") rfile.raise_for_status() - if fileSha1 != rfile.text.strip(): - remove_files([jar_path, profile_path, installer_info_path]) + new_sha1 = rfile.text.strip() + if fileSha1 != new_sha1: + remove_files( + [jar_path, profile_path, installer_info_path, sha1_file] + ) except Exception as e: eprint("Failed to check sha1 %s" % version.url()) eprint("Error is %s" % e) @@ -318,11 +314,18 @@ def main(): # grab the installer if it's not there if not os.path.isfile(jar_path): eprint("Downloading %s" % version.url()) - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) + if new_sha1 is None: + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + new_sha1 = rfile.text.strip() + except Exception as e: + eprint("Failed to download new sha1 %s" % version.url()) + eprint("Error is %s" % e) + if new_sha1 is not None: # this is in case the fetch failed + with open(sha1_file, "w") as file: + file.write(new_sha1) eprint("Processing %s" % version.url()) # harvestables from the installer @@ -387,11 +390,7 @@ def main(): if not os.path.isfile(LEGACYINFO_PATH): # grab the jar/zip if it's not there if not os.path.isfile(jar_path): - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) # find the latest timestamp in the zip file tstamp = datetime.fromtimestamp(0) with zipfile.ZipFile(jar_path) as jar: diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py index c6eb0f3..f6a8960 100644 --- a/meta/run/update_neoforge.py +++ b/meta/run/update_neoforge.py @@ -7,7 +7,6 @@ import json import os import re -import sys import zipfile from contextlib import suppress from datetime import datetime @@ -22,7 +21,11 @@ ensure_upstream_dir, default_session, remove_files, + eprint, + filehash, + get_file_sha1_from_file, ) +from meta.common.http import download_binary_file from meta.common.neoforge import ( JARS_DIR, INSTALLER_INFO_DIR, @@ -52,18 +55,6 @@ sess = default_session() -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def filehash(filename, hashtype, blocksize=65536): - hashtype = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - hashtype.update(block) - return hashtype.hexdigest() - - def find_nth(haystack, needle, n): start = haystack.find(needle) while start >= 0 and n > 1: @@ -248,15 +239,20 @@ def main(): UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version ) + new_sha1 = None + sha1_file = jar_path + ".sha1" if not os.path.isfile(jar_path): remove_files([profile_path, installer_info_path]) else: - fileSha1 = filehash(jar_path, hashlib.sha1) + fileSha1 = get_file_sha1_from_file(jar_path, sha1_file) try: rfile = sess.get(version.url() + ".sha1") rfile.raise_for_status() - if fileSha1 != rfile.text.strip(): - remove_files([jar_path, profile_path, installer_info_path]) + new_sha1 = rfile.text.strip() + if fileSha1 != new_sha1: + remove_files( + [jar_path, profile_path, installer_info_path, sha1_file] + ) except Exception as e: eprint("Failed to check sha1 %s" % version.url()) eprint("Error is %s" % e) @@ -270,16 +266,23 @@ def main(): if not os.path.isfile(jar_path): eprint("Downloading %s" % version.url()) try: - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() Path(jar_path).parent.mkdir(parents=True, exist_ok=True) - with open(jar_path, "wb") as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) + download_binary_file(sess, jar_path, version.url()) except Exception as e: eprint("Failed to download %s" % version.url()) eprint("Error is %s" % e) continue + if new_sha1 is None: + try: + rfile = sess.get(version.url() + ".sha1") + rfile.raise_for_status() + new_sha1 = rfile.text.strip() + except Exception as e: + eprint("Failed to download new sha1 %s" % version.url()) + eprint("Error is %s" % e) + if new_sha1 is not None: # this is in case the fetch failed + with open(sha1_file, "w") as file: + file.write(new_sha1) eprint("Processing %s" % version.url()) # harvestables from the installer From 941ad43e8dc4cde97a0758df941397c7353eee3b Mon Sep 17 00:00:00 2001 From: Trial97 Date: Wed, 16 Apr 2025 16:19:17 +0300 Subject: [PATCH 3/3] chore:add type annotations Signed-off-by: Trial97 --- meta/common/__init__.py | 14 ++++++++------ meta/run/index.py | 15 ++++----------- meta/run/update_fabric.py | 8 -------- meta/run/update_forge.py | 10 +++++----- meta/run/update_neoforge.py | 6 +++--- meta/run/update_quilt.py | 8 -------- 6 files changed, 20 insertions(+), 41 deletions(-) diff --git a/meta/common/__init__.py b/meta/common/__init__.py index 8e42933..1f358cd 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -4,7 +4,7 @@ import hashlib import sys from urllib.parse import urlparse -from typing import Any, Optional +from typing import Any, Optional, Callable import requests from cachecontrol import CacheControl # type: ignore @@ -90,7 +90,7 @@ def default_session(): return sess -def remove_files(file_paths): +def remove_files(file_paths: list[str]) -> None: for file_path in file_paths: try: if os.path.isfile(file_path): @@ -103,7 +103,9 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def filehash(filename, hashtype, blocksize=65536): +def file_hash( + filename: str, hashtype: Callable[[], "hashlib._Hash"], blocksize: int = 65536 +) -> str: hashtype = hashtype() with open(filename, "rb") as f: for block in iter(lambda: f.read(blocksize), b""): @@ -111,12 +113,12 @@ def filehash(filename, hashtype, blocksize=65536): return hashtype.hexdigest() -def get_file_sha1_from_file(file_name, sha1_file): +def get_file_sha1_from_file(file_name: str, sha1_file: str) -> Optional[str]: if os.path.isfile(sha1_file): with open(sha1_file, "r") as file: return file.read() - new_sha1 = filehash(file_name, hashlib.sha1) + new_sha1 = file_hash(file_name, hashlib.sha1) with open(sha1_file, "w") as file: file.write(new_sha1) - return + return None diff --git a/meta/run/index.py b/meta/run/index.py index 23dc233..93997de 100755 --- a/meta/run/index.py +++ b/meta/run/index.py @@ -2,7 +2,9 @@ import os from operator import attrgetter -from meta.common import launcher_path +from meta.common import launcher_path, file_hash + + from meta.model import MetaVersion, MetaPackage from meta.model.index import ( MetaPackageIndex, @@ -14,15 +16,6 @@ LAUNCHER_DIR = launcher_path() -# take the hash type (like hashlib.md5) and filename, return hex string of hash -def hash_file(hash_fn, file_name): - hash_instance = hash_fn() - with open(file_name, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_instance.update(chunk) - return hash_instance.hexdigest() - - # ignore these files when indexing versions ignore = {"index.json", "package.json", ".git", ".github"} @@ -50,7 +43,7 @@ def hash_file(hash_fn, file_name): continue # parse and hash the version file filepath = LAUNCHER_DIR + "/%s/%s" % (package, filename) - filehash = hash_file(hashlib.sha256, filepath) + filehash = file_hash(filepath, hashlib.sha256) versionFile = MetaVersion.parse_file(filepath) is_recommended = versionFile.version in recommendedVersions diff --git a/meta/run/update_fabric.py b/meta/run/update_fabric.py index 850ec8e..b8f57c2 100755 --- a/meta/run/update_fabric.py +++ b/meta/run/update_fabric.py @@ -29,14 +29,6 @@ sess = default_session() -def filehash(filename, hashtype, blocksize=65536): - h = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - h.update(block) - return h.hexdigest() - - def get_maven_url(maven_key, server, ext): parts = maven_key.split(":", 3) maven_ver_url = ( diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py index 30c701f..7ec9e45 100755 --- a/meta/run/update_forge.py +++ b/meta/run/update_forge.py @@ -21,7 +21,7 @@ default_session, remove_files, eprint, - filehash, + file_hash, get_file_sha1_from_file, ) from meta.common.forge import ( @@ -376,8 +376,8 @@ def main(): # installer info v1 if not os.path.isfile(installer_info_path): installer_info = InstallerInfo() - installer_info.sha1hash = filehash(jar_path, hashlib.sha1) - installer_info.sha256hash = filehash(jar_path, hashlib.sha256) + installer_info.sha1hash = file_hash(jar_path, hashlib.sha1) + installer_info.sha256hash = file_hash(jar_path, hashlib.sha256) installer_info.size = os.path.getsize(jar_path) installer_info.write(installer_info_path) else: @@ -400,8 +400,8 @@ def main(): tstamp = tstamp_new legacy_info = ForgeLegacyInfo() legacy_info.release_time = tstamp - legacy_info.sha1 = filehash(jar_path, hashlib.sha1) - legacy_info.sha256 = filehash(jar_path, hashlib.sha256) + legacy_info.sha1 = file_hash(jar_path, hashlib.sha1) + legacy_info.sha256 = file_hash(jar_path, hashlib.sha256) legacy_info.size = os.path.getsize(jar_path) legacy_info_list.number[key] = legacy_info diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py index f6a8960..9ecb734 100644 --- a/meta/run/update_neoforge.py +++ b/meta/run/update_neoforge.py @@ -22,7 +22,7 @@ default_session, remove_files, eprint, - filehash, + file_hash, get_file_sha1_from_file, ) from meta.common.http import download_binary_file @@ -332,8 +332,8 @@ def main(): # installer info v1 if not os.path.isfile(installer_info_path): installer_info = InstallerInfo() - installer_info.sha1hash = filehash(jar_path, hashlib.sha1) - installer_info.sha256hash = filehash(jar_path, hashlib.sha256) + installer_info.sha1hash = file_hash(jar_path, hashlib.sha1) + installer_info.sha256hash = file_hash(jar_path, hashlib.sha256) installer_info.size = os.path.getsize(jar_path) installer_info.write(installer_info_path) diff --git a/meta/run/update_quilt.py b/meta/run/update_quilt.py index 5f3c40c..2f31602 100755 --- a/meta/run/update_quilt.py +++ b/meta/run/update_quilt.py @@ -22,14 +22,6 @@ sess = default_session() -def filehash(filename, hashtype, blocksize=65536): - h = hashtype() - with open(filename, "rb") as f: - for block in iter(lambda: f.read(blocksize), b""): - h.update(block) - return h.hexdigest() - - def get_maven_url(maven_key, server, ext): parts = maven_key.split(":", 3) maven_ver_url = (