Use gradle module metadata when appropriate

Author: raphiz

buildGradleApplication/mkM2Repository.nix

@@ -23,15 +23,38 @@
       "python ${./parse.py} ${filteredSrc}/${verificationFile} ${builtins.toString (builtins.map lib.escapeShellArg repositories)}> $out"
     ))
   );
-  mkDep = depSpec: {
-    inherit (depSpec) url_prefixes path name hash component;
+  mkDep = depSpec: let
+    # Gradle module metadata may specify a different url suffix than the artifact name
+    urlSuffixFallback = depSpec.name;
+    urlSuffix =
+      if (depSpec.module == null)
+      then urlSuffixFallback
+      else let
+        module = fetchArtifact {
+          urls = lib.map (prefix: "${prefix}/${depSpec.module.name}") depSpec.url_prefixes;
+          inherit (depSpec.module) hash name;
+        };
+        moduleMetadata = builtins.fromJSON (builtins.readFile module);
+        variantFiles = lib.flatten (lib.map (variant: variant.files or []) (moduleMetadata.variants or []));
+        matchingVariantFiles =
+          builtins.filter (
+            file: (file.${depSpec.hash_algo} or null) == depSpec.hash_value && file.name == depSpec.name
+          )
+          variantFiles;
+        firstMatchingVariantFile = builtins.head matchingVariantFiles;
+      in
+        if matchingVariantFiles == []
+        then urlSuffixFallback
+        else if builtins.all (f: f.url == firstMatchingVariantFile.url) matchingVariantFiles
+        then firstMatchingVariantFile.url
+        else throw "Found multiple matching urls with name ${depSpec.name} and hash ${depSpec.hash} in ${module}: ${builtins.toString (lib.map (f: f.url) matchingVariantFiles)}";
+    urls = lib.map (prefix: "${prefix}/${urlSuffix}") depSpec.url_prefixes;
+  in {
+    inherit urls urlSuffix;
+    inherit (depSpec) path name hash component;
     jar = fetchArtifact {
-      inherit (depSpec) url_prefixes hash name hash_algo hash_value path;
-      module = if (depSpec.module_name == null) then null else fetchArtifact {
-        inherit (depSpec) url_prefixes hash_algo hash_value path;
-        hash = depSpec.module_hash;
-        name = depSpec.module_name;
-      };
+      inherit urls;
+      inherit (depSpec) hash name;
     };
   };
   dependencies = builtins.map (depSpec: mkDep depSpec) depSpecs;
@@ -42,6 +65,6 @@
     {src = filteredSrc;}
     (
       "mkdir $out"
-      + lib.concatMapStringsSep "\n" (dep: "mkdir -p $out/${dep.path}\nln -s ${builtins.toString dep.jar} $out/${dep.path}/${dep.name}") dependencies
+      + lib.concatMapStringsSep "\n" (dep: "mkdir -p $out/${dep.path}\nln -s ${builtins.toString dep.jar} $out/${dep.path}/${dep.urlSuffix}") dependencies
     );
 in {inherit dependencies m2Repository;}

buildGradleApplication/parse.py

@@ -15,8 +15,13 @@ class Artifact:
     name: str
     hash: object
     component: object
-    module_name: str
-    module_hash: str
+    module: object
+
+@dataclass
+class Module:
+    name: str
+    hash: object
+
 
 @dataclass
 class Hash:
@@ -37,8 +42,12 @@ def main():
             "url_prefixes": [f"{maven_repo}/{path}" for maven_repo in maven_repos],
             "path": path,
             "name": artifact.name,
-            "module_name": artifact.module_name,
-            "module_hash": artifact.module_hash,
+            "module": {
+                "name": artifact.module.name,
+                "hash": toSri(artifact.module.hash.algo, artifact.module.hash.value),
+                "hash_algo": artifact.module.hash.algo,
+                "hash_value": artifact.module.hash.value,
+            } if artifact.module is not None else None,
             "component": {
                 "group": artifact.component.group,
                 "name": artifact.component.name,
@@ -71,9 +80,8 @@ def parse(xml_file):
         name = component_elem.get("name")
         version = component_elem.get("version")
         component_obj = Component(group=group, name=name, version=version)
-        module_name = None
-        module_hash = None
 
+        component_artifacts = []
         for artifact_elem in component_elem.findall("default:artifact", namespaces):
             artifact_name = artifact_elem.get("name")
             hash_obj=None
@@ -83,13 +91,22 @@ def parse(xml_file):
                     value = elem.get("value")
                     hash_obj = Hash(algo=algo, value=value)
 
-            artifact_obj = Artifact(name=artifact_name, hash=hash_obj, component=component_obj, module_name=module_name, module_hash=module_hash)
-            artifacts.append(artifact_obj)
-
-            if artifact_name.endswith(".module"):
-                module_name = artifact_name
-                module_hash = toSri(hash_obj.algo, hash_obj.value)
-
+            artifact_obj = Artifact(name=artifact_name, hash=hash_obj, component=component_obj, module=None)
+            component_artifacts.append(artifact_obj)
+
+        # keep reference to Gradle module metadata if it exist
+        module_name = f"{name}-{version}.module"
+        module_artifact = next(
+            (artifact for artifact in component_artifacts if artifact.name == module_name),
+            None,
+        )
+        if module_artifact is not None:
+            module = Module(name=module_artifact.name, hash=module_artifact.hash)
+            for artifact in component_artifacts:
+                if artifact is not module_artifact:
+                    artifact.module = module
+
+        artifacts.extend(component_artifacts)
     return artifacts
 
 

fetchArtefact/builder.bash

@@ -29,16 +29,10 @@ check_hash() {
 # expected variables to be set:
 name="${name:?}"
 out="${out:?}"
-url_prefixes="${url_prefixes:?}"
+urls="${urls:?}"
 hash="${hash:?}"
 
-url_suffix=$name
-if [[ -n $module ]]; then
-  url_suffix="$(< $module $jq -r '.variants | map(.files) | flatten | map(select(.'$hash_algo' == "'$hash_value'")) | (. + [{ url: "'$name'"}])[0].url')"
-fi
-
-for url_prefix in $url_prefixes; do
-    url="$url_prefix/$url_suffix"
+for url in $urls; do
     echo "Downloading $name from $url"
 
     if "${curl[@]}" --retry 0 --connect-timeout "${NIX_CONNECT_TIMEOUT:-15}" \

fetchArtefact/default.nix

@@ -3,22 +3,13 @@
   curl,
   nix,
   cacert,
-  jq,
-  fetchurl
 }: {
   # A list of URLs specifying alternative download locations. They are tried in order.
-  url_prefixes,
+  urls,
   # SRI hash.
   hash,
   # Name of the file.
   name,
-  # path
-  path,
-  # hash in undecoded form
-  hash_algo,
-  hash_value,
-  # module json file that can contain content with url/name mappings
-  module ? null,
 }:
 stdenvNoCC.mkDerivation {
   inherit name hash;
@@ -28,8 +19,7 @@ stdenvNoCC.mkDerivation {
   builder = ./builder.bash;
   nativeBuildInputs = [curl nix];
   SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
-  inherit url_prefixes module hash_algo hash_value;
-  jq = "${jq}/bin/jq";
+  inherit urls;
 
   # Doing the download on a remote machine just duplicates network
   # traffic, so don't do that