New helper function to help with file downloads

docs/usage.md

@@ -542,3 +542,37 @@ then {
   assert snapshot(sanitizeOutput(process.out, unstableKeys:["zip"])).match()
 }
 ```
+
+### `curlAndExtract()` - Download and extract an archive
+
+The `curlAndExtract()` function is used to download an archive
+from the Internet with `curl` and extract it in the required destination
+directory.
+Zip and Tar archives are currently supported. Tar archives can be compressed
+with any of these algorithms: gzip, gz, bzip2, bz2, xz, lz4, lzma, lzop, zstd.
+By default, the choice of archive format and compression algorithm is based on
+the name of the archive, but it can also be passed as an argument.
+For compressed Tar archives, the format to provide is "tar.bz2" or "tbz2"
+(adapting to your compression algorithm of choice).
+
+You are responsible for deleting the data in the `cleanup` phase.
+
+```groovy
+setup {
+    curlAndExtract("https://www.example.com/pretty_database.zip", "${launchDir}/data_dir")
+    curlAndExtract("https://www.example.com/beautiful_database.tar.gz", "${launchDir}/data_dir")
+    curlAndExtract("https://www.example.com/secret/data", "${launchDir}/data_dir", "tar.bz2")
+}
+
+when {
+  params {
+    pretty_db_path = "${launchDir}/data_dir/pretty_db"
+    beautiful_db_path = "${launchDir}/data_dir/db/beauty_db"
+    secret_db_path = "${launchDir}/data_dir/db/secret"
+  }
+}
+
+cleanup {
+    new File("${launchDir}/data_dir").deleteDir()
+}
+```

src/main/java/nf_core/nf/test/utils/Methods.java

@@ -3,11 +3,8 @@
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
-import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileSystems;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -19,12 +16,6 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 import org.yaml.snakeyaml.Yaml;
 
@@ -752,4 +743,203 @@ public static TreeMap<String,Object> sanitizeOutput(TreeMap<String,Object> chann
   public static TreeMap<String,Object> sanitizeOutput(HashMap<String,Object> options, TreeMap<String,Object> channel) {
     return OutputSanitizer.sanitizeOutput(options, channel);
   }
+
+  /**
+   * Download a tar archive and extract it in the given destination directory.
+   * The file is streamed directly with `curl` into `tar` via a pipe.
+   * The compression type must be provided if applicable.
+   * Uses safe single-quoting for the URL and destination path.
+   *
+   * @param urlString the URL to fetch
+   * @param destPath directory to extract the tarball into
+   * @param compression compression type: tar, gzip, gz, bzip2, bz2, xz, lz4, lzma, lzop, zstd
+   *        or any of these prefixed with "tar." or "t"
+   * @throws IOException on failure
+   */
+  private static void curlAndUntar(String urlString, String destPath, String compression) throws IOException {
+    Path destDir = Paths.get(destPath);
+    Files.createDirectories(destDir);
+
+    String escUrl = Utils.shellEscape(urlString);
+    String escDest = Utils.shellEscape(destPath);
+    String cmd = "curl -L --retry 5 " + escUrl + " | tar xaf - -C " + escDest;
+
+    // Convert compression name to tar option
+    if (compression != null && !compression.equals("tar")) {
+      // Remove leading "tar." or "t" if present
+      if (compression.startsWith("tar.")) {
+        compression = compression.substring(4);
+      } else if (compression.startsWith("t")) {
+        compression = compression.substring(1);
+      }
+
+      if (compression.equals("gzip") || compression.equals("gz")) {
+        compression = "gzip";
+      } else if (compression.equals("bzip2") || compression.equals("bz2")) {
+        compression = "bzip2";
+      } else if (compression.equals("xz")) {
+        compression = "xz";
+      } else if (compression.equals("lz4")) {
+        compression = "lz4";
+      } else if (compression.equals("lzma")) {
+        compression = "lzma";
+      } else if (compression.equals("lzop")) {
+        compression = "lzop";
+      } else if (compression.equals("zstd") || compression.equals("zst")) {
+        compression = "zstd";
+      } else {
+        throw new IllegalArgumentException("Unsupported compression type: " + compression);
+      }
+      cmd += " --" + compression;
+    }
+
+    ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd);
+    try {
+      Utils.ProcessResult result = Utils.runProcess(pb);
+      if (result.exitCode != 0) {
+        System.err.println("Error downloading and extracting file " + urlString + ": exit code " + result.exitCode + "\n");
+        System.out.println("Bash command: \n" + cmd);
+        System.err.println("command output: \n");
+        System.err.println(result.stderr);
+      } else {
+        System.out.println("Successfully downloaded and extracted file: " + urlString);
+      }
+    } catch (IOException | InterruptedException e) {
+      System.err.println("Error downloading and extracting file " + urlString + ": " + e.getMessage());
+      if (e instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
+  /**
+   * Download a zip file and extract it in the given destination directory.
+   * The file is first downloaded with `curl` to a temporary file, then we
+   * call `unzip` and delete the temporary file.
+   *
+   * @param urlString the URL to fetch
+   * @param destPath directory to extract the zip into
+   * @throws IOException on failure
+   */
+  private static void curlAndUnzip(String urlString, String destPath) throws IOException {
+    Path destDir = Paths.get(destPath);
+    Files.createDirectories(destDir);
+
+    // Create a temporary file in the destination directory for the downloaded zip
+    Path tempFile = Files.createTempFile(destDir, "download", ".zip");
+
+    // Run curl
+    ProcessBuilder pb = new ProcessBuilder(
+        "curl",
+        "-L",
+        "--retry",
+        "5",
+        "-o",
+        tempFile.toString(),
+        urlString
+    );
+
+    try {
+      Utils.ProcessResult result = Utils.runProcess(pb);
+      if (result.exitCode != 0) {
+        System.err.println("Error downloading file " + urlString + ": exit code " + result.exitCode + "\n");
+        System.out.println("Command: " + String.join(" ", pb.command()));
+        System.err.println("command output: \n");
+        System.err.println(result.stderr);
+        return;
+      }
+      // Run unzip
+      pb = new ProcessBuilder(
+          "unzip",
+          "-o",
+          tempFile.toString(),
+          "-d",
+          destPath
+      );
+      result = Utils.runProcess(pb);
+      if (result.exitCode != 0) {
+        System.err.println("Error extracting zip " + tempFile + ": exit code " + result.exitCode + "\n");
+        System.out.println("Command: " + String.join(" ", pb.command()));
+        System.err.println("command output: \n");
+        System.err.println(result.stderr);
+      } else {
+        System.out.println("Successfully downloaded and extracted file: " + urlString);
+      }
+    } catch (IOException | InterruptedException e) {
+      System.err.println("Error downloading and extracting file " + urlString + ": " + e.getMessage());
+      if (e instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
+    } finally {
+      try {
+        Files.deleteIfExists(tempFile);
+      } catch (IOException e) {
+        // Do not fail the operation if temp file cleanup fails; just log it
+        System.err.println("Warning: failed to delete temporary file " + tempFile + ": " + e.getMessage());
+      }
+    }
+  }
+
+  /**
+   * Download an archive and extract it in the given destination directory.
+   * Dispatches to `curlAndUnzip` for ZIP files and to `curlAndUntar` for
+   * tar archives based on the URL's file extension.
+   *
+   * @param urlString the URL to fetch
+   * @param destPath directory to extract the archive into
+   * @throws IOException on failure or if archive type is unsupported
+   */
+  public static void curlAndExtract(String urlString, String destPath) throws IOException {
+    String lower = Utils.getURLFileName(urlString);
+
+    if (lower.endsWith(".zip")) {
+      curlAndUnzip(urlString, destPath);
+      return;
+    }
+
+    for (String suffix: new String [] {"gz", "bz2", "xz", "lz4", "lzma", "lzop", "zst", "zstd"}) {
+      if (lower.endsWith(".tar." + suffix) || lower.endsWith(".t" + suffix)) {
+        curlAndUntar(urlString, destPath, suffix);
+        return;
+      }
+    }
+
+    if (lower.endsWith(".tar")) {
+      curlAndUntar(urlString, destPath, null);
+      return;
+    }
+
+    throw new IllegalArgumentException("Unsupported archive type in URL: " + urlString);
+  }
+
+  /**
+   * Download an archive and extract it in the given destination directory.
+   * Dispatches to `curlAndUnzip` for ZIP files and to `curlAndUntar` for
+   * tar archives based on the `compression` parameter.
+   *
+   * @param urlString the URL to fetch
+   * @param destPath directory to extract the archive into
+   * @param compression compression type: zip, tar, or any of the following prefixed with "tar." or "t":
+   *                    gzip, gz, bzip2, bz2, xz, lz4, lzma, lzop, zstd
+   * @throws IOException on failure or if archive type is unsupported
+   */
+  public static void curlAndExtract(String urlString, String destPath, String compression) throws IOException {
+    if (compression == null || compression.isEmpty()) {
+      throw new IllegalArgumentException("The 'compression' parameter is required.");
+    }
+    String lower = compression.toLowerCase(Locale.ROOT);
+    // Zip is the only clearly defined archive format.
+    // Everything else is assumed to be Tar
+    if (lower.equals("zip")) {
+      curlAndUnzip(urlString, destPath);
+    } else if (lower.equals("tar")) {
+      curlAndUntar(urlString, destPath, null);
+    } else if (lower.startsWith("tar.")) {
+      curlAndUntar(urlString, destPath, compression);
+    } else if (lower.startsWith("t")) {
+      curlAndUntar(urlString, destPath, compression);
+    } else {
+      throw new IllegalArgumentException("Unsupported compression type: " + compression);
+    }
+  }
 }

src/main/java/nf_core/nf/test/utils/NfCoreUtils.java

@@ -1,10 +1,8 @@
 package nf_core.nf.test.utils;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.file.Files;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -99,23 +97,14 @@ private static void installModule(String libDir, String name, String sha, String
       }
 
       ProcessBuilder processBuilder = new ProcessBuilder("bash", "-c", command.toString());
-      Process process = processBuilder.start();
-
-      // Capture stderr from nf-core tools
-      BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
-      StringBuilder stderr = new StringBuilder();
-      String line;
-      while ((line = stderrReader.readLine()) != null) {
-        stderr.append(line).append("\n");
-      }
-      int exitCode = process.waitFor();
+      Utils.ProcessResult result = Utils.runProcess(processBuilder);
 
       // Spit out nf-core tools stderr if install fails
-      if (exitCode != 0) {
-        System.err.println("Error installing module " + name + ": exit code " + exitCode + "\n");
+      if (result.exitCode != 0) {
+        System.err.println("Error installing module " + name + ": exit code " + result.exitCode + "\n");
         System.out.println("Installation command: \n" + command.toString());
         System.err.println("nf-core tools output: \n");
-        System.err.println(stderr.toString());
+        System.err.println(result.stderr);
       } else {
         System.out.println("Successfully installed module: " + name);
       }

src/main/java/nf_core/nf/test/utils/Utils.java

@@ -0,0 +1,69 @@
+package nf_core.nf.test.utils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Locale;
+
+public class Utils {
+
+  /**
+   * Result of running a process started from a {@link ProcessBuilder}.
+   */
+  public static class ProcessResult {
+    public final int exitCode;
+    public final String stderr;
+
+    public ProcessResult(int exitCode, String stderr) {
+      this.exitCode = exitCode;
+      this.stderr = stderr;
+    }
+  }
+
+  /**
+   * Starts the given {@link ProcessBuilder}, captures stderr, waits for exit,
+   * and returns a {@link ProcessResult}.
+   */
+  public static ProcessResult runProcess(ProcessBuilder pb) throws IOException, InterruptedException {
+    pb.redirectOutput(ProcessBuilder.Redirect.DISCARD);
+    Process process = pb.start();
+    BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
+    StringBuilder stderr = new StringBuilder();
+    String line;
+    while ((line = stderrReader.readLine()) != null) {
+      stderr.append(line).append("\n");
+    }
+    int exitCode = process.waitFor();
+    return new ProcessResult(exitCode, stderr.toString());
+  }
+
+  /**
+   * Shell escape a string by wrapping in single quotes and escaping existing single quotes.
+   * @param s
+   * @return The shell-escaped string
+   */
+  public static String shellEscape(String s) {
+    if (s == null) return "''";
+    return "'" + s.replace("'", "'" + "\"'\"" + "'") + "'";
+  }
+
+  /**
+   * Extract a lower-cased file name portion from a URL string for extension checking.
+   * @param urlString The URL string
+   * @return The lower-cased file name portion of the URL
+   */
+  public static String getURLFileName(String urlString) {
+    // Try to extract a path portion from the URL (strip query strings)
+    String pathPart = urlString;
+    try {
+      java.net.URI uri = new java.net.URI(urlString);
+      if (uri.getPath() != null && !uri.getPath().isEmpty()) {
+        pathPart = uri.getPath();
+      }
+    } catch (Exception e) {
+      // If parsing fails, fall back to raw urlString
+      pathPart = urlString;
+    }
+    return pathPart.toLowerCase(Locale.ROOT);
+  }
+}

tests/curlAndExtract/main.nf

@@ -0,0 +1,9 @@
+process TEST_MODULE {
+    output:
+    path ("test.txt"), emit: output
+
+    script:
+    """
+    echo "hello!" > test.txt
+    """
+}