From fc8903cb195ec80594369e8087f023070c746fa6 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Wed, 30 Apr 2025 11:52:58 +0100 Subject: [PATCH 01/17] This commit adds support in rose for zstd and xz (with default options) using the same syntax as for gzip, including for compressing tarballs of various flavours. It adds tests for zstd (46) and xz (47) that are clones of the gzip rose_arch test (32) and a slightly beefier test for zstd (48) that is a clone of (07). --- .../rose_arch_compressions/rose_arch_tar.py | 26 ++- .../rose_arch_compressions/rose_arch_xz.py | 49 ++++ .../rose_arch_compressions/rose_arch_zstd.py | 49 ++++ t/rose-task-run/46-app-arch-zstd.t | 58 +++++ .../app/archive/rose-app.conf | 16 ++ t/rose-task-run/46-app-arch-zstd/flow.cylc | 21 ++ .../46-app-arch-zstd/rose-suite.conf | 0 t/rose-task-run/47-app-arch-xz.t | 58 +++++ .../47-app-arch-xz/app/archive/rose-app.conf | 16 ++ t/rose-task-run/47-app-arch-xz/flow.cylc | 21 ++ .../47-app-arch-xz/rose-suite.conf | 0 .../48-app-arch-20130101T0000Z.out | 83 +++++++ .../48-app-arch-20130101T1200Z.out | 83 +++++++ .../48-app-arch-20130102T0000Z.out | 83 +++++++ t/rose-task-run/48-app-arch-bad-9.out | 29 +++ .../48-app-arch-db-20130101T0000Z-1.out | 66 ++++++ .../48-app-arch-db-20130101T0000Z-2.out | 66 ++++++ .../48-app-arch-db-20130101T1200Z-1.out | 66 ++++++ .../48-app-arch-db-20130101T1200Z-2.out | 66 ++++++ .../48-app-arch-db-20130102T0000Z-1.out | 66 ++++++ .../48-app-arch-db-20130102T0000Z-2.out | 66 ++++++ t/rose-task-run/48-app-arch-find-foo.out | 61 +++++ t/rose-task-run/48-app-arch-planet-n.out | 9 + t/rose-task-run/48-app-arch-unknown-stuff.out | 9 + t/rose-task-run/48-app-arch.t | 214 ++++++++++++++++++ .../48-app-arch/app/archive/rose-app.conf | 69 ++++++ .../app/archive_bad_1/rose-app.conf | 9 + .../app/archive_bad_2/rose-app.conf | 9 + .../app/archive_bad_3/rose-app.conf | 9 + .../app/archive_bad_4/rose-app.conf | 9 + .../app/archive_bad_5/rose-app.conf | 13 ++ .../app/archive_bad_6/rose-app.conf | 9 + .../app/archive_bad_7/rose-app.conf | 12 + .../app/archive_bad_8/rose-app.conf | 12 + .../app/archive_bad_9/bin/my-bad-command | 4 + .../app/archive_bad_9/rose-app.conf | 10 + .../48-app-arch/app/install/bin/my-install | 40 ++++ .../48-app-arch/app/install/rose-app.conf | 2 + t/rose-task-run/48-app-arch/bin/foo | 35 +++ t/rose-task-run/48-app-arch/flow.cylc | 49 ++++ t/rose-task-run/48-app-arch/rose-suite.conf | 0 41 files changed, 1570 insertions(+), 2 deletions(-) create mode 100644 metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py create mode 100644 metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py create mode 100755 t/rose-task-run/46-app-arch-zstd.t create mode 100644 t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf create mode 100644 t/rose-task-run/46-app-arch-zstd/flow.cylc create mode 100644 t/rose-task-run/46-app-arch-zstd/rose-suite.conf create mode 100755 t/rose-task-run/47-app-arch-xz.t create mode 100644 t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf create mode 100644 t/rose-task-run/47-app-arch-xz/flow.cylc create mode 100644 t/rose-task-run/47-app-arch-xz/rose-suite.conf create mode 100644 t/rose-task-run/48-app-arch-20130101T0000Z.out create mode 100644 t/rose-task-run/48-app-arch-20130101T1200Z.out create mode 100644 t/rose-task-run/48-app-arch-20130102T0000Z.out create mode 100644 t/rose-task-run/48-app-arch-bad-9.out create mode 100644 t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out create mode 100644 t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out create mode 100644 t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out create mode 100644 t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out create mode 100644 t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out create mode 100644 t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out create mode 100644 t/rose-task-run/48-app-arch-find-foo.out create mode 100644 t/rose-task-run/48-app-arch-planet-n.out create mode 100644 t/rose-task-run/48-app-arch-unknown-stuff.out create mode 100755 t/rose-task-run/48-app-arch.t create mode 100644 t/rose-task-run/48-app-arch/app/archive/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf create mode 100755 t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command create mode 100644 t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf create mode 100755 t/rose-task-run/48-app-arch/app/install/bin/my-install create mode 100644 t/rose-task-run/48-app-arch/app/install/rose-app.conf create mode 100755 t/rose-task-run/48-app-arch/bin/foo create mode 100644 t/rose-task-run/48-app-arch/flow.cylc create mode 100644 t/rose-task-run/48-app-arch/rose-suite.conf diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 6d297979d5..1d59304898 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -25,9 +25,11 @@ class RoseArchTarGzip: """Compress archive sources in tar.""" - SCHEMES = ["pax", "pax.gz", "tar", "tar.gz", "tgz"] - SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, "pax.gz": tarfile.PAX_FORMAT} + SCHEMES = ["pax", "pax.gz", "pax.zst", "pax.xz", "tar", "tar.gz", "tgz", "tar.zst", "tar.xz", "txz"] + SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, "pax.gz": tarfile.PAX_FORMAT, "pax.zst": tarfile.PAX_FORMAT, "pax.xz": tarfile.PAX_FORMAT} GZIP_EXTS = ["pax.gz", "tar.gz", "tgz"] + ZSTD_EXTS = ["pax.zst", "tar.zst"] + XZ_EXTS = ["pax.xz", "tar.xz", "txz"] def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner @@ -70,3 +72,23 @@ def compress_sources(self, target, work_dir): command = "gzip -c '%s' >'%s'" % (tar_name, gz_name) self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) + + if target.compress_scheme in self.ZSTD_EXTS: + fdsec, zst_name = mkstemp( + suffix="." + target.compress_scheme, dir=work_dir + ) + os.close(fdsec) + target.work_source_path = zst_name + command = "zstd -c '%s' >'%s'" % (tar_name, zst_name) + self.app_runner.popen.run_simple(command, shell=True) + self.app_runner.fs_util.delete(tar_name) + + if target.compress_scheme in self.XZ_EXTS: + fdsec, xz_name = mkstemp( + suffix="." + target.compress_scheme, dir=work_dir + ) + os.close(fdsec) + target.work_source_path = xz_name + command = "xz -c '%s' >'%s'" % (tar_name, xz_name) + self.app_runner.popen.run_simple(command, shell=True) + self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py new file mode 100644 index 0000000000..97a455bdf7 --- /dev/null +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py @@ -0,0 +1,49 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Compress archive sources using xz.""" + + +import os + + +class RoseArchXz: + + """Compress archive sources in xz.""" + + SCHEMES = ["xz"] + + def __init__(self, app_runner, *args, **kwargs): + self.app_runner = app_runner + + def compress_sources(self, target, work_dir): + """xz each source in target. + + Use work_dir to dump results. + + """ + for source in target.sources.values(): + if source.path.endswith("." + target.compress_scheme): + continue # assume already done + name_xz = source.name + "." + target.compress_scheme + work_path_xz = os.path.join(work_dir, name_xz) + self.app_runner.fs_util.makedirs( + self.app_runner.fs_util.dirname(work_path_xz) + ) + + command = "xz -c '%s' >'%s'" % (source.path, work_path_xz) + self.app_runner.popen.run_simple(command, shell=True) + source.path = work_path_xz diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py new file mode 100644 index 0000000000..5c659ed12a --- /dev/null +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -0,0 +1,49 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Compress archive sources using zstd.""" + + +import os + + +class RoseArchZstd: + + """Compress archive sources in zstd.""" + + SCHEMES = ["zst", "zstd"] + + def __init__(self, app_runner, *args, **kwargs): + self.app_runner = app_runner + + def compress_sources(self, target, work_dir): + """zstd each source in target. + + Use work_dir to dump results. + + """ + for source in target.sources.values(): + if source.path.endswith("." + target.compress_scheme): + continue # assume already done + name_zst = source.name + "." + target.compress_scheme + work_path_zst = os.path.join(work_dir, name_zst) + self.app_runner.fs_util.makedirs( + self.app_runner.fs_util.dirname(work_path_zst) + ) + + command = "zstd -c '%s' >'%s'" % (source.path, work_path_zst) + self.app_runner.popen.run_simple(command, shell=True) + source.path = work_path_zst diff --git a/t/rose-task-run/46-app-arch-zstd.t b/t/rose-task-run/46-app-arch-zstd.t new file mode 100755 index 0000000000..22d71fd404 --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd.t @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.zst +./archive.d/whatever.tar.zst +__FIND__ +#------------------------------------------------------------------------------- +purge +exit 0 diff --git a/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf b/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf new file mode 100644 index 0000000000..9c557e203e --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf @@ -0,0 +1,16 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.zst] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.zst + +[arch:archive.d/whatever.tar.zst] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.zst diff --git a/t/rose-task-run/46-app-arch-zstd/flow.cylc b/t/rose-task-run/46-app-arch-zstd/flow.cylc new file mode 100644 index 0000000000..fe8e475a31 --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +zstd -f - <<<'MMXVI' >'2016.txt.zst' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I zstd -cf 'whatever.tar.zst' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/46-app-arch-zstd/rose-suite.conf b/t/rose-task-run/46-app-arch-zstd/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/rose-task-run/47-app-arch-xz.t b/t/rose-task-run/47-app-arch-xz.t new file mode 100755 index 0000000000..4f19cb184e --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz.t @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.xz +./archive.d/whatever.tar.xz +__FIND__ +#------------------------------------------------------------------------------- +purge +exit 0 diff --git a/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf b/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf new file mode 100644 index 0000000000..0aa738ebea --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf @@ -0,0 +1,16 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.xz] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.xz + +[arch:archive.d/whatever.tar.xz] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.xz diff --git a/t/rose-task-run/47-app-arch-xz/flow.cylc b/t/rose-task-run/47-app-arch-xz/flow.cylc new file mode 100644 index 0000000000..3ef861d58e --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +xz -f - <<<'MMXVI' >'2016.txt.xz' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I xz -cf 'whatever.tar.xz' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/47-app-arch-xz/rose-suite.conf b/t/rose-task-run/47-app-arch-xz/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/rose-task-run/48-app-arch-20130101T0000Z.out b/t/rose-task-run/48-app-arch-20130101T0000Z.out new file mode 100644 index 0000000000..c21954156f --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130101T0000Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130101T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130101T0000Z-star-1.txt (star-1.txt) +[INFO] + 20130101T0000Z-star-2.txt (star-2.txt) +[INFO] + 20130101T0000Z-star-3.txt (star-3.txt) +[INFO] + 20130101T0000Z-star-4.txt (star-4.txt) +[INFO] + 20130101T0000Z-star-5.txt (star-5.txt) +[INFO] + 20130101T0000Z-star-6.txt (star-6.txt) +[INFO] + 20130101T0000Z-star-7.txt (star-7.txt) +[INFO] + 20130101T0000Z-star-8.txt (star-8.txt) +[INFO] + 20130101T0000Z-star-9.txt (star-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130101T0000Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130101T0000Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130101T0000Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130101T0000Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130101T0000Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130101T0000Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130101T0000Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130101T0000Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130101T0000Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130101T0000Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130101T0000Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130101T0000Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T0000Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130101T0000Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130101T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130101T0000Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130101T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-20130101T1200Z.out b/t/rose-task-run/48-app-arch-20130101T1200Z.out new file mode 100644 index 0000000000..2a8f42d423 --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130101T1200Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130101T1200Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130101T1200Z-star-1.txt (star-1.txt) +[INFO] + 20130101T1200Z-star-2.txt (star-2.txt) +[INFO] + 20130101T1200Z-star-3.txt (star-3.txt) +[INFO] + 20130101T1200Z-star-4.txt (star-4.txt) +[INFO] + 20130101T1200Z-star-5.txt (star-5.txt) +[INFO] + 20130101T1200Z-star-6.txt (star-6.txt) +[INFO] + 20130101T1200Z-star-7.txt (star-7.txt) +[INFO] + 20130101T1200Z-star-8.txt (star-8.txt) +[INFO] + 20130101T1200Z-star-9.txt (star-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130101T1200Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130101T1200Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130101T1200Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130101T1200Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130101T1200Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130101T1200Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130101T1200Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130101T1200Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130101T1200Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130101T1200Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130101T1200Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130101T1200Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T1200Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130101T1200Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130101T1200Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130101T1200Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130101T1200Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-20130102T0000Z.out b/t/rose-task-run/48-app-arch-20130102T0000Z.out new file mode 100644 index 0000000000..8b6394405e --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130102T0000Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130102T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130102T0000Z-star-1.txt (star-1.txt) +[INFO] + 20130102T0000Z-star-2.txt (star-2.txt) +[INFO] + 20130102T0000Z-star-3.txt (star-3.txt) +[INFO] + 20130102T0000Z-star-4.txt (star-4.txt) +[INFO] + 20130102T0000Z-star-5.txt (star-5.txt) +[INFO] + 20130102T0000Z-star-6.txt (star-6.txt) +[INFO] + 20130102T0000Z-star-7.txt (star-7.txt) +[INFO] + 20130102T0000Z-star-8.txt (star-8.txt) +[INFO] + 20130102T0000Z-star-9.txt (star-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130102T0000Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130102T0000Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130102T0000Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130102T0000Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130102T0000Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130102T0000Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130102T0000Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130102T0000Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130102T0000Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130102T0000Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130102T0000Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130102T0000Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130102T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130102T0000Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130102T0000Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130102T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130102T0000Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130102T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-bad-9.out b/t/rose-task-run/48-app-arch-bad-9.out new file mode 100644 index 0000000000..d83fae253f --- /dev/null +++ b/t/rose-task-run/48-app-arch-bad-9.out @@ -0,0 +1,29 @@ +[INFO] ! foo://20130101T1200Z/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] ! hello/earth.txt (hello/earth.txt) +[INFO] ! hello/jupiter-moon-1.txt (hello/jupiter-moon-1.txt) +[INFO] ! hello/jupiter.txt (hello/jupiter.txt) +[INFO] ! hello/mars.txt (hello/mars.txt) +[INFO] ! hello/mercury.txt (hello/mercury.txt) +[INFO] ! hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] ! hello/planet-1.txt (hello/planet-1.txt) +[INFO] ! hello/planet-2.txt (hello/planet-2.txt) +[INFO] ! hello/planet-3.txt (hello/planet-3.txt) +[INFO] ! hello/planet-4.txt (hello/planet-4.txt) +[INFO] ! hello/planet-5.txt (hello/planet-5.txt) +[INFO] ! hello/planet-6.txt (hello/planet-6.txt) +[INFO] ! hello/planet-7.txt (hello/planet-7.txt) +[INFO] ! hello/planet-8.txt (hello/planet-8.txt) +[INFO] ! hello/planet-9.txt (hello/planet-9.txt) +[INFO] ! hello/saturn.txt (hello/saturn.txt) +[INFO] ! hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] ! hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] ! hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] ! hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] ! hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] ! hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] ! hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] ! hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] ! hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] ! hello/uranus.txt (hello/uranus.txt) +[INFO] ! hello/venus.txt (hello/venus.txt) diff --git a/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out b/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out new file mode 100644 index 0000000000..55332bc787 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out @@ -0,0 +1,66 @@ +foo://20130101T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130101T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/jupiter.txt|0| +foo://20130101T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/saturn.txt|0| +foo://20130101T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T0000Z/hello/uranus.txt|0| +foo://20130101T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|0218271ecf447dade73fa9b20ca806ac +foo://20130101T0000Z/hello/worlds/earth.txt|hello/earth.txt|64a40fb18eeb19581b1a3b9a7fd00b77 +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|ad79c5ad231e48d9986e7b32ef78ecfb +foo://20130101T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130101T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|cac4e4ff00049b6656e77e53612fbb2d +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|d5c0f47b697168163badd9b58c263726 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|f4657ee8b08b153d0d2a3ded6b943e72 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|600f9b25d93c8d4f3e4666da5e1c1be8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|339ec467c56ce5b50deb2f5ef9442a64 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|118dcd67a2d96968bd5a552a15f109a8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|1d8a5bf446b565ca4fe461ed89bc2143 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|9790ab68d589f0a3c93a4f93eb8d0979 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|9402ad6c5ede0bd21eccaa71cca635b6 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|a860bc83f223a37ddd3560ce9d9cf128 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|0842b4e54f373cc6daf6a72ef3bacef5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|e07d1be49e0116266ddd0b5b52eca0f5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|64e75062f6ca23f78f489286cdf1befb +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|b2813c2febf24d6b9c460d6e91d5584a +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|83ddd3b9422015989eb33f9f213cb1d3 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|0ef7af3d861cb997d9318410686ddeb2 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|055509281d7b83f60b867f39d817ad72 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|6c56036bfc0f7fbf855e67ebe632db04 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|eeaa51883a8b1ae0acaaecfa1ff85090 +foo://20130101T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|5a7a6e3b1b7976a07bac7fc0f8890e19 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|4b8aa78c4db8f5a17852821bc5856fd1 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|82ef697d04d3e175a6653d5047e4379e +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|c9bfb5f29d4c94bbd6cfeaddac383535 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|9e120d5961a129fcf7e16baf2c9c2d35 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b21bd2b81d440e9d1dbf82db12334bb4 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|d37b95e99a7f9647be60bfc4176cca9a +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|050e316a433bc33923d61052b17afb14 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|7e79965ca3fa019c9bec8bdd4a745067 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|9ee1dfff7161007ff211325b909775af +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-1.txt|39184686037c4f089065df1cea2243db +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-2.txt|86603e313db52af9313b49ab8b7d03a2 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-3.txt|eb9a6efe551ae81ab0867087b60b9456 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-4.txt|04276542f5259f022f85e99094364d26 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-5.txt|76df6a7b327a0a873e9d99708ee31e53 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-6.txt|fc1a89e038a378ce58e3d90921e478b4 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-7.txt|c097220252b6623d69815f0d405832e0 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-8.txt|447d11a036427f2271edc11287a3be84 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-9.txt|b3d436b8a13089629c196ccbae5a3dcd +foo://20130101T0000Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-1.txt|9def5f332e2904c360a5c88cb1abbbfb +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-2.txt|e4295cbe0bd6b64331b097eb6e4baf8d +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-3.txt|437f605878bee78144f1fdb42d839289 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-4.txt|7a50b985d13f98b81cd2e4afbb7f1e29 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-5.txt|7d2a84842e1321f1cddceeb94329d778 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-6.txt|d93899e3495e231c1dc8c42b8240ceb5 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-7.txt|6622fd0c176f91d5107946db1c98b9bc +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-8.txt|ed91e4d43abc966d9b1c4d5ccb49af55 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-9.txt|3266b847bb54c55c87f3329bead70801 +foo://20130101T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|f1737a14222ad46cb3cb567a39c00b11 diff --git a/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out b/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out new file mode 100644 index 0000000000..0b9700e893 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out @@ -0,0 +1,66 @@ +foo://20130101T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/jupiter.txt|0| +foo://20130101T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/saturn.txt|0| +foo://20130101T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T0000Z/hello/uranus.txt|0| +foo://20130101T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|0218271ecf447dade73fa9b20ca806ac +foo://20130101T0000Z/hello/worlds/earth.txt|hello/earth.txt|64a40fb18eeb19581b1a3b9a7fd00b77 +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|ad79c5ad231e48d9986e7b32ef78ecfb +foo://20130101T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130101T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|cac4e4ff00049b6656e77e53612fbb2d +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|d5c0f47b697168163badd9b58c263726 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|f4657ee8b08b153d0d2a3ded6b943e72 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|600f9b25d93c8d4f3e4666da5e1c1be8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|339ec467c56ce5b50deb2f5ef9442a64 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|118dcd67a2d96968bd5a552a15f109a8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|1d8a5bf446b565ca4fe461ed89bc2143 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|9790ab68d589f0a3c93a4f93eb8d0979 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|9402ad6c5ede0bd21eccaa71cca635b6 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|a860bc83f223a37ddd3560ce9d9cf128 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|0842b4e54f373cc6daf6a72ef3bacef5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|e07d1be49e0116266ddd0b5b52eca0f5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|64e75062f6ca23f78f489286cdf1befb +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|b2813c2febf24d6b9c460d6e91d5584a +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|83ddd3b9422015989eb33f9f213cb1d3 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|0ef7af3d861cb997d9318410686ddeb2 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|055509281d7b83f60b867f39d817ad72 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|6c56036bfc0f7fbf855e67ebe632db04 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|eeaa51883a8b1ae0acaaecfa1ff85090 +foo://20130101T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|5a7a6e3b1b7976a07bac7fc0f8890e19 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|4b8aa78c4db8f5a17852821bc5856fd1 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|82ef697d04d3e175a6653d5047e4379e +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|c9bfb5f29d4c94bbd6cfeaddac383535 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|9e120d5961a129fcf7e16baf2c9c2d35 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b21bd2b81d440e9d1dbf82db12334bb4 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|d37b95e99a7f9647be60bfc4176cca9a +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|050e316a433bc33923d61052b17afb14 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|7e79965ca3fa019c9bec8bdd4a745067 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|9ee1dfff7161007ff211325b909775af +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-1.txt|39184686037c4f089065df1cea2243db +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-2.txt|86603e313db52af9313b49ab8b7d03a2 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-3.txt|eb9a6efe551ae81ab0867087b60b9456 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-4.txt|04276542f5259f022f85e99094364d26 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-5.txt|76df6a7b327a0a873e9d99708ee31e53 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-6.txt|fc1a89e038a378ce58e3d90921e478b4 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-7.txt|c097220252b6623d69815f0d405832e0 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-8.txt|447d11a036427f2271edc11287a3be84 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-9.txt|b3d436b8a13089629c196ccbae5a3dcd +foo://20130101T0000Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-1.txt|9def5f332e2904c360a5c88cb1abbbfb +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-2.txt|e4295cbe0bd6b64331b097eb6e4baf8d +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-3.txt|437f605878bee78144f1fdb42d839289 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-4.txt|7a50b985d13f98b81cd2e4afbb7f1e29 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-5.txt|7d2a84842e1321f1cddceeb94329d778 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-6.txt|d93899e3495e231c1dc8c42b8240ceb5 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-7.txt|6622fd0c176f91d5107946db1c98b9bc +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-8.txt|ed91e4d43abc966d9b1c4d5ccb49af55 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-9.txt|3266b847bb54c55c87f3329bead70801 +foo://20130101T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out b/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out new file mode 100644 index 0000000000..e86d46b7bc --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out @@ -0,0 +1,66 @@ +foo://20130101T1200Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130101T1200Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/jupiter.txt|0| +foo://20130101T1200Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/saturn.txt|0| +foo://20130101T1200Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T1200Z/hello/uranus.txt|0| +foo://20130101T1200Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|e621a29f24a71f23b3780e2bb0117bb9 +foo://20130101T1200Z/hello/worlds/earth.txt|hello/earth.txt|788bad396b603b9783f9e7bcafa6c797 +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|94c7fccf08f8b7959d884eaff7312227 +foo://20130101T1200Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130101T1200Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|f6674a8308b4d43791f9c52e07e1fa7c +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|f9c382939532a88c769a4221b1e22503 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|c48fcde460349f75ac39e850354ec628 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|052aebda2673e7a0e155486563b90f39 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|b68a019e7d41306580c5dcb78553040b +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|ad22c9d8f490d06fb5bc67cc5a169240 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|9a698b7c5d5118f4d1d3eb176b54c023 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|66bb8f516bafed7658826592ba352eb5 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|1dea09e75c6ce4461078a7ded7252e55 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|e1f7d05200a141edf26cfc4b1156e798 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|8a98f0c613e771f5d331a2d34c2895da +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|8cad6bcdf4d22e1dfe1d35fa03c6f5cb +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|275ab56c89605dd66078b80834253baa +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|dfe0cff95c79512d1bbc1d87c74cd36a +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|a90b5e210b99eb5f922a6ee672f5565b +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|1663ba51e68369147d92569515134353 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|6afc5fc439021fcf4089c282da667816 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|ba860ddc208de7748145144eb7d83b62 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|7557bf0caf752172b29fc74afea77ce8 +foo://20130101T1200Z/hello/worlds/saturn.txt|hello/saturn.txt|6bb3a3d31e658b6f33cc74aa29a15275 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-1.txt|d184f4517e6210d7c0f076127441572f +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-2.txt|d1222f82c23304d5d9c52f6e7e973522 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-3.txt|38fd825c464f691d9cdc0f45456fe4ac +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-4.txt|6c501adbd95198f88e756b6b6db59116 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-5.txt|44d8c92a12c4b7c9eb278f52ced5e234 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-6.txt|4e6150d0e385688e71b6b3947d500adf +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-7.txt|4cc45c56b453604e197fc79f39cd484c +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-8.txt|45dd9c8b7454e9aa007616224a196c6e +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-9.txt|0e2a93472b0faaf22a0b790679e102e1 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-1.txt|9c0756f2e12955569750f9143e75f1a6 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-2.txt|9a017867af5ffc2156b426bdffaaf444 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-3.txt|7416e2c7a8577dbdce044c809e173a4d +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-4.txt|e6659108a95d8197f6167ec23a217179 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-5.txt|b043475305dc7e845a01935eb8ae6363 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-6.txt|1869ae024acdd3c797d6c8e530d89888 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-7.txt|4512974ee6609d1c23079163e8f16120 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-8.txt|c24a8e117d5de41b79af875eb382eb7f +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-9.txt|113ba9f4a4e3bd7d82b7a317b9bfec9e +foo://20130101T1200Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-1.txt|f64291009d30056102802980bf80e1c9 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-2.txt|c94c43b3e3c3dc55dfbca86890902816 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-3.txt|d4959cb344dd139b33766337e00f9d85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-4.txt|571c83c56b247d8d95fbd4225016e2b4 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-5.txt|98343306a5ff665121a754f2fedbce1b +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-6.txt|989d6be0f0f83840d7959df50cb25ee5 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-7.txt|fa121b4fb2a8e3bcf99f0d758a885682 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-8.txt|2e9be382c5071a6d0032ca166ead2d39 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-9.txt|99c4e7f4d54851314a8a6aea2d1d4a02 +foo://20130101T1200Z/hello/worlds/uranus.txt|hello/uranus.txt|47afc5ee013b505bcb6f8eb244ddd42a diff --git a/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out b/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out new file mode 100644 index 0000000000..b53941af70 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out @@ -0,0 +1,66 @@ +foo://20130101T1200Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/jupiter.txt|0| +foo://20130101T1200Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/saturn.txt|0| +foo://20130101T1200Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T1200Z/hello/uranus.txt|0| +foo://20130101T1200Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|e621a29f24a71f23b3780e2bb0117bb9 +foo://20130101T1200Z/hello/worlds/earth.txt|hello/earth.txt|788bad396b603b9783f9e7bcafa6c797 +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|94c7fccf08f8b7959d884eaff7312227 +foo://20130101T1200Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130101T1200Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|f6674a8308b4d43791f9c52e07e1fa7c +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|f9c382939532a88c769a4221b1e22503 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|c48fcde460349f75ac39e850354ec628 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|052aebda2673e7a0e155486563b90f39 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|b68a019e7d41306580c5dcb78553040b +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|ad22c9d8f490d06fb5bc67cc5a169240 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|9a698b7c5d5118f4d1d3eb176b54c023 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|66bb8f516bafed7658826592ba352eb5 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|1dea09e75c6ce4461078a7ded7252e55 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|e1f7d05200a141edf26cfc4b1156e798 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|8a98f0c613e771f5d331a2d34c2895da +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|8cad6bcdf4d22e1dfe1d35fa03c6f5cb +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|275ab56c89605dd66078b80834253baa +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|dfe0cff95c79512d1bbc1d87c74cd36a +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|a90b5e210b99eb5f922a6ee672f5565b +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|1663ba51e68369147d92569515134353 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|6afc5fc439021fcf4089c282da667816 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|ba860ddc208de7748145144eb7d83b62 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|7557bf0caf752172b29fc74afea77ce8 +foo://20130101T1200Z/hello/worlds/saturn.txt|hello/saturn.txt|6bb3a3d31e658b6f33cc74aa29a15275 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-1.txt|d184f4517e6210d7c0f076127441572f +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-2.txt|d1222f82c23304d5d9c52f6e7e973522 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-3.txt|38fd825c464f691d9cdc0f45456fe4ac +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-4.txt|6c501adbd95198f88e756b6b6db59116 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-5.txt|44d8c92a12c4b7c9eb278f52ced5e234 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-6.txt|4e6150d0e385688e71b6b3947d500adf +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-7.txt|4cc45c56b453604e197fc79f39cd484c +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-8.txt|45dd9c8b7454e9aa007616224a196c6e +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-9.txt|0e2a93472b0faaf22a0b790679e102e1 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-1.txt|9c0756f2e12955569750f9143e75f1a6 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-2.txt|9a017867af5ffc2156b426bdffaaf444 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-3.txt|7416e2c7a8577dbdce044c809e173a4d +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-4.txt|e6659108a95d8197f6167ec23a217179 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-5.txt|b043475305dc7e845a01935eb8ae6363 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-6.txt|1869ae024acdd3c797d6c8e530d89888 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-7.txt|4512974ee6609d1c23079163e8f16120 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-8.txt|c24a8e117d5de41b79af875eb382eb7f +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-9.txt|113ba9f4a4e3bd7d82b7a317b9bfec9e +foo://20130101T1200Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-1.txt|f64291009d30056102802980bf80e1c9 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-2.txt|c94c43b3e3c3dc55dfbca86890902816 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-3.txt|d4959cb344dd139b33766337e00f9d85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-4.txt|571c83c56b247d8d95fbd4225016e2b4 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-5.txt|98343306a5ff665121a754f2fedbce1b +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-6.txt|989d6be0f0f83840d7959df50cb25ee5 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-7.txt|fa121b4fb2a8e3bcf99f0d758a885682 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-8.txt|2e9be382c5071a6d0032ca166ead2d39 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-9.txt|99c4e7f4d54851314a8a6aea2d1d4a02 +foo://20130101T1200Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out b/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out new file mode 100644 index 0000000000..91f590e58a --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out @@ -0,0 +1,66 @@ +foo://20130102T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130102T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/jupiter.txt|0| +foo://20130102T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/saturn.txt|0| +foo://20130102T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130102T0000Z/hello/uranus.txt|0| +foo://20130102T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|8ee65f3e701e07c0c3ad586dc58a948c +foo://20130102T0000Z/hello/worlds/earth.txt|hello/earth.txt|d912f30f89b5d40f0ffa434e9e9714a8 +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|7a4d5d5aba14ba1e059086ac6e0e34d0 +foo://20130102T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130102T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|52be17cb5032ab1ee2b3a34407e4f78f +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|bf410362cfc403e2542bdcb1ae46acce +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|8ba02aef72f42796d591e8858178bf0e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|2989e4dbaa92e00fb7a05cc2db97468d +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|4ad3cf583dc9618d4986ff27d8fafdfd +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|233105195d5f897efaa46f2e0483d24e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|c63b821746a19144b2b507ef8744d1af +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|656ec00852d9d90f929a689b1dfe1f81 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|7798ae798fffcb529a8729cf5f5ba128 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|bdaecbceb631bac47bd2749bd20acab5 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|d1bf2cfa960311278a3aea9afaf60fca +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|bbebf2273dab1dc56b104f6373c1b249 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|f4f61419fc7464ce8b132f8b9729740e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|0df8d342722232d196dd1049e78b3e77 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|dea2da39535cfc5af553f04402bbfc0e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|980aac5825abde73528a0c823309fc64 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|97ef84d6a5bfab093523e20e09aa5137 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|a53efd68f81f975bd317f1e39b39f7f3 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|ff0fbf9027e58543bacc15e5b7d289a8 +foo://20130102T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|2610a66e838d75c745d8a64ff8541812 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|9612fa0c9617830d044e9e402f74d834 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|8010aa72a3d5fe2ef4c4bf9d38751de3 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|83cf0703fc41171eade4510496574063 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|1ee60607b68bd42f5df90b586f5ca0ce +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b0cc0a54e9354bba052b40dc7cd6cb3f +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|2d53a0b073804ee611bf86ec73a09589 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|f8fc808d5003015b6d3643e8825cc214 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|85a07828e07ab8aa8afa6f8262081403 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|de6b8c8c53d0d91c960644a18129cb0c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-1.txt|8839564acc51805ff31ff04ae5d968fa +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-2.txt|c499c20d00e4608a1917e3029b53b53e +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-3.txt|864b4d48a5c392c346bae087d0fc2b1b +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-4.txt|7a29d25e015216980de1f209b8ab11f5 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-5.txt|b8f755ca24d6021b4d2b7d0d5c7e7b1c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-6.txt|16bd791b242665fcb42a2457dca44dc1 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-7.txt|f0b3afb481ce3d593d9fc9820cef0456 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-8.txt|bd89739f414b8cb132bc7c1e9dcdf185 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-9.txt|326c18a6f265aabd6fe26e8c69dcad60 +foo://20130102T0000Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-1.txt|a69330fee599fe039f6cb1d467f70393 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-2.txt|7b1e0742b9df08316e24fcabb957b482 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-3.txt|c8b60d5d4bf9b42c411c07c853952283 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-4.txt|ea557c22f3a35303b19dcdfd45dacd01 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-5.txt|468cd05e391cbdaf628af7283a407381 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-6.txt|75dcc0652cf14ea3cab8a88335a0918c +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-7.txt|6d5d10d843d9d04b975bbef7bfe908e5 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-8.txt|ca0fb6bef96ec3e9dadaea8fc799bd54 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-9.txt|8d988c43da1dd7df887ec5a9d4acd3e6 +foo://20130102T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|e5079393fefe526d4074c2dcc8ef2462 diff --git a/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out b/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out new file mode 100644 index 0000000000..56d245981e --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out @@ -0,0 +1,66 @@ +foo://20130102T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/jupiter.txt|0| +foo://20130102T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/saturn.txt|0| +foo://20130102T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130102T0000Z/hello/uranus.txt|0| +foo://20130102T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|8ee65f3e701e07c0c3ad586dc58a948c +foo://20130102T0000Z/hello/worlds/earth.txt|hello/earth.txt|d912f30f89b5d40f0ffa434e9e9714a8 +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|7a4d5d5aba14ba1e059086ac6e0e34d0 +foo://20130102T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130102T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|52be17cb5032ab1ee2b3a34407e4f78f +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|bf410362cfc403e2542bdcb1ae46acce +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|8ba02aef72f42796d591e8858178bf0e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|2989e4dbaa92e00fb7a05cc2db97468d +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|4ad3cf583dc9618d4986ff27d8fafdfd +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|233105195d5f897efaa46f2e0483d24e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|c63b821746a19144b2b507ef8744d1af +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|656ec00852d9d90f929a689b1dfe1f81 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|7798ae798fffcb529a8729cf5f5ba128 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|bdaecbceb631bac47bd2749bd20acab5 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|d1bf2cfa960311278a3aea9afaf60fca +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|bbebf2273dab1dc56b104f6373c1b249 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|f4f61419fc7464ce8b132f8b9729740e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|0df8d342722232d196dd1049e78b3e77 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|dea2da39535cfc5af553f04402bbfc0e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|980aac5825abde73528a0c823309fc64 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|97ef84d6a5bfab093523e20e09aa5137 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|a53efd68f81f975bd317f1e39b39f7f3 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|ff0fbf9027e58543bacc15e5b7d289a8 +foo://20130102T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|2610a66e838d75c745d8a64ff8541812 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|9612fa0c9617830d044e9e402f74d834 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|8010aa72a3d5fe2ef4c4bf9d38751de3 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|83cf0703fc41171eade4510496574063 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|1ee60607b68bd42f5df90b586f5ca0ce +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b0cc0a54e9354bba052b40dc7cd6cb3f +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|2d53a0b073804ee611bf86ec73a09589 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|f8fc808d5003015b6d3643e8825cc214 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|85a07828e07ab8aa8afa6f8262081403 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|de6b8c8c53d0d91c960644a18129cb0c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-1.txt|8839564acc51805ff31ff04ae5d968fa +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-2.txt|c499c20d00e4608a1917e3029b53b53e +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-3.txt|864b4d48a5c392c346bae087d0fc2b1b +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-4.txt|7a29d25e015216980de1f209b8ab11f5 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-5.txt|b8f755ca24d6021b4d2b7d0d5c7e7b1c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-6.txt|16bd791b242665fcb42a2457dca44dc1 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-7.txt|f0b3afb481ce3d593d9fc9820cef0456 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-8.txt|bd89739f414b8cb132bc7c1e9dcdf185 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-9.txt|326c18a6f265aabd6fe26e8c69dcad60 +foo://20130102T0000Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-1.txt|a69330fee599fe039f6cb1d467f70393 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-2.txt|7b1e0742b9df08316e24fcabb957b482 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-3.txt|c8b60d5d4bf9b42c411c07c853952283 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-4.txt|ea557c22f3a35303b19dcdfd45dacd01 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-5.txt|468cd05e391cbdaf628af7283a407381 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-6.txt|75dcc0652cf14ea3cab8a88335a0918c +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-7.txt|6d5d10d843d9d04b975bbef7bfe908e5 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-8.txt|ca0fb6bef96ec3e9dadaea8fc799bd54 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-9.txt|8d988c43da1dd7df887ec5a9d4acd3e6 +foo://20130102T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-find-foo.out b/t/rose-task-run/48-app-arch-find-foo.out new file mode 100644 index 0000000000..e770df9790 --- /dev/null +++ b/t/rose-task-run/48-app-arch-find-foo.out @@ -0,0 +1,61 @@ +foo/20130101T0000Z/hello/worlds/dark-matter.txt +foo/20130101T0000Z/hello/worlds/earth.txt +foo/20130101T0000Z/hello/worlds/jupiter-moons.tar.zst +foo/20130101T0000Z/hello/worlds/jupiter.txt +foo/20130101T0000Z/hello/worlds/neptune-1.txt +foo/20130101T0000Z/hello/worlds/organisms.tar.zst +foo/20130101T0000Z/hello/worlds/planet-n.tar.zst +foo/20130101T0000Z/hello/worlds/saturn.txt +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130101T0000Z/hello/worlds/try.nl +foo/20130101T0000Z/hello/worlds/unknown/stuff.pax +foo/20130101T0000Z/hello/worlds/uranus.txt +foo/20130101T1200Z/hello/worlds/dark-matter.txt +foo/20130101T1200Z/hello/worlds/earth.txt +foo/20130101T1200Z/hello/worlds/jupiter-moons.tar.zst +foo/20130101T1200Z/hello/worlds/jupiter.txt +foo/20130101T1200Z/hello/worlds/mars.txt.zst +foo/20130101T1200Z/hello/worlds/neptune-1.txt +foo/20130101T1200Z/hello/worlds/organisms.tar.zst +foo/20130101T1200Z/hello/worlds/planet-n.tar.zst +foo/20130101T1200Z/hello/worlds/saturn.txt +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130101T1200Z/hello/worlds/try.nl +foo/20130101T1200Z/hello/worlds/unknown/stuff.pax +foo/20130101T1200Z/hello/worlds/uranus.txt +foo/20130102T0000Z/hello/worlds/dark-matter.txt +foo/20130102T0000Z/hello/worlds/earth.txt +foo/20130102T0000Z/hello/worlds/jupiter-moons.tar.zst +foo/20130102T0000Z/hello/worlds/jupiter.txt +foo/20130102T0000Z/hello/worlds/neptune-1.txt +foo/20130102T0000Z/hello/worlds/organisms.tar.zst +foo/20130102T0000Z/hello/worlds/planet-n.tar.zst +foo/20130102T0000Z/hello/worlds/saturn.txt +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130102T0000Z/hello/worlds/try.nl +foo/20130102T0000Z/hello/worlds/unknown/stuff.pax +foo/20130102T0000Z/hello/worlds/uranus.txt diff --git a/t/rose-task-run/48-app-arch-planet-n.out b/t/rose-task-run/48-app-arch-planet-n.out new file mode 100644 index 0000000000..a7600f6bd4 --- /dev/null +++ b/t/rose-task-run/48-app-arch-planet-n.out @@ -0,0 +1,9 @@ +hello/planet-1.txt +hello/planet-2.txt +hello/planet-3.txt +hello/planet-4.txt +hello/planet-5.txt +hello/planet-6.txt +hello/planet-7.txt +hello/planet-8.txt +hello/planet-9.txt diff --git a/t/rose-task-run/48-app-arch-unknown-stuff.out b/t/rose-task-run/48-app-arch-unknown-stuff.out new file mode 100644 index 0000000000..0ee5c02929 --- /dev/null +++ b/t/rose-task-run/48-app-arch-unknown-stuff.out @@ -0,0 +1,9 @@ +hello/$CYCLE-stuff-1.txt +hello/$CYCLE-stuff-2.txt +hello/$CYCLE-stuff-3.txt +hello/$CYCLE-stuff-4.txt +hello/$CYCLE-stuff-5.txt +hello/$CYCLE-stuff-6.txt +hello/$CYCLE-stuff-7.txt +hello/$CYCLE-stuff-8.txt +hello/$CYCLE-stuff-9.txt diff --git a/t/rose-task-run/48-app-arch.t b/t/rose-task-run/48-app-arch.t new file mode 100755 index 0000000000..b70722c6f2 --- /dev/null +++ b/t/rose-task-run/48-app-arch.t @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test rose_arch built-in application. +#------------------------------------------------------------------------------- +. $(dirname $0)/test_header + + +#------------------------------------------------------------------------------- +tests 49 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export ROSE_CONF_PATH= +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --workflow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "$FLOW" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +cp ${TEST_KEY}.err ~/temp +cp ${TEST_KEY}.out ~/temp +#------------------------------------------------------------------------------- +# Results, good ones +TEST_KEY="$TEST_KEY_BASE-find-foo" +(cd $FLOW_RUN_DIR; find foo -type f | LANG=C sort) >"$TEST_KEY.out" +file_cmp "$TEST_KEY.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" "$TEST_KEY.out" +for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do + TEST_KEY="$TEST_KEY_BASE-$CYCLE.out" + sed '/^\[INFO\] [=!+]/!d; + s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; + s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ + $FLOW_RUN_DIR/log/job/$CYCLE/archive/0*/job.out >"$TEST_KEY" + file_cmp "$TEST_KEY" "$TEST_KEY" $TEST_SOURCE_DIR/$TEST_KEY_BASE-$CYCLE.out + TEST_KEY="$TEST_KEY_BASE-planet-n" + tar -I zstd -tf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/planet-n.tar.zst | \ + LANG=C sort >"$TEST_KEY-$CYCLE.out" + file_cmp "$TEST_KEY-$CYCLE.out" \ + "$TEST_KEY-$CYCLE.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" + TEST_KEY="$TEST_KEY_BASE-unknown-stuff" + tar -tf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/unknown/stuff.pax | \ + LANG=C sort >"$TEST_KEY-$CYCLE.out" + sed "s/\\\$CYCLE/$CYCLE/" "$TEST_SOURCE_DIR/$TEST_KEY.out" \ + >"$TEST_KEY-$CYCLE.out.expected" + file_cmp "$TEST_KEY-$CYCLE.out" \ + "$TEST_KEY-$CYCLE.out" "$TEST_KEY-$CYCLE.out.expected" + TEST_KEY="$TEST_KEY_BASE-db" + for TRY in 1 2; do + FILE=$FLOW_RUN_DIR/work/$CYCLE/archive/rose-arch-db-$TRY.out + sed "s?\\\$ROSE_DATAC?$FLOW_RUN_DIR/share/cycle/$CYCLE?" \ + "$TEST_SOURCE_DIR/$TEST_KEY-$CYCLE-$TRY.out" >$FILE.expected + file_cmp "$TEST_KEY-$CYCLE.out" $FILE.expected $FILE + done + for KEY in dark-matter.txt jupiter.txt try.nl uranus.txt; do + TEST_KEY="$TEST_KEY_BASE-$CYCLE-grep-$KEY-foo-log-2" + file_grep "$TEST_KEY" $KEY $FLOW_RUN_DIR/foo.log.$CYCLE.2 + done + if test $(wc -l <$FLOW_RUN_DIR/foo.log.$CYCLE.2) -eq 4; then + pass "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" + else + fail "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" + fi + TEST_KEY="$TEST_KEY_BASE-$CYCLE-neptune-1.txt" + file_cmp "$TEST_KEY" \ + $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/neptune-1.txt <<__TXT__ +[$CYCLE] Greet Triton +__TXT__ + TEST_KEY="$TEST_KEY_BASE-$CYCLE-jupiter-moons.tar.zst" + tar -xf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/jupiter-moons.tar.zst -O \ + >"$TEST_KEY.out" + file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__TXT__ +[$CYCLE] Greet Io +__TXT__ +done +#------------------------------------------------------------------------------- +# Results, bad ones +CYCLE=20130101T1200Z +TEST_KEY="$TEST_KEY_BASE-bad-archive-1" +FILE_PREFIX="$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}1/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/hello/worlds/planet-n.tar.zst: bad command-format: foo put %(target)s %(source)s: KeyError: 'source' +[FAIL] ! foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[FAIL] ! hello/planet-1.txt (hello/planet-1.txt) +[FAIL] ! hello/planet-2.txt (hello/planet-2.txt) +[FAIL] ! hello/planet-3.txt (hello/planet-3.txt) +[FAIL] ! hello/planet-4.txt (hello/planet-4.txt) +[FAIL] ! hello/planet-5.txt (hello/planet-5.txt) +[FAIL] ! hello/planet-6.txt (hello/planet-6.txt) +[FAIL] ! hello/planet-7.txt (hello/planet-7.txt) +[FAIL] ! hello/planet-8.txt (hello/planet-8.txt) +[FAIL] ! hello/planet-9.txt (hello/planet-9.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-2" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}2/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] source=None: missing configuration error: 'source' +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-3" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}3/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] source=$UNBOUND_PLANET-[1-9].txt: configuration value error: [UNDEFINED ENVIRONMENT VARIABLE] UNBOUND_PLANET +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-4" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}4/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] [arch:$UNKNOWN_DARK_PLANETS.tar.zst]=: configuration value error: [UNDEFINED ENVIRONMENT VARIABLE] UNKNOWN_DARK_PLANETS +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-5" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}5/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ +[FAIL] [arch:inner.tar.zst]source=hello/mercurry.txt: configuration value error: [Errno 2] No such file or directory: '${FLOW_RUN_DIR}/share/cycle/20130101T1200Z/hello/mercurry.txt' +[FAIL] ! foo://20130101T1200Z/hello/worlds/inner.tar.zst [compress=tar.zst] +[FAIL] ! hello/venus.txt (hello/venus.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-6" +# NOTE: /BASH_XTRACEFD/d is to remove any Cylc errors of the form: +# BASH_XTRACEFD: 19: invalid value for trace file descriptor +sed \ + -e '/^\[FAIL\] /!d; s/ \[compress.*\]$//' \ + -e '/BASH_XTRACEFD/d' \ + "$TEST_KEY.err" \ + "${FILE_PREFIX}6/01/job.err" >"$TEST_KEY.err" +file_cmp "$TEST_KEY.err" "$TEST_KEY.err" <<__ERR__ +[FAIL] foo push foo://20130101T1200Z/hello/worlds/mars.txt.zst $FLOW_RUN_DIR/share/cycle/20130101T1200Z/hello/mars.txt # return-code=1, stderr= +[FAIL] foo: push: unknown action +[FAIL] ! foo://20130101T1200Z/hello/worlds/mars.txt.zst +[FAIL] ! hello/mars.txt (hello/mars.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-7" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}7/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/planet-n.tar.zst: bad rename-format: %(planet?maybedwarfplanet???)s: KeyError: 'planet?maybedwarfplanet???' +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-8" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}8/01/job.err" >"${TEST_KEY}.err" +# Iron out differences between error messages in Python versions: +sed -i 's/PatternError:/error:/g' "$TEST_KEY.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/planet-n.tar.zst: bad rename-parser: planet-(?P[MVEJSUN]\w+.txt: error: missing ), unterminated subpattern at position 7 +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-9" +sed '/^\[INFO\] [=!+]/!d; + s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; + s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.out" >"$TEST_KEY.out" +file_cmp "$TEST_KEY.out" \ + "$TEST_SOURCE_DIR/$TEST_KEY_BASE-bad-9.out" "$TEST_KEY.out" +sed -e '/^\[FAIL\] /!d' \ + -e 's/^\(\[FAIL\] my-bad-command\) .*\( # return-code=1, stderr=\)$/\1\2/' \ + -e '/^\[FAIL\] \[my-bad-command\]/d' \ + -e 's/ \[compress.*]$//' \ + -e '/BASH_XTRACEFD/d' \ + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.err" >"$TEST_KEY.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ +[FAIL] ! foo://20130101T1200Z/planet-n.tar.zst +[FAIL] ! hello/dark-matter.txt (hello/dark-matter.txt) +[FAIL] ! hello/earth.txt (hello/earth.txt) +[FAIL] ! hello/jupiter-moon-1.txt (hello/jupiter-moon-1.txt) +[FAIL] ! hello/jupiter.txt (hello/jupiter.txt) +[FAIL] ! hello/mars.txt (hello/mars.txt) +[FAIL] ! hello/mercury.txt (hello/mercury.txt) +[FAIL] ! hello/neptune-1.txt (hello/neptune-1.txt) +[FAIL] ! hello/planet-1.txt (hello/planet-1.txt) +[FAIL] ! hello/planet-2.txt (hello/planet-2.txt) +[FAIL] ! hello/planet-3.txt (hello/planet-3.txt) +[FAIL] ! hello/planet-4.txt (hello/planet-4.txt) +[FAIL] ! hello/planet-5.txt (hello/planet-5.txt) +[FAIL] ! hello/planet-6.txt (hello/planet-6.txt) +[FAIL] ! hello/planet-7.txt (hello/planet-7.txt) +[FAIL] ! hello/planet-8.txt (hello/planet-8.txt) +[FAIL] ! hello/planet-9.txt (hello/planet-9.txt) +[FAIL] ! hello/saturn.txt (hello/saturn.txt) +[FAIL] ! hello/spaceship-1.txt (hello/spaceship-1.txt) +[FAIL] ! hello/spaceship-2.txt (hello/spaceship-2.txt) +[FAIL] ! hello/spaceship-3.txt (hello/spaceship-3.txt) +[FAIL] ! hello/spaceship-4.txt (hello/spaceship-4.txt) +[FAIL] ! hello/spaceship-5.txt (hello/spaceship-5.txt) +[FAIL] ! hello/spaceship-6.txt (hello/spaceship-6.txt) +[FAIL] ! hello/spaceship-7.txt (hello/spaceship-7.txt) +[FAIL] ! hello/spaceship-8.txt (hello/spaceship-8.txt) +[FAIL] ! hello/spaceship-9.txt (hello/spaceship-9.txt) +[FAIL] ! hello/uranus.txt (hello/uranus.txt) +[FAIL] ! hello/venus.txt (hello/venus.txt) +[FAIL] my-bad-command # return-code=1, stderr= +__ERR__ + +#------------------------------------------------------------------------------- +purge +exit 0 diff --git a/t/rose-task-run/48-app-arch/app/archive/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive/rose-app.conf new file mode 100644 index 0000000000..6a85fd52cb --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive/rose-app.conf @@ -0,0 +1,69 @@ +mode=rose_arch + +[env] +FOO_SESSION=$ROSE_TASK_CYCLE_TIME.$CYLC_TASK_TRY_NUMBER +MY_UNKNOWN=unknown +MY_STUFF=stuff + +[file:$ROSE_DATAC/try.nl] +source=namelist:try + +[namelist:try] +n=$CYLC_TASK_TRY_NUMBER + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:earth.txt] +source=hello/earth.txt + +[arch:try.nl] +source=try.nl + +[arch:planet-n.tar.zst] +source=hello/planet-[1-9].txt + +[arch:spaceships/] +compress=zst +source=hello/spaceship-[1-9].txt + +[arch:stars/] +rename-format=%(cycle)s-%(name)s +source=star-[1-9].txt + +[arch:${MY_UNKNOWN}/${MY_STUFF}.pax] +rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s +rename-parser=^(?Pstuff)ing(?P-[1-9]\.txt)$ +source=stuffing-*.txt + +[arch:dark-matter.txt] +command-format=FOO_RC=$((2 - $CYLC_TASK_TRY_NUMBER)) foo put %(target)s %(sources)s +source=hello/dark-matter.txt + +[arch:organisms.tar.zst] +source=hello/organisms/ + +[arch:jupiter.txt] +command-format=foo put %(target)s %(sources)s; touch -r share/cycle/$ROSE_TASK_CYCLE_TIME/hello/t-ref share/cycle/$ROSE_TASK_CYCLE_TIME/hello/jupiter.txt +source=hello/jupiter.txt +update-check=mtime+size + +[arch:saturn.txt] +command-format=foo put %(target)s %(sources)s; touch -r share/cycle/$ROSE_TASK_CYCLE_TIME/hello/t-ref share/cycle/$ROSE_TASK_CYCLE_TIME/hello/saturn.txt +source=hello/saturn.txt + +[arch:uranus.txt] +command-format=foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/$ROSE_TASK_CYCLE_TIME/hello/uranus.txt +source=hello/uranus.txt + +[arch:jupiter-moons.tar.zst] +source=hello/jupiter-moon-*.txt +rename-parser=^(?Phello/jupiter)-moon(?P-[1-9]\.txt)$ +rename-format=%(name_head)s%(name_tail)s +source-edit-format=sed 's/Hello/Greet/' %(in)s >%(out)s + +[arch:neptune-1.txt] +source=hello/neptune-1.txt +source-edit-format=sed 's/Hello/Greet/' %(in)s >%(out)s diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf new file mode 100644 index 0000000000..27467972c4 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(source)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:planet-n.tar.zst] +source=hello/planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf new file mode 100644 index 0000000000..f44b882db0 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +sources=planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf new file mode 100644 index 0000000000..2f12889168 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=$UNBOUND_PLANET-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf new file mode 100644 index 0000000000..a50f97c185 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(source)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:$UNKNOWN_DARK_PLANETS.tar.zst] +source=hello/planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf new file mode 100644 index 0000000000..368c8f05e1 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf @@ -0,0 +1,13 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:mars.txt.zst] +source=hello/mars.txt + +# Too much curry last night +[arch:inner.tar.zst] +source=hello/venus.txt hello/mercurry.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf new file mode 100644 index 0000000000..5dfdee811f --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo push %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:mars.txt.zst] +source=hello/mars.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf new file mode 100644 index 0000000000..b1f132118d --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf @@ -0,0 +1,12 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +# The following format will fail: +rename-format=%(planet?maybedwarfplanet???)s +rename-parser=planet-(?P[MVEJSUN]\w+).txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf new file mode 100644 index 0000000000..272a17a8a8 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf @@ -0,0 +1,12 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +rename-format=%(planet)s +# The following will fail Python regex compilation: +rename-parser=planet-(?P[MVEJSUN]\w+.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command b/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command new file mode 100755 index 0000000000..7abc5524da --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +THIS=$(basename $0) +echo "[$THIS] $*" >&2 +exit 1 diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf new file mode 100644 index 0000000000..8716d955dd --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf @@ -0,0 +1,10 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +source-edit-format=my-bad-command %(in)s %(out)s diff --git a/t/rose-task-run/48-app-arch/app/install/bin/my-install b/t/rose-task-run/48-app-arch/app/install/bin/my-install new file mode 100755 index 0000000000..4db149c402 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/install/bin/my-install @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -eu +PREFIX= +if [[ ${ROSE_TASK_CYCLE_TIME:-} ]]; then + PREFIX="[$ROSE_TASK_CYCLE_TIME] " +fi +mkdir -p $ROSE_DATAC/hello +echo "${PREFIX}Hello Earth" >$ROSE_DATAC/hello/earth.txt +echo "${PREFIX}Hello Dark Matter" >$ROSE_DATAC/hello/dark-matter.txt +echo "${PREFIX}Hello Mars" >$ROSE_DATAC/hello/mars.txt +echo "${PREFIX}Hello Venus" >$ROSE_DATAC/hello/venus.txt +echo "${PREFIX}Hello mercury" >$ROSE_DATAC/hello/mercury.txt +for I in $(seq 1 9); do + echo "${PREFIX}Hello Planet $I" >$ROSE_DATAC/hello/planet-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Spaceship $I" >$ROSE_DATAC/hello/spaceship-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Star $I" >$ROSE_DATAC/star-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Stuff $I" >$ROSE_DATAC/stuffing-$I.txt +done +mkdir -p $ROSE_DATAC/hello/organisms/{plants,animals} +for NAME in lily jasmine holly iris daisy; do + echo "${PREFIX}Hello $NAME" >$ROSE_DATAC/hello/organisms/plants/$NAME.txt +done +for NAME in elephant tiger goose crocodile; do + echo "${PREFIX}Hello $NAME" >$ROSE_DATAC/hello/organisms/animals/$NAME.txt +done +touch $ROSE_DATAC/hello/t-ref +touch -t 201212202012.00 $ROSE_DATAC/hello/t-ref +echo "${PREFIX}Hello jupiter" >$ROSE_DATAC/hello/jupiter.txt +touch -t 201111202011.00 $ROSE_DATAC/hello/jupiter.txt +echo "${PREFIX}Hello saturn" >$ROSE_DATAC/hello/saturn.txt +touch -t 201111202011.00 $ROSE_DATAC/hello/saturn.txt +echo "${PREFIX}Hello uranus" >$ROSE_DATAC/hello/uranus.txt +echo "${PREFIX}Hello Io" >$ROSE_DATAC/hello/jupiter-moon-1.txt +echo "${PREFIX}Hello Triton" >$ROSE_DATAC/hello/neptune-1.txt diff --git a/t/rose-task-run/48-app-arch/app/install/rose-app.conf b/t/rose-task-run/48-app-arch/app/install/rose-app.conf new file mode 100644 index 0000000000..4f2ceb8dbe --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/install/rose-app.conf @@ -0,0 +1,2 @@ +[command] +default=my-install diff --git a/t/rose-task-run/48-app-arch/bin/foo b/t/rose-task-run/48-app-arch/bin/foo new file mode 100755 index 0000000000..532940fe39 --- /dev/null +++ b/t/rose-task-run/48-app-arch/bin/foo @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +set -eu +FOO=$(basename $0) +FOO_SESSION=${FOO_SESSION:-0} +FOO_RC=${FOO_RC:-0} +echo "$FOO $* # rc=$FOO_RC" 1>>"$FOO.log.$FOO_SESSION" +ACTION=$1 +shift 1 +if [[ $ACTION != 'put' ]]; then + echo "foo: $ACTION: unknown action" >&2 + exit 1 +fi +TARGET=${1#foo://} +mkdir -p $(dirname foo/$TARGET) +shift 1 +rsync -a "$@" foo/$TARGET +exit $FOO_RC diff --git a/t/rose-task-run/48-app-arch/flow.cylc b/t/rose-task-run/48-app-arch/flow.cylc new file mode 100644 index 0000000000..0787739379 --- /dev/null +++ b/t/rose-task-run/48-app-arch/flow.cylc @@ -0,0 +1,49 @@ +#!jinja2 +{% set bad_nums=["1", "2", "3", "4", "5", "6", "7", "8", "9"] %} +[cylc] + UTC mode=True + [[events]] + abort on timeout=True + timeout=PT1M +[scheduling] + initial cycle point=20130101 + final cycle point=20130102 + [[dependencies]] + [[[T12]]] + graph=""" +{% for bad_num in bad_nums %} +install => archive_bad_{{bad_num}} +{% endfor %} +""" + [[[T00, T12]]] + graph=""" +install => archive +archive[-PT12H] => archive +""" + +[runtime] + [[root]] + script=""" +RC=0 +rose task-run --debug || RC=$? +""" + [[[job]]] + execution retry delays=PT1S + execution time limit=PT1M + [[archive]] + post-script=""" +{ +sqlite3 .rose-arch.db 'SELECT * FROM targets;' | sort +sqlite3 .rose-arch.db 'SELECT * FROM sources;' | sort +} >rose-arch-db-$CYLC_TASK_TRY_NUMBER.out +((RC==0)) +""" + [[ARCHIVE_BAD]] + script = """rose task-run || true; RC=$?""" + [[[job]]] + execution retry delays=P0Y +{% for bad_num in bad_nums %} + [[archive_bad_{{bad_num}}]] + inherit=ARCHIVE_BAD +{% endfor %} + [[install]] diff --git a/t/rose-task-run/48-app-arch/rose-suite.conf b/t/rose-task-run/48-app-arch/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 From b1ad44d34d39c7849f3aa6d64187066711284c78 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Wed, 30 Apr 2025 12:59:11 +0100 Subject: [PATCH 02/17] now with added flake8 compliance add myself to Contributors --- metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 1d59304898..17bf4287f2 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -25,8 +25,12 @@ class RoseArchTarGzip: """Compress archive sources in tar.""" - SCHEMES = ["pax", "pax.gz", "pax.zst", "pax.xz", "tar", "tar.gz", "tgz", "tar.zst", "tar.xz", "txz"] - SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, "pax.gz": tarfile.PAX_FORMAT, "pax.zst": tarfile.PAX_FORMAT, "pax.xz": tarfile.PAX_FORMAT} + SCHEMES = ["pax", "pax.gz", "pax.zst", "pax.xz", + "tar", "tar.gz", "tgz", "tar.zst", "tar.xz", "txz"] + SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, + "pax.gz": tarfile.PAX_FORMAT, + "pax.zst": tarfile.PAX_FORMAT, + "pax.xz": tarfile.PAX_FORMAT} GZIP_EXTS = ["pax.gz", "tar.gz", "tgz"] ZSTD_EXTS = ["pax.zst", "tar.zst"] XZ_EXTS = ["pax.xz", "tar.xz", "txz"] From 31b6d0ba98092a038377bc3b7e3e076f94443618 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Wed, 30 Apr 2025 14:06:51 +0100 Subject: [PATCH 03/17] Add line after me in Contributors Faithfully reproduce behaviour of gzip/xz with zstd by removing the original file upon successful compression. (add --rm to command line) --- CONTRIBUTING.md | 2 ++ metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py | 2 +- metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d1581f22d..a43c0e3b43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,8 @@ below: - Dimitrios Theodorakis (Met Office, UK) - Joseph Abram (Met Office, UK) - James Frost (Met Office, UK) + - David Rundle (Met Office, UK) + (All contributors are identifiable with email addresses in the version control diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 17bf4287f2..802e948324 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -83,7 +83,7 @@ def compress_sources(self, target, work_dir): ) os.close(fdsec) target.work_source_path = zst_name - command = "zstd -c '%s' >'%s'" % (tar_name, zst_name) + command = "zstd --rm -c '%s' >'%s'" % (tar_name, zst_name) self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py index 5c659ed12a..3fde58322e 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -44,6 +44,6 @@ def compress_sources(self, target, work_dir): self.app_runner.fs_util.dirname(work_path_zst) ) - command = "zstd -c '%s' >'%s'" % (source.path, work_path_zst) + command = "zstd --rm -c '%s' >'%s'" % (source.path, work_path_zst) self.app_runner.popen.run_simple(command, shell=True) source.path = work_path_zst From 525a6f34961c20cb83a9079aa1ebdf86f67b2378 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Wed, 30 Apr 2025 15:19:08 +0100 Subject: [PATCH 04/17] added support for multithreaded compression (for zstd only) controlled by a keyword compress-cores (default 1 implies off) updated test 46 where it is explicitly set to 1 core. added test 49-app-arch-zstd-mp where compress-cores=0 (auto) is tested and compress-cores=4 is tested. --- metomi/rose/apps/rose_arch.py | 12 +++- .../rose_arch_compressions/rose_arch_tar.py | 4 +- .../rose_arch_compressions/rose_arch_zstd.py | 4 +- t/rose-task-run/46-app-arch-zstd.t | 3 +- t/rose-task-run/46-app-arch-zstd/flow.cylc | 4 +- t/rose-task-run/49-app-arch-zstd-mp.t | 59 +++++++++++++++++++ .../app/archive/rose-app.conf | 19 ++++++ t/rose-task-run/49-app-arch-zstd-mp/flow.cylc | 21 +++++++ .../49-app-arch-zstd-mp/flow.cylc~ | 21 +++++++ .../49-app-arch-zstd-mp/rose-suite.conf | 0 10 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 t/rose-task-run/49-app-arch-zstd-mp.t create mode 100644 t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf create mode 100644 t/rose-task-run/49-app-arch-zstd-mp/flow.cylc create mode 100644 t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ create mode 100644 t/rose-task-run/49-app-arch-zstd-mp/rose-suite.conf diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index 09dd9ccab9..e63e03a108 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -314,6 +314,15 @@ def _run_target_setup( ) ) target.status = target.ST_BAD + + target.compress_cores = self._get_conf(config, t_node, "compress-cores", default="1") + if not target.compress_cores.isdigit() or int(target.compress_cores) < 0: + raise ConfigValueError( + [t_key, "compress-cores"], + target.compress_cores, + ValueError("compress-cores must be a 0 (automatic) or a positive integer"), + ) + rename_format = self._get_conf(config, t_node, "rename-format") if rename_format: rename_parser_str = self._get_conf(config, t_node, "rename-parser") @@ -398,7 +407,8 @@ def _run_target_update(cls, dao, app_runner, compress_manager, target): # Compress sources if target.compress_scheme: handler = compress_manager.get_handler(target.compress_scheme) - handler.compress_sources(target, work_dir) + compress_args = {"cores": target.compress_cores} + handler.compress_sources(target, work_dir, **compress_args) times[1] = time() # transformed time # Run archive command sources = [] diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 802e948324..20576f6b23 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -38,7 +38,7 @@ class RoseArchTarGzip: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, cores="1"): """Create a tar archive of all files in target. Use work_dir to dump results. @@ -83,7 +83,7 @@ def compress_sources(self, target, work_dir): ) os.close(fdsec) target.work_source_path = zst_name - command = "zstd --rm -c '%s' >'%s'" % (tar_name, zst_name) + command = "zstd --rm -T%s -c '%s' >'%s'" % (cores, tar_name, zst_name) self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py index 3fde58322e..4c73e5b39f 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -29,7 +29,7 @@ class RoseArchZstd: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, cores="1"): """zstd each source in target. Use work_dir to dump results. @@ -44,6 +44,6 @@ def compress_sources(self, target, work_dir): self.app_runner.fs_util.dirname(work_path_zst) ) - command = "zstd --rm -c '%s' >'%s'" % (source.path, work_path_zst) + command = "zstd --rm -T%s -c '%s' >'%s'" % (cores, source.path, work_path_zst) self.app_runner.popen.run_simple(command, shell=True) source.path = work_path_zst diff --git a/t/rose-task-run/46-app-arch-zstd.t b/t/rose-task-run/46-app-arch-zstd.t index 22d71fd404..bc717397b5 100755 --- a/t/rose-task-run/46-app-arch-zstd.t +++ b/t/rose-task-run/46-app-arch-zstd.t @@ -53,6 +53,5 @@ file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' ./archive.d/2016.txt.zst ./archive.d/whatever.tar.zst __FIND__ -#------------------------------------------------------------------------------- -purge + exit 0 diff --git a/t/rose-task-run/46-app-arch-zstd/flow.cylc b/t/rose-task-run/46-app-arch-zstd/flow.cylc index fe8e475a31..9666f3574a 100644 --- a/t/rose-task-run/46-app-arch-zstd/flow.cylc +++ b/t/rose-task-run/46-app-arch-zstd/flow.cylc @@ -11,11 +11,11 @@ [runtime] [[archive]] script=""" -zstd -f - <<<'MMXVI' >'2016.txt.zst' +zstd --rm -T1 -f - <<<'MMXVI' >'2016.txt.zst' mkdir 'whatever' echo "I don't care." >'whatever/idontcare.txt' echo "You may be right." >'whatever/youmayberight.txt' -tar -I zstd -cf 'whatever.tar.zst' 'whatever' +tar -I "zstd --rm -T1" -cf 'whatever.tar.zst' 'whatever' rm -fr 'whatever' rose task-run --debug """ diff --git a/t/rose-task-run/49-app-arch-zstd-mp.t b/t/rose-task-run/49-app-arch-zstd-mp.t new file mode 100644 index 0000000000..eba331e14a --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp.t @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.zst +./archive.d/whatever.tar.zst +__FIND__ +TEST_KEY="${TEST_KEY_BASE}-zstd-command" +#------------------------------------------------------------------------------- +purge +exit 0 \ No newline at end of file diff --git a/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf new file mode 100644 index 0000000000..35b91da80d --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf @@ -0,0 +1,19 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.zst] +compress=zstd +compress-cores=4 +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.zst + +[arch:archive.d/whatever.tar.zst] +compress-cores=0 +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.zst diff --git a/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc new file mode 100644 index 0000000000..7551ffebda --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +zstd --rm -T4 -f - <<<'MMXVI' >'2016.txt.zst' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I "zstd --rm -T0" -cf 'whatever.tar.zst' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ new file mode 100644 index 0000000000..9666f3574a --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +zstd --rm -T1 -f - <<<'MMXVI' >'2016.txt.zst' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I "zstd --rm -T1" -cf 'whatever.tar.zst' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/49-app-arch-zstd-mp/rose-suite.conf b/t/rose-task-run/49-app-arch-zstd-mp/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 From cadee344d6168f8c0b59d00c2af924ed718655e0 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Wed, 30 Apr 2025 15:23:54 +0100 Subject: [PATCH 05/17] remove extraneous backup file --- .../49-app-arch-zstd-mp/flow.cylc~ | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ diff --git a/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ deleted file mode 100644 index 9666f3574a..0000000000 --- a/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc~ +++ /dev/null @@ -1,21 +0,0 @@ -#!jinja2 -[cylc] - UTC mode=True - [[events]] - timeout=PT1M - abort on timeout=True -[scheduling] - [[dependencies]] - graph=archive - -[runtime] - [[archive]] - script=""" -zstd --rm -T1 -f - <<<'MMXVI' >'2016.txt.zst' -mkdir 'whatever' -echo "I don't care." >'whatever/idontcare.txt' -echo "You may be right." >'whatever/youmayberight.txt' -tar -I "zstd --rm -T1" -cf 'whatever.tar.zst' 'whatever' -rm -fr 'whatever' -rose task-run --debug -""" From 020099ae7ecd602c5971966afe551ffb31421a75 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Thu, 1 May 2025 09:34:39 +0100 Subject: [PATCH 06/17] flake8 is super cool and clean and makes us write better code --- metomi/rose/apps/rose_arch.py | 9 ++++++--- metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py | 3 ++- .../rose/apps/rose_arch_compressions/rose_arch_zstd.py | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index e63e03a108..2d33c8c58f 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -315,12 +315,15 @@ def _run_target_setup( ) target.status = target.ST_BAD - target.compress_cores = self._get_conf(config, t_node, "compress-cores", default="1") - if not target.compress_cores.isdigit() or int(target.compress_cores) < 0: + target.compress_cores = self._get_conf(config, t_node, + "compress-cores", default="1") + if not target.compress_cores.isdigit() or + int(target.compress_cores) < 0: raise ConfigValueError( [t_key, "compress-cores"], target.compress_cores, - ValueError("compress-cores must be a 0 (automatic) or a positive integer"), + ValueError("compress-cores must be a 0 (automatic) or + a positive integer") ) rename_format = self._get_conf(config, t_node, "rename-format") diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 20576f6b23..251ea6a9ff 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -83,7 +83,8 @@ def compress_sources(self, target, work_dir, cores="1"): ) os.close(fdsec) target.work_source_path = zst_name - command = "zstd --rm -T%s -c '%s' >'%s'" % (cores, tar_name, zst_name) + command = "zstd --rm -T%s -c '%s' >'%s'" % + (cores, tar_name, zst_name) self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py index 4c73e5b39f..4962a26a80 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -44,6 +44,7 @@ def compress_sources(self, target, work_dir, cores="1"): self.app_runner.fs_util.dirname(work_path_zst) ) - command = "zstd --rm -T%s -c '%s' >'%s'" % (cores, source.path, work_path_zst) + command = "zstd --rm -T%s -c '%s' >'%s'" % + (cores, source.path, work_path_zst) self.app_runner.popen.run_simple(command, shell=True) source.path = work_path_zst From 4a4c65822babdbc85db54c4e09a7fb29591873a5 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Fri, 2 May 2025 12:10:20 +0100 Subject: [PATCH 07/17] who knew python was so picky about whitespace. Added myself to the .mailmap --- .mailmap | 1 + metomi/rose/apps/rose_arch.py | 8 ++++---- metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py | 3 +-- metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py | 4 +--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.mailmap b/.mailmap index 7d49e391cb..165eb4ba5d 100644 --- a/.mailmap +++ b/.mailmap @@ -39,3 +39,4 @@ Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> <17361029+marshrossney@users.noreply.github.com> Joseph Abram J-J-Abram Joseph Abram J-J-Abram <98320699+J-J-Abram@users.noreply.github.com> +David Rundle davidrundle \ No newline at end of file diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index 2d33c8c58f..a159fdae39 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -315,14 +315,14 @@ def _run_target_setup( ) target.status = target.ST_BAD - target.compress_cores = self._get_conf(config, t_node, + target.compress_cores = self._get_conf(config, t_node, "compress-cores", default="1") - if not target.compress_cores.isdigit() or - int(target.compress_cores) < 0: + if not target.compress_cores.isdigit() or \ + int(target.compress_cores) < 0: raise ConfigValueError( [t_key, "compress-cores"], target.compress_cores, - ValueError("compress-cores must be a 0 (automatic) or + ValueError("compress-cores must be a 0 (automatic) or \ a positive integer") ) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 251ea6a9ff..db42b175b7 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -83,8 +83,7 @@ def compress_sources(self, target, work_dir, cores="1"): ) os.close(fdsec) target.work_source_path = zst_name - command = "zstd --rm -T%s -c '%s' >'%s'" % - (cores, tar_name, zst_name) + command = f"zstd --rm -T{cores} -c '{tar_name}' >'{zst_name}'" self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py index 4962a26a80..28906fef1c 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -43,8 +43,6 @@ def compress_sources(self, target, work_dir, cores="1"): self.app_runner.fs_util.makedirs( self.app_runner.fs_util.dirname(work_path_zst) ) - - command = "zstd --rm -T%s -c '%s' >'%s'" % - (cores, source.path, work_path_zst) + command = f"zstd --rm -T{cores} -c {source.path} > {work_path_zst}" self.app_runner.popen.run_simple(command, shell=True) source.path = work_path_zst From 7c946fd36fb81037e3690a7ea648168a01676922 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Fri, 2 May 2025 12:26:00 +0100 Subject: [PATCH 08/17] oof. Who even knows their own github username? --- .mailmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 165eb4ba5d..bb833bc2d3 100644 --- a/.mailmap +++ b/.mailmap @@ -39,4 +39,4 @@ Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> <17361029+marshrossney@users.noreply.github.com> Joseph Abram J-J-Abram Joseph Abram J-J-Abram <98320699+J-J-Abram@users.noreply.github.com> -David Rundle davidrundle \ No newline at end of file +David Rundle david-rundle \ No newline at end of file From 800f6467b37250231c31482455e9ba11ecc0bd77 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Fri, 2 May 2025 13:18:10 +0100 Subject: [PATCH 09/17] First bash at documentation --- sphinx/api/built-in/rose_arch.rst | 84 +++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 21 deletions(-) diff --git a/sphinx/api/built-in/rose_arch.rst b/sphinx/api/built-in/rose_arch.rst index 393bd1681f..d58acf75ce 100644 --- a/sphinx/api/built-in/rose_arch.rst +++ b/sphinx/api/built-in/rose_arch.rst @@ -32,7 +32,7 @@ The application provides some useful functionalities: source files and the return code of archive command. In a retry, it would only redo targets that did not succeed in the previous attempts. * Rename source files. -* Tar-Gzip or Gzip source files before sending them to the archive. +* Tar and/or compress source files before sending them to the archive. Invocation @@ -124,8 +124,8 @@ on: [arch:(black-box/)] source=cats.txt dogs.txt -Zipping files -^^^^^^^^^^^^^ +Compressing files +^^^^^^^^^^^^^^^^^ There are multiple ways of specifying that you want your archive to be compressed: @@ -147,9 +147,9 @@ not recognized by rose arch as an extension to be compressed.) For more details see :rose:conf:`rose_arch[arch]compress` -Zipping directories -^^^^^^^^^^^^^^^^^^^ -You can tar and zip entire directories - as with single files Rose Arch will +Compressing directories +^^^^^^^^^^^^^^^^^^^^^^^ +You can tar and compress entire directories - as with single files Rose Arch will attempt to infer archive and compression from ``[arch:TARGET.extension]`` if it can: @@ -206,6 +206,22 @@ with names in the form ``data_001.txt``: rename-parser=^//some//path//data_(?P[0-9]{3})(?P.*)$ rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s +Using Multiple Cores for Compression (zstd only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When using `zstd`, the number of CPU cores to use for compression is controlled using the +`compress-cores` keyword. This is useful for large files +where multi-threaded compression can significantly improve throughput. + +.. code-block:: rose + + [arch:large-data.tar.zst] + compress=zst + compress-cores=8 + source=large-data/* + +In this example, the `zstd` will use 8 CPU cores. + Output ------ @@ -245,7 +261,6 @@ taken to run the archive command and the return code of the archive command. For a source line, the third column contains the original name of the source. - Configuration ------------- @@ -262,7 +277,7 @@ Configuration and ``%(target)s`` for substitution of the sources and the target respectively. - .. rose:conf:: compress=pax|tar|pax.gz|tar.gz|tgz|gz + .. rose:conf:: compress=pax|tar|pax.gz|tar.gz|tgz|gz|pax.xz|tar.xz|txz|xz|pax.zst|tar.zst|zst If specified, compress source files scheme before sending them to the archive. If not set Rose Arch will attempt to set a compression scheme @@ -272,19 +287,46 @@ Configuration Each compression scheme works slightly differently: - +------------------+-----------------------------------------------+ - |Compression Scheme|Behaviour | - +------------------+-----------------------------------------------+ - |``pax`` or ``tar``|Sources will be placed in a TAR archive before | - | |being sent to the target. | - +------------------+-----------------------------------------------+ - |``pax.gz``, |Sources will be placed in a TAR-GZIP file | - |``tar.gz`` or |before being sent to the target. | - |``tgz`` | | - +------------------+-----------------------------------------------+ - |``gz`` |Each source file will be compressed by GZIP | - | |before being sent to the target. | - +------------------+-----------------------------------------------+ + +-----------------------+-----------------------------------------------+ + |Compression Scheme |Behaviour | + +-----------------------+-----------------------------------------------+ + |``pax`` or ``tar`` |Sources will be placed in a TAR archive before | + | |being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``pax.{gz,xz,zst}`` or |Sources will be placed in a TAR archive and be | + |``tar.{gz,xz,zst}`` or |compressed using the corresponding compressor | + |or ``tgz`` or ``txz`` |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``gz`` |Each source file will be compressed by gzip | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``xz`` |Each source file will be compressed by xz | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``zstd`` |Each source file will be compressed by zstd | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + + .. rose:conf:: compress-cores=0|1|2|... + + Specify the number of CPU cores to use for compression. This setting + is optional and defaults to `1` (single-threaded compression). + + * `0`: Let the compression tool automatically determine the number of + cores to use. + * A positive integer: Specifies the exact number of cores to use for + compression. + + This setting is currently only supported by `zstd`. + + Example: + + .. code-block:: rose + + [arch:example.tar.zst] + compress=tar.zst + compress-cores=4 + source=example/* .. rose:conf:: rename-format From cc009c54197e31dd6cb112a27ab0bcbb1d46ce2f Mon Sep 17 00:00:00 2001 From: David Rundle Date: Fri, 2 May 2025 13:24:20 +0100 Subject: [PATCH 10/17] pass local contributorcheck --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a43c0e3b43..0b680e706e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,6 @@ below: - Joseph Abram (Met Office, UK) - James Frost (Met Office, UK) - David Rundle (Met Office, UK) - (All contributors are identifiable with email addresses in the version control From fc9cca8a3b2bd494f6af2453b8bb113dca70cb23 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Tue, 6 May 2025 11:58:53 +0100 Subject: [PATCH 11/17] Update .mailmap --- .mailmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index bb833bc2d3..2c391c31d0 100644 --- a/.mailmap +++ b/.mailmap @@ -39,4 +39,4 @@ Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> <17361029+marshrossney@users.noreply.github.com> Joseph Abram J-J-Abram Joseph Abram J-J-Abram <98320699+J-J-Abram@users.noreply.github.com> -David Rundle david-rundle \ No newline at end of file +David Rundle david-rundle <37152257+david-rundle@users.noreply.github.com> From 14660d7493ae5c6d233e9ea1ad1a2e6623763c0f Mon Sep 17 00:00:00 2001 From: david-rundle <37152257+david-rundle@users.noreply.github.com> Date: Tue, 6 May 2025 12:49:30 +0100 Subject: [PATCH 12/17] Update rose_arch.rst replace compress-cores with compress-threads and update documentation to clarify how -T controls operation. Add guidance about how to use multi-threading --- sphinx/api/built-in/rose_arch.rst | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sphinx/api/built-in/rose_arch.rst b/sphinx/api/built-in/rose_arch.rst index d58acf75ce..7e4de047be 100644 --- a/sphinx/api/built-in/rose_arch.rst +++ b/sphinx/api/built-in/rose_arch.rst @@ -206,21 +206,21 @@ with names in the form ``data_001.txt``: rename-parser=^//some//path//data_(?P[0-9]{3})(?P.*)$ rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s -Using Multiple Cores for Compression (zstd only) +Using Multiple threads for Compression (zstd only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When using `zstd`, the number of CPU cores to use for compression is controlled using the -`compress-cores` keyword. This is useful for large files +When using `zstd`, the number of threads to use for compression is controlled using the +`compress-threads` keyword. This is useful for large files where multi-threaded compression can significantly improve throughput. .. code-block:: rose [arch:large-data.tar.zst] compress=zst - compress-cores=8 + compress-threads=8 source=large-data/* -In this example, the `zstd` will use 8 CPU cores. +In this example, the `zstd` will use 8 threads. Output ------ @@ -307,15 +307,17 @@ Configuration | |before being sent to the target. | +-----------------------+-----------------------------------------------+ - .. rose:conf:: compress-cores=0|1|2|... + .. rose:conf:: compress-threads=0|1|2|... - Specify the number of CPU cores to use for compression. This setting + Specify the number of threads to use for compression. This setting is optional and defaults to `1` (single-threaded compression). * `0`: Let the compression tool automatically determine the number of - cores to use. - * A positive integer: Specifies the exact number of cores to use for - compression. + threads to use which is typically equal to the number of detected physical CPU cores + (and should be used with caution on shared resources). + + * A positive integer: Specifies the exact number of threads to use for compression. + It is not recommended to exceed the number of physical CPU cores on the target resource. This setting is currently only supported by `zstd`. @@ -325,7 +327,7 @@ Configuration [arch:example.tar.zst] compress=tar.zst - compress-cores=4 + compress-threads=4 source=example/* .. rose:conf:: rename-format From a64df95fe0b8d6184013c9686956cba31e0c67c7 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Tue, 6 May 2025 13:03:11 +0100 Subject: [PATCH 13/17] update code to use the term 'threads' instead of 'cores' throughout to clarify how -T multi-processing works --- metomi/rose/apps/rose_arch.py | 17 +++++++++-------- .../rose_arch_compressions/rose_arch_tar.py | 4 ++-- .../rose_arch_compressions/rose_arch_zstd.py | 6 ++++-- .../app/archive/rose-app.conf | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index a159fdae39..8045b7abf3 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -315,14 +315,15 @@ def _run_target_setup( ) target.status = target.ST_BAD - target.compress_cores = self._get_conf(config, t_node, - "compress-cores", default="1") - if not target.compress_cores.isdigit() or \ - int(target.compress_cores) < 0: + target.compress_threads = self._get_conf(config, t_node, + "compress-threads", + default="1") + if not target.compress_threads.isdigit() or \ + int(target.compress_threads) < 0: raise ConfigValueError( - [t_key, "compress-cores"], - target.compress_cores, - ValueError("compress-cores must be a 0 (automatic) or \ + [t_key, "compress-threads"], + target.compress_threads, + ValueError("compress-threads must be a 0 (automatic) or \ a positive integer") ) @@ -410,7 +411,7 @@ def _run_target_update(cls, dao, app_runner, compress_manager, target): # Compress sources if target.compress_scheme: handler = compress_manager.get_handler(target.compress_scheme) - compress_args = {"cores": target.compress_cores} + compress_args = {"threads": target.compress_threads} handler.compress_sources(target, work_dir, **compress_args) times[1] = time() # transformed time # Run archive command diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index db42b175b7..2f00a03783 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -38,7 +38,7 @@ class RoseArchTarGzip: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir, cores="1"): + def compress_sources(self, target, work_dir, threads="1"): """Create a tar archive of all files in target. Use work_dir to dump results. @@ -83,7 +83,7 @@ def compress_sources(self, target, work_dir, cores="1"): ) os.close(fdsec) target.work_source_path = zst_name - command = f"zstd --rm -T{cores} -c '{tar_name}' >'{zst_name}'" + command = f"zstd --rm -T{threads} -c '{tar_name}' >'{zst_name}'" self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py index 28906fef1c..7ad8a45028 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -29,7 +29,7 @@ class RoseArchZstd: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir, cores="1"): + def compress_sources(self, target, work_dir, threads="1"): """zstd each source in target. Use work_dir to dump results. @@ -43,6 +43,8 @@ def compress_sources(self, target, work_dir, cores="1"): self.app_runner.fs_util.makedirs( self.app_runner.fs_util.dirname(work_path_zst) ) - command = f"zstd --rm -T{cores} -c {source.path} > {work_path_zst}" + command = \ + f"zstd --rm -T{threads} -c {source.path} > {work_path_zst}" + self.app_runner.popen.run_simple(command, shell=True) source.path = work_path_zst diff --git a/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf index 35b91da80d..a4a65ac5b1 100644 --- a/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf +++ b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf @@ -9,11 +9,11 @@ target-prefix=$ROSE_SUITE_DIR/share/backup/ [arch:archive.d/2016.txt.zst] compress=zstd -compress-cores=4 +compress-threads=4 source-prefix=work/1/$ROSE_TASK_NAME/ source=2016.txt.zst [arch:archive.d/whatever.tar.zst] -compress-cores=0 +compress-threads=0 source-prefix=work/1/$ROSE_TASK_NAME/ source=whatever.tar.zst From d8cb802f7fa2df8dfbbbb9d652f5b332f8577129 Mon Sep 17 00:00:00 2001 From: David Rundle Date: Tue, 6 May 2025 13:22:36 +0100 Subject: [PATCH 14/17] commit reviewer comments and fix sphinx documentation --- metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py | 4 ++-- sphinx/api/built-in/rose_arch.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 2f00a03783..c6d8e60970 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -77,7 +77,7 @@ def compress_sources(self, target, work_dir, threads="1"): self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) - if target.compress_scheme in self.ZSTD_EXTS: + elif target.compress_scheme in self.ZSTD_EXTS: fdsec, zst_name = mkstemp( suffix="." + target.compress_scheme, dir=work_dir ) @@ -87,7 +87,7 @@ def compress_sources(self, target, work_dir, threads="1"): self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) - if target.compress_scheme in self.XZ_EXTS: + elif target.compress_scheme in self.XZ_EXTS: fdsec, xz_name = mkstemp( suffix="." + target.compress_scheme, dir=work_dir ) diff --git a/sphinx/api/built-in/rose_arch.rst b/sphinx/api/built-in/rose_arch.rst index 7e4de047be..4bcd655fbb 100644 --- a/sphinx/api/built-in/rose_arch.rst +++ b/sphinx/api/built-in/rose_arch.rst @@ -206,8 +206,8 @@ with names in the form ``data_001.txt``: rename-parser=^//some//path//data_(?P[0-9]{3})(?P.*)$ rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s -Using Multiple threads for Compression (zstd only) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using multiple threads for compression (zstd only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When using `zstd`, the number of threads to use for compression is controlled using the `compress-threads` keyword. This is useful for large files From a0b86e5bf27e10156f556c82d8f1bdac7e79d596 Mon Sep 17 00:00:00 2001 From: Tim Pillinger Date: Tue, 6 May 2025 15:39:37 +0100 Subject: [PATCH 15/17] add an integration test --- metomi/rose/app_run.py | 1 - .../rose_arch_compressions/rose_arch_gzip.py | 5 +- .../rose_arch_compressions/rose_arch_xz.py | 2 +- .../tests/apps/rose_arch/test_rose_arch.py | 78 +++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 metomi/rose/tests/apps/rose_arch/test_rose_arch.py diff --git a/metomi/rose/app_run.py b/metomi/rose/app_run.py index ce68804beb..f3f297ce2a 100644 --- a/metomi/rose/app_run.py +++ b/metomi/rose/app_run.py @@ -378,7 +378,6 @@ def __init__(self, *args, **kwargs): def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" - # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py index 0a5e1770cc..0d1df676df 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py @@ -29,12 +29,15 @@ class RoseArchGzip: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, threads="1"): """Gzip each source in target. Use work_dir to dump results. """ + if threads != "1": + raise NotImplementedError("Gzip does not support multi-threading") + for source in target.sources.values(): if source.path.endswith("." + target.compress_scheme): continue # assume already done diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py index 97a455bdf7..3bc6d37ee8 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py @@ -29,7 +29,7 @@ class RoseArchXz: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, threads="1"): """xz each source in target. Use work_dir to dump results. diff --git a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py new file mode 100644 index 0000000000..fbdd4924a5 --- /dev/null +++ b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py @@ -0,0 +1,78 @@ +from metomi.rose.task_run import main as rose_task_run +from metomi.rose.task_run import TaskRunner +from metomi.rose.reporter import Reporter + +import pytest +from textwrap import dedent +from types import SimpleNamespace + +ZIP_METHODS = ["nozip", "gunzipme", "targunzipme", "zstdme", "xzme"] + + +@pytest.fixture +def setup_rose_arch_env(monkeypatch): + """Mock the environment variables needed for a rose_arch task. + """ + monkeypatch.setenv("CYLC_WORKFLOW_ID", "foo") + monkeypatch.setenv("CYLC_TASK_ID", "bar") + monkeypatch.setenv("CYLC_TASK_NAME", "baz") + monkeypatch.setenv("CYLC_TASK_CYCLE_POINT", "qux") + monkeypatch.setenv("CYLC_TASK_LOG_ROOT", "woteva") + + +def test_rose_arch(tmp_path, setup_rose_arch_env): + """Test the rose_arch app against different compression methods. + + Possible future enhancements: + might be nice to abstract the setup so that we can + test other aspects of rose_arch. + """ + (tmp_path / 'archive').mkdir() + (tmp_path / 'source').mkdir(parents=True, exist_ok=True) + for source in ZIP_METHODS: + (tmp_path / 'source' / source).write_text( + "The Quick Brown Fox Jumps Over The Lazy Dog.\n") + + (tmp_path / 'rose-app.conf').write_text( + dedent( + f""" + mode=rose_arch + + [arch] + command-format=cp %(sources)s %(target)s + target-prefix={tmp_path}/archive/ + source-prefix={tmp_path}/source/ + + [arch:nozip] + source=nozip + + [arch:gunzipme.gz] + source=gunzipme + + [arch:targunzipme.tar.gz] + source=targunzipme + + [arch:zstdme.zst] + source=zstdme + + [arch:xzme.xz] + source=xzme + """ + ) + ) + + # Create a crude options object: + opts = SimpleNamespace() + for key in TaskRunner.OPTIONS: + setattr(opts, key, None) + opts.conf_dir = str(tmp_path) + + # Run our task: + runner = TaskRunner(event_handler=Reporter()) + runner(opts, []) + + assert (tmp_path / 'archive' / 'nozip').exists() + assert (tmp_path / 'archive' / 'gunzipme.gz').exists() + assert (tmp_path / 'archive' / 'targunzipme.tar.gz').exists() + assert (tmp_path / 'archive' / 'zstdme.zst').exists() + assert (tmp_path / 'archive' / 'xzme.xz').exists() From d1a0ff615da4b5205a086fc1cd4b8dd5de88df97 Mon Sep 17 00:00:00 2001 From: Tim Pillinger Date: Tue, 6 May 2025 16:52:36 +0100 Subject: [PATCH 16/17] Add a couple of functional tests for rose-arch, and made some suggestions about code - notably that a failure should appear if thread != 1 if the compression app doesn't support that refactor tests to provide nicer test architecture for other tests. flake8 --- metomi/rose/apps/rose_arch.py | 13 +- .../rose_arch_compressions/rose_arch_tar.py | 3 + .../rose_arch_compressions/rose_arch_xz.py | 4 + .../tests/apps/rose_arch/test_rose_arch.py | 198 +++++++++++++----- 4 files changed, 158 insertions(+), 60 deletions(-) diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index 8045b7abf3..040ef773c3 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -291,6 +291,7 @@ def _run_target_setup( target.sources[checksum] = RoseArchSource( checksum, name, path ) + if not target.sources: if is_compulsory_target: target.status = target.ST_BAD @@ -318,13 +319,17 @@ def _run_target_setup( target.compress_threads = self._get_conf(config, t_node, "compress-threads", default="1") - if not target.compress_threads.isdigit() or \ - int(target.compress_threads) < 0: + if ( + not target.compress_threads.isdigit() + or int(target.compress_threads) < 0 + ): raise ConfigValueError( [t_key, "compress-threads"], target.compress_threads, - ValueError("compress-threads must be a 0 (automatic) or \ - a positive integer") + ValueError( + "compress-threads must be a 0 (automatic) or" + " a positive integer" + ) ) rename_format = self._get_conf(config, t_node, "rename-format") diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index c6d8e60970..6d6beab888 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -44,6 +44,9 @@ def compress_sources(self, target, work_dir, threads="1"): Use work_dir to dump results. """ + if threads != "1": + raise NotImplementedError("xz does not support multi-threading") + sources = list(target.sources.values()) if len(sources) == 1 and sources[0].path.endswith( "." + target.compress_scheme diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py index 3bc6d37ee8..d086e324d5 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py @@ -35,6 +35,10 @@ def compress_sources(self, target, work_dir, threads="1"): Use work_dir to dump results. """ + + if threads != "1": + raise NotImplementedError("xz does not support multi-threading") + for source in target.sources.values(): if source.path.endswith("." + target.compress_scheme): continue # assume already done diff --git a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py index fbdd4924a5..7b124a38f1 100644 --- a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py +++ b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py @@ -1,78 +1,164 @@ -from metomi.rose.task_run import main as rose_task_run -from metomi.rose.task_run import TaskRunner -from metomi.rose.reporter import Reporter + +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Integration tests for Rose Arch.""" import pytest from textwrap import dedent from types import SimpleNamespace +from typing import Dict + + +from metomi.rose.task_run import TaskRunner +from metomi.rose.reporter import Reporter + -ZIP_METHODS = ["nozip", "gunzipme", "targunzipme", "zstdme", "xzme"] +ZIP_METHODS = { + "nozip": "nozip", + "gunzipme": "gunzipme.gz", + "targunzipme": "targunzipme.tar.gz", + "zstdme": "zstdme.zst", + "xzme": "xzme.xz" +} @pytest.fixture -def setup_rose_arch_env(monkeypatch): +def setup_rose_arch_env(monkeypatch, tmp_path): """Mock the environment variables needed for a rose_arch task. """ + tmp_path_str = str(tmp_path) monkeypatch.setenv("CYLC_WORKFLOW_ID", "foo") monkeypatch.setenv("CYLC_TASK_ID", "bar") monkeypatch.setenv("CYLC_TASK_NAME", "baz") monkeypatch.setenv("CYLC_TASK_CYCLE_POINT", "qux") - monkeypatch.setenv("CYLC_TASK_LOG_ROOT", "woteva") + monkeypatch.setenv("CYLC_TASK_LOG_ROOT", tmp_path_str) + monkeypatch.setenv("ROSE_SUITE_NAME", tmp_path_str) + monkeypatch.setenv("ROSE_SUITE_DIR", tmp_path_str) + monkeypatch.setenv("CYLC_RUN_DIR", tmp_path_str) -def test_rose_arch(tmp_path, setup_rose_arch_env): +@pytest.fixture +def setup_rose_arch_task(tmp_path): + """Set up a rose_arch task with a given config and source files. + """ + def _inner( + config: str, + source_files: Dict[str, str], + ): + (tmp_path / 'archive').mkdir() + (tmp_path / 'source').mkdir() + for source_file in source_files: + (tmp_path / 'source' / source_file).write_text( + source_files[source_file]) + (tmp_path / 'rose-app.conf').write_text(dedent(config)) + return _inner + + +@pytest.fixture +def run_rose_arch_task(tmp_path, setup_rose_arch_env): + """Run a rose_arch task.""" + + def _inner(): + # Create a crude options object: + opts = SimpleNamespace() + for key in TaskRunner.OPTIONS: + setattr(opts, key, None) + opts.conf_dir = str(tmp_path) + + # Run our task: + runner = TaskRunner(event_handler=Reporter()) + runner(opts, []) + return _inner + + +@pytest.mark.parametrize( + "source_file, expected", + [ + ("nozip", "nozip"), + ("gunzipme", "gunzipme.gz"), + ("targunzipme", "targunzipme.tar.gz"), + ("zstdme", "zstdme.zst"), + ("xzme", "xzme.xz") + ] +) +def test_rose_arch( + tmp_path, + setup_rose_arch_task, + run_rose_arch_task, + source_file, + expected +): """Test the rose_arch app against different compression methods. Possible future enhancements: - might be nice to abstract the setup so that we can - test other aspects of rose_arch. + - More checks on the compressed files, ensuring that the compression + has worked as expected. + """ + setup_rose_arch_task( + f""" + mode=rose_arch + + [arch] + command-format=cp %(sources)s %(target)s + target-prefix={tmp_path}/archive/ + source-prefix={tmp_path}/source/ + + [arch:{expected}] + source={source_file} + """, + ZIP_METHODS, + ) + run_rose_arch_task() + assert (tmp_path / 'archive' / expected).exists() + + +@pytest.mark.parametrize( + 'archiver', + [ + 'gz', + 'tar.gz', + 'xz', + ] +) +def test_raises_too_many_threads_err( + tmp_path, setup_rose_arch_task, run_rose_arch_task, archiver +): + """Test that we get an error if an archiver does not support threads, + but thread number (other than 1) is passed in. + """ - (tmp_path / 'archive').mkdir() - (tmp_path / 'source').mkdir(parents=True, exist_ok=True) - for source in ZIP_METHODS: - (tmp_path / 'source' / source).write_text( - "The Quick Brown Fox Jumps Over The Lazy Dog.\n") - - (tmp_path / 'rose-app.conf').write_text( - dedent( - f""" - mode=rose_arch - - [arch] - command-format=cp %(sources)s %(target)s - target-prefix={tmp_path}/archive/ - source-prefix={tmp_path}/source/ - - [arch:nozip] - source=nozip - - [arch:gunzipme.gz] - source=gunzipme - - [arch:targunzipme.tar.gz] - source=targunzipme - - [arch:zstdme.zst] - source=zstdme - - [arch:xzme.xz] - source=xzme - """ - ) + setup_rose_arch_task( + f""" + mode=rose_arch + + [arch] + command-format=gzip %(sources)s %(target)s + target-prefix={tmp_path}/archive/ + source-prefix={tmp_path}/source/ + + [arch:zipme.{archiver}] + source=zipme + compress-threads=2 + """, + {"zipme": "Any old text file"}, ) - # Create a crude options object: - opts = SimpleNamespace() - for key in TaskRunner.OPTIONS: - setattr(opts, key, None) - opts.conf_dir = str(tmp_path) - - # Run our task: - runner = TaskRunner(event_handler=Reporter()) - runner(opts, []) - - assert (tmp_path / 'archive' / 'nozip').exists() - assert (tmp_path / 'archive' / 'gunzipme.gz').exists() - assert (tmp_path / 'archive' / 'targunzipme.tar.gz').exists() - assert (tmp_path / 'archive' / 'zstdme.zst').exists() - assert (tmp_path / 'archive' / 'xzme.xz').exists() + with pytest.raises( + NotImplementedError, + match="does not support multi-threading" + ): + run_rose_arch_task() From c0b61d2d9adeba884d93922bb6b16e5371fa9b19 Mon Sep 17 00:00:00 2001 From: Tim Pillinger Date: Wed, 7 May 2025 17:30:07 +0100 Subject: [PATCH 17/17] * Made some tests pytest style, not unittest. * Changed a test based on this conversation: https://github.com/metomi/rose/pull/2286/files#r258325340 * Fixed a problem cased by a mutable default arg. --- metomi/rose/env.py | 11 +++- .../tests/apps/rose_arch/test_rose_arch.py | 5 +- metomi/rose/tests/test_env.py | 53 ++++++++----------- metomi/rose/tests/test_env_cat.py | 2 - 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/metomi/rose/env.py b/metomi/rose/env.py index 7ffd14d626..7be03c5ec3 100644 --- a/metomi/rose/env.py +++ b/metomi/rose/env.py @@ -107,11 +107,16 @@ def __repr__(self): def env_export(key, value, event_handler=None): """Export an environment variable.""" - if key not in _EXPORTED_ENVS or os.environ.get(key) != value: + if ( + key not in _EXPORTED_ENVS + or os.environb.get(key.encode("UTF-8")) + != value.encode("UTF-8") + ): # N.B. Should be safe, because the list of environment variables is # normally quite small. _EXPORTED_ENVS[key] = value os.environb[key.encode('UTF-8')] = value.encode('UTF-8') + if callable(event_handler): event_handler(EnvExportEvent(key, value)) @@ -134,7 +139,7 @@ def env_var_escape(text, match_mode=None): return ret -def env_var_process(text, unbound=None, match_mode=None, environ=os.environ): +def env_var_process(text, unbound=None, match_mode=None, environ=None): """Substitute environment variables into a string. For each $NAME and ${NAME} in "text", substitute with the value @@ -145,6 +150,8 @@ def env_var_process(text, unbound=None, match_mode=None, environ=os.environ): value of "unbound". """ + if environ is None: + environ = os.environ ret = "" try: tail = text.decode() diff --git a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py index 7b124a38f1..9b2154cf97 100644 --- a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py +++ b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py @@ -41,14 +41,17 @@ def setup_rose_arch_env(monkeypatch, tmp_path): """Mock the environment variables needed for a rose_arch task. """ tmp_path_str = str(tmp_path) + # Avoids leaving rose database and rose-app-run.conf files in your CWD. + monkeypatch.chdir(tmp_path_str) monkeypatch.setenv("CYLC_WORKFLOW_ID", "foo") - monkeypatch.setenv("CYLC_TASK_ID", "bar") + monkeypatch.setenv("CYLC_TASK_ID", "test_rose_arch") monkeypatch.setenv("CYLC_TASK_NAME", "baz") monkeypatch.setenv("CYLC_TASK_CYCLE_POINT", "qux") monkeypatch.setenv("CYLC_TASK_LOG_ROOT", tmp_path_str) monkeypatch.setenv("ROSE_SUITE_NAME", tmp_path_str) monkeypatch.setenv("ROSE_SUITE_DIR", tmp_path_str) monkeypatch.setenv("CYLC_RUN_DIR", tmp_path_str) + yield @pytest.fixture diff --git a/metomi/rose/tests/test_env.py b/metomi/rose/tests/test_env.py index 8ca4f3d1c5..61c32dd44d 100644 --- a/metomi/rose/tests/test_env.py +++ b/metomi/rose/tests/test_env.py @@ -14,39 +14,30 @@ # You should have received a copy of the GNU General Public License # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import os -import unittest - from metomi.rose.env import env_export -class _TestEnvExport(unittest.TestCase): - """Test "env_export" function.""" - - def test_report_new(self): - """Ensure that env_export only reports 1st time or on change.""" - events = [] - env_export("FOO", "foo", events.append) - env_export("FOO", "foo", events.append) - env_export("FOO", "food", events.append) - env_export("FOO", "foot", events.append) - env_export("FOO", "foot", events.append) - event_args = [event.args[1] for event in events] - self.assertEqual(event_args, ["foo", "food", "foot"], "events") - - def test_report_old(self): - """Ensure that env_export only reports 1st time or on change.""" - events = [] - os.environ["BAR"] = "bar" - env_export("BAR", "bar", events.append) - env_export("BAR", "bar", events.append) - env_export("BAR", "bar", events.append) - env_export("BAR", "barley", events.append) - env_export("BAR", "barley", events.append) - env_export("BAR", "barber", events.append) - event_args = [event.args[1] for event in events] - self.assertEqual(event_args, ["bar", "barley", "barber"], "events") +def test_report_new(): + """Ensure that env_export only reports 1st time or on change.""" + events = [] + env_export("FOO", "foo", events.append) + env_export("FOO", "foo", events.append) + env_export("FOO", "food", events.append) + env_export("FOO", "foot", events.append) + env_export("FOO", "foot", events.append) + event_args = [event.args[1] for event in events] + assert event_args == ["foo", "food", "foot"] -if __name__ == "__main__": - unittest.main() +def test_report_old(monkeypatch): + """Ensure that env_export only reports 1st time or on change.""" + events = [] + monkeypatch.setenv("BAR", "bar") + env_export("BAR", "bar", events.append) + env_export("BAR", "bar", events.append) + env_export("BAR", "bar", events.append) + env_export("BAR", "barley", events.append) + env_export("BAR", "barley", events.append) + env_export("BAR", "barber", events.append) + event_args = [event.args[1] for event in events] + assert event_args == ["bar", "barley", "barber"] diff --git a/metomi/rose/tests/test_env_cat.py b/metomi/rose/tests/test_env_cat.py index 2f25fadbc2..285cc331f7 100644 --- a/metomi/rose/tests/test_env_cat.py +++ b/metomi/rose/tests/test_env_cat.py @@ -35,14 +35,12 @@ def test_output_to_file(tmp_path, monkeypatch): outputfile = tmp_path / 'outputfile' inputfile.write_text(r'Hello ${WORLD}') monkeypatch.setenv('WORLD', 'Jupiter') - opts = SimpleNamespace( match_mode=None, output_file=str(outputfile), unbound=None ) args = [str(inputfile)] - assert rose_env_cat(args, opts) is None assert outputfile.read_text() == 'Hello Jupiter'