diff --git a/.mailmap b/.mailmap index e1a4dd1ca0..39adb324c2 100644 --- a/.mailmap +++ b/.mailmap @@ -39,4 +39,5 @@ Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> Joe Marsh Rossney <17361029+jmarshrossney@users.noreply.github.com> <17361029+marshrossney@users.noreply.github.com> Joseph Abram J-J-Abram Joseph Abram J-J-Abram <98320699+J-J-Abram@users.noreply.github.com> +David Rundle david-rundle <37152257+david-rundle@users.noreply.github.com> Christopher Bennett christopher.bennett diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 59c92a3c28..df229b74c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,7 @@ below: - Dimitrios Theodorakis (Met Office, UK) - Joseph Abram (Met Office, UK) - James Frost (Met Office, UK) + - David Rundle (Met Office, UK) - Christopher Bennett (Met Office, UK) diff --git a/metomi/rose/app_run.py b/metomi/rose/app_run.py index ce68804beb..f3f297ce2a 100644 --- a/metomi/rose/app_run.py +++ b/metomi/rose/app_run.py @@ -378,7 +378,6 @@ def __init__(self, *args, **kwargs): def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" - # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) diff --git a/metomi/rose/apps/rose_arch.py b/metomi/rose/apps/rose_arch.py index 09dd9ccab9..040ef773c3 100644 --- a/metomi/rose/apps/rose_arch.py +++ b/metomi/rose/apps/rose_arch.py @@ -291,6 +291,7 @@ def _run_target_setup( target.sources[checksum] = RoseArchSource( checksum, name, path ) + if not target.sources: if is_compulsory_target: target.status = target.ST_BAD @@ -314,6 +315,23 @@ def _run_target_setup( ) ) target.status = target.ST_BAD + + target.compress_threads = self._get_conf(config, t_node, + "compress-threads", + default="1") + if ( + not target.compress_threads.isdigit() + or int(target.compress_threads) < 0 + ): + raise ConfigValueError( + [t_key, "compress-threads"], + target.compress_threads, + ValueError( + "compress-threads must be a 0 (automatic) or" + " a positive integer" + ) + ) + rename_format = self._get_conf(config, t_node, "rename-format") if rename_format: rename_parser_str = self._get_conf(config, t_node, "rename-parser") @@ -398,7 +416,8 @@ def _run_target_update(cls, dao, app_runner, compress_manager, target): # Compress sources if target.compress_scheme: handler = compress_manager.get_handler(target.compress_scheme) - handler.compress_sources(target, work_dir) + compress_args = {"threads": target.compress_threads} + handler.compress_sources(target, work_dir, **compress_args) times[1] = time() # transformed time # Run archive command sources = [] diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py index 0a5e1770cc..0d1df676df 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_gzip.py @@ -29,12 +29,15 @@ class RoseArchGzip: def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, threads="1"): """Gzip each source in target. Use work_dir to dump results. """ + if threads != "1": + raise NotImplementedError("Gzip does not support multi-threading") + for source in target.sources.values(): if source.path.endswith("." + target.compress_scheme): continue # assume already done diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py index 6d297979d5..6d6beab888 100644 --- a/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -25,19 +25,28 @@ class RoseArchTarGzip: """Compress archive sources in tar.""" - SCHEMES = ["pax", "pax.gz", "tar", "tar.gz", "tgz"] - SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, "pax.gz": tarfile.PAX_FORMAT} + SCHEMES = ["pax", "pax.gz", "pax.zst", "pax.xz", + "tar", "tar.gz", "tgz", "tar.zst", "tar.xz", "txz"] + SCHEME_FORMATS = {"pax": tarfile.PAX_FORMAT, + "pax.gz": tarfile.PAX_FORMAT, + "pax.zst": tarfile.PAX_FORMAT, + "pax.xz": tarfile.PAX_FORMAT} GZIP_EXTS = ["pax.gz", "tar.gz", "tgz"] + ZSTD_EXTS = ["pax.zst", "tar.zst"] + XZ_EXTS = ["pax.xz", "tar.xz", "txz"] def __init__(self, app_runner, *args, **kwargs): self.app_runner = app_runner - def compress_sources(self, target, work_dir): + def compress_sources(self, target, work_dir, threads="1"): """Create a tar archive of all files in target. Use work_dir to dump results. """ + if threads != "1": + raise NotImplementedError("xz does not support multi-threading") + sources = list(target.sources.values()) if len(sources) == 1 and sources[0].path.endswith( "." + target.compress_scheme @@ -70,3 +79,23 @@ def compress_sources(self, target, work_dir): command = "gzip -c '%s' >'%s'" % (tar_name, gz_name) self.app_runner.popen.run_simple(command, shell=True) self.app_runner.fs_util.delete(tar_name) + + elif target.compress_scheme in self.ZSTD_EXTS: + fdsec, zst_name = mkstemp( + suffix="." + target.compress_scheme, dir=work_dir + ) + os.close(fdsec) + target.work_source_path = zst_name + command = f"zstd --rm -T{threads} -c '{tar_name}' >'{zst_name}'" + self.app_runner.popen.run_simple(command, shell=True) + self.app_runner.fs_util.delete(tar_name) + + elif target.compress_scheme in self.XZ_EXTS: + fdsec, xz_name = mkstemp( + suffix="." + target.compress_scheme, dir=work_dir + ) + os.close(fdsec) + target.work_source_path = xz_name + command = "xz -c '%s' >'%s'" % (tar_name, xz_name) + self.app_runner.popen.run_simple(command, shell=True) + self.app_runner.fs_util.delete(tar_name) diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py new file mode 100644 index 0000000000..d086e324d5 --- /dev/null +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_xz.py @@ -0,0 +1,53 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Compress archive sources using xz.""" + + +import os + + +class RoseArchXz: + + """Compress archive sources in xz.""" + + SCHEMES = ["xz"] + + def __init__(self, app_runner, *args, **kwargs): + self.app_runner = app_runner + + def compress_sources(self, target, work_dir, threads="1"): + """xz each source in target. + + Use work_dir to dump results. + + """ + + if threads != "1": + raise NotImplementedError("xz does not support multi-threading") + + for source in target.sources.values(): + if source.path.endswith("." + target.compress_scheme): + continue # assume already done + name_xz = source.name + "." + target.compress_scheme + work_path_xz = os.path.join(work_dir, name_xz) + self.app_runner.fs_util.makedirs( + self.app_runner.fs_util.dirname(work_path_xz) + ) + + command = "xz -c '%s' >'%s'" % (source.path, work_path_xz) + self.app_runner.popen.run_simple(command, shell=True) + source.path = work_path_xz diff --git a/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py new file mode 100644 index 0000000000..7ad8a45028 --- /dev/null +++ b/metomi/rose/apps/rose_arch_compressions/rose_arch_zstd.py @@ -0,0 +1,50 @@ +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Compress archive sources using zstd.""" + + +import os + + +class RoseArchZstd: + + """Compress archive sources in zstd.""" + + SCHEMES = ["zst", "zstd"] + + def __init__(self, app_runner, *args, **kwargs): + self.app_runner = app_runner + + def compress_sources(self, target, work_dir, threads="1"): + """zstd each source in target. + + Use work_dir to dump results. + + """ + for source in target.sources.values(): + if source.path.endswith("." + target.compress_scheme): + continue # assume already done + name_zst = source.name + "." + target.compress_scheme + work_path_zst = os.path.join(work_dir, name_zst) + self.app_runner.fs_util.makedirs( + self.app_runner.fs_util.dirname(work_path_zst) + ) + command = \ + f"zstd --rm -T{threads} -c {source.path} > {work_path_zst}" + + self.app_runner.popen.run_simple(command, shell=True) + source.path = work_path_zst diff --git a/metomi/rose/env.py b/metomi/rose/env.py index 7ffd14d626..7be03c5ec3 100644 --- a/metomi/rose/env.py +++ b/metomi/rose/env.py @@ -107,11 +107,16 @@ def __repr__(self): def env_export(key, value, event_handler=None): """Export an environment variable.""" - if key not in _EXPORTED_ENVS or os.environ.get(key) != value: + if ( + key not in _EXPORTED_ENVS + or os.environb.get(key.encode("UTF-8")) + != value.encode("UTF-8") + ): # N.B. Should be safe, because the list of environment variables is # normally quite small. _EXPORTED_ENVS[key] = value os.environb[key.encode('UTF-8')] = value.encode('UTF-8') + if callable(event_handler): event_handler(EnvExportEvent(key, value)) @@ -134,7 +139,7 @@ def env_var_escape(text, match_mode=None): return ret -def env_var_process(text, unbound=None, match_mode=None, environ=os.environ): +def env_var_process(text, unbound=None, match_mode=None, environ=None): """Substitute environment variables into a string. For each $NAME and ${NAME} in "text", substitute with the value @@ -145,6 +150,8 @@ def env_var_process(text, unbound=None, match_mode=None, environ=os.environ): value of "unbound". """ + if environ is None: + environ = os.environ ret = "" try: tail = text.decode() diff --git a/metomi/rose/tests/apps/rose_arch/test_rose_arch.py b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py new file mode 100644 index 0000000000..9b2154cf97 --- /dev/null +++ b/metomi/rose/tests/apps/rose_arch/test_rose_arch.py @@ -0,0 +1,167 @@ + +# Copyright (C) British Crown (Met Office) & Contributors. +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +# ----------------------------------------------------------------------------- +"""Integration tests for Rose Arch.""" + +import pytest +from textwrap import dedent +from types import SimpleNamespace +from typing import Dict + + +from metomi.rose.task_run import TaskRunner +from metomi.rose.reporter import Reporter + + +ZIP_METHODS = { + "nozip": "nozip", + "gunzipme": "gunzipme.gz", + "targunzipme": "targunzipme.tar.gz", + "zstdme": "zstdme.zst", + "xzme": "xzme.xz" +} + + +@pytest.fixture +def setup_rose_arch_env(monkeypatch, tmp_path): + """Mock the environment variables needed for a rose_arch task. + """ + tmp_path_str = str(tmp_path) + # Avoids leaving rose database and rose-app-run.conf files in your CWD. + monkeypatch.chdir(tmp_path_str) + monkeypatch.setenv("CYLC_WORKFLOW_ID", "foo") + monkeypatch.setenv("CYLC_TASK_ID", "test_rose_arch") + monkeypatch.setenv("CYLC_TASK_NAME", "baz") + monkeypatch.setenv("CYLC_TASK_CYCLE_POINT", "qux") + monkeypatch.setenv("CYLC_TASK_LOG_ROOT", tmp_path_str) + monkeypatch.setenv("ROSE_SUITE_NAME", tmp_path_str) + monkeypatch.setenv("ROSE_SUITE_DIR", tmp_path_str) + monkeypatch.setenv("CYLC_RUN_DIR", tmp_path_str) + yield + + +@pytest.fixture +def setup_rose_arch_task(tmp_path): + """Set up a rose_arch task with a given config and source files. + """ + def _inner( + config: str, + source_files: Dict[str, str], + ): + (tmp_path / 'archive').mkdir() + (tmp_path / 'source').mkdir() + for source_file in source_files: + (tmp_path / 'source' / source_file).write_text( + source_files[source_file]) + (tmp_path / 'rose-app.conf').write_text(dedent(config)) + return _inner + + +@pytest.fixture +def run_rose_arch_task(tmp_path, setup_rose_arch_env): + """Run a rose_arch task.""" + + def _inner(): + # Create a crude options object: + opts = SimpleNamespace() + for key in TaskRunner.OPTIONS: + setattr(opts, key, None) + opts.conf_dir = str(tmp_path) + + # Run our task: + runner = TaskRunner(event_handler=Reporter()) + runner(opts, []) + return _inner + + +@pytest.mark.parametrize( + "source_file, expected", + [ + ("nozip", "nozip"), + ("gunzipme", "gunzipme.gz"), + ("targunzipme", "targunzipme.tar.gz"), + ("zstdme", "zstdme.zst"), + ("xzme", "xzme.xz") + ] +) +def test_rose_arch( + tmp_path, + setup_rose_arch_task, + run_rose_arch_task, + source_file, + expected +): + """Test the rose_arch app against different compression methods. + + Possible future enhancements: + - More checks on the compressed files, ensuring that the compression + has worked as expected. + """ + setup_rose_arch_task( + f""" + mode=rose_arch + + [arch] + command-format=cp %(sources)s %(target)s + target-prefix={tmp_path}/archive/ + source-prefix={tmp_path}/source/ + + [arch:{expected}] + source={source_file} + """, + ZIP_METHODS, + ) + run_rose_arch_task() + assert (tmp_path / 'archive' / expected).exists() + + +@pytest.mark.parametrize( + 'archiver', + [ + 'gz', + 'tar.gz', + 'xz', + ] +) +def test_raises_too_many_threads_err( + tmp_path, setup_rose_arch_task, run_rose_arch_task, archiver +): + """Test that we get an error if an archiver does not support threads, + but thread number (other than 1) is passed in. + + """ + setup_rose_arch_task( + f""" + mode=rose_arch + + [arch] + command-format=gzip %(sources)s %(target)s + target-prefix={tmp_path}/archive/ + source-prefix={tmp_path}/source/ + + [arch:zipme.{archiver}] + source=zipme + compress-threads=2 + """, + {"zipme": "Any old text file"}, + ) + + with pytest.raises( + NotImplementedError, + match="does not support multi-threading" + ): + run_rose_arch_task() diff --git a/metomi/rose/tests/test_env.py b/metomi/rose/tests/test_env.py index 8ca4f3d1c5..61c32dd44d 100644 --- a/metomi/rose/tests/test_env.py +++ b/metomi/rose/tests/test_env.py @@ -14,39 +14,30 @@ # You should have received a copy of the GNU General Public License # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import os -import unittest - from metomi.rose.env import env_export -class _TestEnvExport(unittest.TestCase): - """Test "env_export" function.""" - - def test_report_new(self): - """Ensure that env_export only reports 1st time or on change.""" - events = [] - env_export("FOO", "foo", events.append) - env_export("FOO", "foo", events.append) - env_export("FOO", "food", events.append) - env_export("FOO", "foot", events.append) - env_export("FOO", "foot", events.append) - event_args = [event.args[1] for event in events] - self.assertEqual(event_args, ["foo", "food", "foot"], "events") - - def test_report_old(self): - """Ensure that env_export only reports 1st time or on change.""" - events = [] - os.environ["BAR"] = "bar" - env_export("BAR", "bar", events.append) - env_export("BAR", "bar", events.append) - env_export("BAR", "bar", events.append) - env_export("BAR", "barley", events.append) - env_export("BAR", "barley", events.append) - env_export("BAR", "barber", events.append) - event_args = [event.args[1] for event in events] - self.assertEqual(event_args, ["bar", "barley", "barber"], "events") +def test_report_new(): + """Ensure that env_export only reports 1st time or on change.""" + events = [] + env_export("FOO", "foo", events.append) + env_export("FOO", "foo", events.append) + env_export("FOO", "food", events.append) + env_export("FOO", "foot", events.append) + env_export("FOO", "foot", events.append) + event_args = [event.args[1] for event in events] + assert event_args == ["foo", "food", "foot"] -if __name__ == "__main__": - unittest.main() +def test_report_old(monkeypatch): + """Ensure that env_export only reports 1st time or on change.""" + events = [] + monkeypatch.setenv("BAR", "bar") + env_export("BAR", "bar", events.append) + env_export("BAR", "bar", events.append) + env_export("BAR", "bar", events.append) + env_export("BAR", "barley", events.append) + env_export("BAR", "barley", events.append) + env_export("BAR", "barber", events.append) + event_args = [event.args[1] for event in events] + assert event_args == ["bar", "barley", "barber"] diff --git a/metomi/rose/tests/test_env_cat.py b/metomi/rose/tests/test_env_cat.py index 2f25fadbc2..285cc331f7 100644 --- a/metomi/rose/tests/test_env_cat.py +++ b/metomi/rose/tests/test_env_cat.py @@ -35,14 +35,12 @@ def test_output_to_file(tmp_path, monkeypatch): outputfile = tmp_path / 'outputfile' inputfile.write_text(r'Hello ${WORLD}') monkeypatch.setenv('WORLD', 'Jupiter') - opts = SimpleNamespace( match_mode=None, output_file=str(outputfile), unbound=None ) args = [str(inputfile)] - assert rose_env_cat(args, opts) is None assert outputfile.read_text() == 'Hello Jupiter' diff --git a/sphinx/api/built-in/rose_arch.rst b/sphinx/api/built-in/rose_arch.rst index 393bd1681f..4bcd655fbb 100644 --- a/sphinx/api/built-in/rose_arch.rst +++ b/sphinx/api/built-in/rose_arch.rst @@ -32,7 +32,7 @@ The application provides some useful functionalities: source files and the return code of archive command. In a retry, it would only redo targets that did not succeed in the previous attempts. * Rename source files. -* Tar-Gzip or Gzip source files before sending them to the archive. +* Tar and/or compress source files before sending them to the archive. Invocation @@ -124,8 +124,8 @@ on: [arch:(black-box/)] source=cats.txt dogs.txt -Zipping files -^^^^^^^^^^^^^ +Compressing files +^^^^^^^^^^^^^^^^^ There are multiple ways of specifying that you want your archive to be compressed: @@ -147,9 +147,9 @@ not recognized by rose arch as an extension to be compressed.) For more details see :rose:conf:`rose_arch[arch]compress` -Zipping directories -^^^^^^^^^^^^^^^^^^^ -You can tar and zip entire directories - as with single files Rose Arch will +Compressing directories +^^^^^^^^^^^^^^^^^^^^^^^ +You can tar and compress entire directories - as with single files Rose Arch will attempt to infer archive and compression from ``[arch:TARGET.extension]`` if it can: @@ -206,6 +206,22 @@ with names in the form ``data_001.txt``: rename-parser=^//some//path//data_(?P[0-9]{3})(?P.*)$ rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s +Using multiple threads for compression (zstd only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When using `zstd`, the number of threads to use for compression is controlled using the +`compress-threads` keyword. This is useful for large files +where multi-threaded compression can significantly improve throughput. + +.. code-block:: rose + + [arch:large-data.tar.zst] + compress=zst + compress-threads=8 + source=large-data/* + +In this example, the `zstd` will use 8 threads. + Output ------ @@ -245,7 +261,6 @@ taken to run the archive command and the return code of the archive command. For a source line, the third column contains the original name of the source. - Configuration ------------- @@ -262,7 +277,7 @@ Configuration and ``%(target)s`` for substitution of the sources and the target respectively. - .. rose:conf:: compress=pax|tar|pax.gz|tar.gz|tgz|gz + .. rose:conf:: compress=pax|tar|pax.gz|tar.gz|tgz|gz|pax.xz|tar.xz|txz|xz|pax.zst|tar.zst|zst If specified, compress source files scheme before sending them to the archive. If not set Rose Arch will attempt to set a compression scheme @@ -272,19 +287,48 @@ Configuration Each compression scheme works slightly differently: - +------------------+-----------------------------------------------+ - |Compression Scheme|Behaviour | - +------------------+-----------------------------------------------+ - |``pax`` or ``tar``|Sources will be placed in a TAR archive before | - | |being sent to the target. | - +------------------+-----------------------------------------------+ - |``pax.gz``, |Sources will be placed in a TAR-GZIP file | - |``tar.gz`` or |before being sent to the target. | - |``tgz`` | | - +------------------+-----------------------------------------------+ - |``gz`` |Each source file will be compressed by GZIP | - | |before being sent to the target. | - +------------------+-----------------------------------------------+ + +-----------------------+-----------------------------------------------+ + |Compression Scheme |Behaviour | + +-----------------------+-----------------------------------------------+ + |``pax`` or ``tar`` |Sources will be placed in a TAR archive before | + | |being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``pax.{gz,xz,zst}`` or |Sources will be placed in a TAR archive and be | + |``tar.{gz,xz,zst}`` or |compressed using the corresponding compressor | + |or ``tgz`` or ``txz`` |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``gz`` |Each source file will be compressed by gzip | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``xz`` |Each source file will be compressed by xz | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + |``zstd`` |Each source file will be compressed by zstd | + | |before being sent to the target. | + +-----------------------+-----------------------------------------------+ + + .. rose:conf:: compress-threads=0|1|2|... + + Specify the number of threads to use for compression. This setting + is optional and defaults to `1` (single-threaded compression). + + * `0`: Let the compression tool automatically determine the number of + threads to use which is typically equal to the number of detected physical CPU cores + (and should be used with caution on shared resources). + + * A positive integer: Specifies the exact number of threads to use for compression. + It is not recommended to exceed the number of physical CPU cores on the target resource. + + This setting is currently only supported by `zstd`. + + Example: + + .. code-block:: rose + + [arch:example.tar.zst] + compress=tar.zst + compress-threads=4 + source=example/* .. rose:conf:: rename-format diff --git a/t/rose-task-run/46-app-arch-zstd.t b/t/rose-task-run/46-app-arch-zstd.t new file mode 100755 index 0000000000..bc717397b5 --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd.t @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.zst +./archive.d/whatever.tar.zst +__FIND__ + +exit 0 diff --git a/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf b/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf new file mode 100644 index 0000000000..9c557e203e --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd/app/archive/rose-app.conf @@ -0,0 +1,16 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.zst] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.zst + +[arch:archive.d/whatever.tar.zst] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.zst diff --git a/t/rose-task-run/46-app-arch-zstd/flow.cylc b/t/rose-task-run/46-app-arch-zstd/flow.cylc new file mode 100644 index 0000000000..9666f3574a --- /dev/null +++ b/t/rose-task-run/46-app-arch-zstd/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +zstd --rm -T1 -f - <<<'MMXVI' >'2016.txt.zst' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I "zstd --rm -T1" -cf 'whatever.tar.zst' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/46-app-arch-zstd/rose-suite.conf b/t/rose-task-run/46-app-arch-zstd/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/rose-task-run/47-app-arch-xz.t b/t/rose-task-run/47-app-arch-xz.t new file mode 100755 index 0000000000..4f19cb184e --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz.t @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.xz +./archive.d/whatever.tar.xz +__FIND__ +#------------------------------------------------------------------------------- +purge +exit 0 diff --git a/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf b/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf new file mode 100644 index 0000000000..0aa738ebea --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz/app/archive/rose-app.conf @@ -0,0 +1,16 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.xz] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.xz + +[arch:archive.d/whatever.tar.xz] +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.xz diff --git a/t/rose-task-run/47-app-arch-xz/flow.cylc b/t/rose-task-run/47-app-arch-xz/flow.cylc new file mode 100644 index 0000000000..3ef861d58e --- /dev/null +++ b/t/rose-task-run/47-app-arch-xz/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +xz -f - <<<'MMXVI' >'2016.txt.xz' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I xz -cf 'whatever.tar.xz' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/47-app-arch-xz/rose-suite.conf b/t/rose-task-run/47-app-arch-xz/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/rose-task-run/48-app-arch-20130101T0000Z.out b/t/rose-task-run/48-app-arch-20130101T0000Z.out new file mode 100644 index 0000000000..c21954156f --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130101T0000Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130101T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130101T0000Z-star-1.txt (star-1.txt) +[INFO] + 20130101T0000Z-star-2.txt (star-2.txt) +[INFO] + 20130101T0000Z-star-3.txt (star-3.txt) +[INFO] + 20130101T0000Z-star-4.txt (star-4.txt) +[INFO] + 20130101T0000Z-star-5.txt (star-5.txt) +[INFO] + 20130101T0000Z-star-6.txt (star-6.txt) +[INFO] + 20130101T0000Z-star-7.txt (star-7.txt) +[INFO] + 20130101T0000Z-star-8.txt (star-8.txt) +[INFO] + 20130101T0000Z-star-9.txt (star-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130101T0000Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130101T0000Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130101T0000Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130101T0000Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130101T0000Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130101T0000Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130101T0000Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130101T0000Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130101T0000Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130101T0000Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130101T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130101T0000Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130101T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130101T0000Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T0000Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130101T0000Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130101T0000Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130101T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130101T0000Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130101T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-20130101T1200Z.out b/t/rose-task-run/48-app-arch-20130101T1200Z.out new file mode 100644 index 0000000000..2a8f42d423 --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130101T1200Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130101T1200Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130101T1200Z-star-1.txt (star-1.txt) +[INFO] + 20130101T1200Z-star-2.txt (star-2.txt) +[INFO] + 20130101T1200Z-star-3.txt (star-3.txt) +[INFO] + 20130101T1200Z-star-4.txt (star-4.txt) +[INFO] + 20130101T1200Z-star-5.txt (star-5.txt) +[INFO] + 20130101T1200Z-star-6.txt (star-6.txt) +[INFO] + 20130101T1200Z-star-7.txt (star-7.txt) +[INFO] + 20130101T1200Z-star-8.txt (star-8.txt) +[INFO] + 20130101T1200Z-star-9.txt (star-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130101T1200Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130101T1200Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130101T1200Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130101T1200Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130101T1200Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130101T1200Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130101T1200Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130101T1200Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130101T1200Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130101T1200Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130101T1200Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130101T1200Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130101T1200Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130101T1200Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130101T1200Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130101T1200Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130101T1200Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130101T1200Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130101T1200Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130101T1200Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-20130102T0000Z.out b/t/rose-task-run/48-app-arch-20130102T0000Z.out new file mode 100644 index 0000000000..8b6394405e --- /dev/null +++ b/t/rose-task-run/48-app-arch-20130102T0000Z.out @@ -0,0 +1,83 @@ +[INFO] ! foo://20130102T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=1] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/earth.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/earth.txt (hello/earth.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter-1.txt (hello/jupiter-moon-1.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/neptune-1.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/organisms/animals/crocodile.txt (hello/organisms/animals/crocodile.txt) +[INFO] + hello/organisms/animals/elephant.txt (hello/organisms/animals/elephant.txt) +[INFO] + hello/organisms/animals/goose.txt (hello/organisms/animals/goose.txt) +[INFO] + hello/organisms/animals/tiger.txt (hello/organisms/animals/tiger.txt) +[INFO] + hello/organisms/plants/daisy.txt (hello/organisms/plants/daisy.txt) +[INFO] + hello/organisms/plants/holly.txt (hello/organisms/plants/holly.txt) +[INFO] + hello/organisms/plants/iris.txt (hello/organisms/plants/iris.txt) +[INFO] + hello/organisms/plants/jasmine.txt (hello/organisms/plants/jasmine.txt) +[INFO] + hello/organisms/plants/lily.txt (hello/organisms/plants/lily.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/planet-1.txt (hello/planet-1.txt) +[INFO] + hello/planet-2.txt (hello/planet-2.txt) +[INFO] + hello/planet-3.txt (hello/planet-3.txt) +[INFO] + hello/planet-4.txt (hello/planet-4.txt) +[INFO] + hello/planet-5.txt (hello/planet-5.txt) +[INFO] + hello/planet-6.txt (hello/planet-6.txt) +[INFO] + hello/planet-7.txt (hello/planet-7.txt) +[INFO] + hello/planet-8.txt (hello/planet-8.txt) +[INFO] + hello/planet-9.txt (hello/planet-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/saturn.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/saturn.txt (hello/saturn.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/spaceships/ [compress=zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] + hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] + hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] + hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] + hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] + hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] + hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] + hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] + hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/stars/ [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + 20130102T0000Z-star-1.txt (star-1.txt) +[INFO] + 20130102T0000Z-star-2.txt (star-2.txt) +[INFO] + 20130102T0000Z-star-3.txt (star-3.txt) +[INFO] + 20130102T0000Z-star-4.txt (star-4.txt) +[INFO] + 20130102T0000Z-star-5.txt (star-5.txt) +[INFO] + 20130102T0000Z-star-6.txt (star-6.txt) +[INFO] + 20130102T0000Z-star-7.txt (star-7.txt) +[INFO] + 20130102T0000Z-star-8.txt (star-8.txt) +[INFO] + 20130102T0000Z-star-9.txt (star-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] + foo://20130102T0000Z/hello/worlds/unknown/stuff.pax [compress=pax, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/20130102T0000Z-stuff-1.txt (stuffing-1.txt) +[INFO] + hello/20130102T0000Z-stuff-2.txt (stuffing-2.txt) +[INFO] + hello/20130102T0000Z-stuff-3.txt (stuffing-3.txt) +[INFO] + hello/20130102T0000Z-stuff-4.txt (stuffing-4.txt) +[INFO] + hello/20130102T0000Z-stuff-5.txt (stuffing-5.txt) +[INFO] + hello/20130102T0000Z-stuff-6.txt (stuffing-6.txt) +[INFO] + hello/20130102T0000Z-stuff-7.txt (stuffing-7.txt) +[INFO] + hello/20130102T0000Z-stuff-8.txt (stuffing-8.txt) +[INFO] + hello/20130102T0000Z-stuff-9.txt (stuffing-9.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) +[INFO] + foo://20130102T0000Z/hello/worlds/dark-matter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] = foo://20130102T0000Z/hello/worlds/earth.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst [compress=tar.zst] +[INFO] + foo://20130102T0000Z/hello/worlds/jupiter.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/jupiter.txt (hello/jupiter.txt) +[INFO] = foo://20130102T0000Z/hello/worlds/neptune-1.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/organisms.tar.zst [compress=tar.zst] +[INFO] = foo://20130102T0000Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[INFO] = foo://20130102T0000Z/hello/worlds/saturn.txt [compress=None] +[INFO] = foo://20130102T0000Z/hello/worlds/spaceships/ [compress=zst] +[INFO] = foo://20130102T0000Z/hello/worlds/stars/ [compress=None] +[INFO] + foo://20130102T0000Z/hello/worlds/try.nl [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + try.nl (try.nl) +[INFO] = foo://20130102T0000Z/hello/worlds/unknown/stuff.pax [compress=pax] +[INFO] + foo://20130102T0000Z/hello/worlds/uranus.txt [compress=None, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs, ret-code=0] +[INFO] + hello/uranus.txt (hello/uranus.txt) diff --git a/t/rose-task-run/48-app-arch-bad-9.out b/t/rose-task-run/48-app-arch-bad-9.out new file mode 100644 index 0000000000..d83fae253f --- /dev/null +++ b/t/rose-task-run/48-app-arch-bad-9.out @@ -0,0 +1,29 @@ +[INFO] ! foo://20130101T1200Z/planet-n.tar.zst [compress=tar.zst, t(init)=YYYY-mm-DDTHH:MM:SSZ, dt(tran)=SSSSs, dt(arch)=SSSSs] +[INFO] ! hello/dark-matter.txt (hello/dark-matter.txt) +[INFO] ! hello/earth.txt (hello/earth.txt) +[INFO] ! hello/jupiter-moon-1.txt (hello/jupiter-moon-1.txt) +[INFO] ! hello/jupiter.txt (hello/jupiter.txt) +[INFO] ! hello/mars.txt (hello/mars.txt) +[INFO] ! hello/mercury.txt (hello/mercury.txt) +[INFO] ! hello/neptune-1.txt (hello/neptune-1.txt) +[INFO] ! hello/planet-1.txt (hello/planet-1.txt) +[INFO] ! hello/planet-2.txt (hello/planet-2.txt) +[INFO] ! hello/planet-3.txt (hello/planet-3.txt) +[INFO] ! hello/planet-4.txt (hello/planet-4.txt) +[INFO] ! hello/planet-5.txt (hello/planet-5.txt) +[INFO] ! hello/planet-6.txt (hello/planet-6.txt) +[INFO] ! hello/planet-7.txt (hello/planet-7.txt) +[INFO] ! hello/planet-8.txt (hello/planet-8.txt) +[INFO] ! hello/planet-9.txt (hello/planet-9.txt) +[INFO] ! hello/saturn.txt (hello/saturn.txt) +[INFO] ! hello/spaceship-1.txt (hello/spaceship-1.txt) +[INFO] ! hello/spaceship-2.txt (hello/spaceship-2.txt) +[INFO] ! hello/spaceship-3.txt (hello/spaceship-3.txt) +[INFO] ! hello/spaceship-4.txt (hello/spaceship-4.txt) +[INFO] ! hello/spaceship-5.txt (hello/spaceship-5.txt) +[INFO] ! hello/spaceship-6.txt (hello/spaceship-6.txt) +[INFO] ! hello/spaceship-7.txt (hello/spaceship-7.txt) +[INFO] ! hello/spaceship-8.txt (hello/spaceship-8.txt) +[INFO] ! hello/spaceship-9.txt (hello/spaceship-9.txt) +[INFO] ! hello/uranus.txt (hello/uranus.txt) +[INFO] ! hello/venus.txt (hello/venus.txt) diff --git a/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out b/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out new file mode 100644 index 0000000000..55332bc787 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T0000Z-1.out @@ -0,0 +1,66 @@ +foo://20130101T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130101T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/jupiter.txt|0| +foo://20130101T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/saturn.txt|0| +foo://20130101T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T0000Z/hello/uranus.txt|0| +foo://20130101T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|0218271ecf447dade73fa9b20ca806ac +foo://20130101T0000Z/hello/worlds/earth.txt|hello/earth.txt|64a40fb18eeb19581b1a3b9a7fd00b77 +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|ad79c5ad231e48d9986e7b32ef78ecfb +foo://20130101T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130101T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|cac4e4ff00049b6656e77e53612fbb2d +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|d5c0f47b697168163badd9b58c263726 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|f4657ee8b08b153d0d2a3ded6b943e72 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|600f9b25d93c8d4f3e4666da5e1c1be8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|339ec467c56ce5b50deb2f5ef9442a64 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|118dcd67a2d96968bd5a552a15f109a8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|1d8a5bf446b565ca4fe461ed89bc2143 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|9790ab68d589f0a3c93a4f93eb8d0979 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|9402ad6c5ede0bd21eccaa71cca635b6 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|a860bc83f223a37ddd3560ce9d9cf128 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|0842b4e54f373cc6daf6a72ef3bacef5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|e07d1be49e0116266ddd0b5b52eca0f5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|64e75062f6ca23f78f489286cdf1befb +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|b2813c2febf24d6b9c460d6e91d5584a +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|83ddd3b9422015989eb33f9f213cb1d3 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|0ef7af3d861cb997d9318410686ddeb2 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|055509281d7b83f60b867f39d817ad72 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|6c56036bfc0f7fbf855e67ebe632db04 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|eeaa51883a8b1ae0acaaecfa1ff85090 +foo://20130101T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|5a7a6e3b1b7976a07bac7fc0f8890e19 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|4b8aa78c4db8f5a17852821bc5856fd1 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|82ef697d04d3e175a6653d5047e4379e +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|c9bfb5f29d4c94bbd6cfeaddac383535 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|9e120d5961a129fcf7e16baf2c9c2d35 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b21bd2b81d440e9d1dbf82db12334bb4 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|d37b95e99a7f9647be60bfc4176cca9a +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|050e316a433bc33923d61052b17afb14 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|7e79965ca3fa019c9bec8bdd4a745067 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|9ee1dfff7161007ff211325b909775af +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-1.txt|39184686037c4f089065df1cea2243db +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-2.txt|86603e313db52af9313b49ab8b7d03a2 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-3.txt|eb9a6efe551ae81ab0867087b60b9456 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-4.txt|04276542f5259f022f85e99094364d26 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-5.txt|76df6a7b327a0a873e9d99708ee31e53 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-6.txt|fc1a89e038a378ce58e3d90921e478b4 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-7.txt|c097220252b6623d69815f0d405832e0 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-8.txt|447d11a036427f2271edc11287a3be84 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-9.txt|b3d436b8a13089629c196ccbae5a3dcd +foo://20130101T0000Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-1.txt|9def5f332e2904c360a5c88cb1abbbfb +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-2.txt|e4295cbe0bd6b64331b097eb6e4baf8d +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-3.txt|437f605878bee78144f1fdb42d839289 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-4.txt|7a50b985d13f98b81cd2e4afbb7f1e29 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-5.txt|7d2a84842e1321f1cddceeb94329d778 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-6.txt|d93899e3495e231c1dc8c42b8240ceb5 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-7.txt|6622fd0c176f91d5107946db1c98b9bc +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-8.txt|ed91e4d43abc966d9b1c4d5ccb49af55 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-9.txt|3266b847bb54c55c87f3329bead70801 +foo://20130101T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|f1737a14222ad46cb3cb567a39c00b11 diff --git a/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out b/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out new file mode 100644 index 0000000000..0b9700e893 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T0000Z-2.out @@ -0,0 +1,66 @@ +foo://20130101T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/jupiter.txt|0| +foo://20130101T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T0000Z/hello/t-ref share/cycle/20130101T0000Z/hello/saturn.txt|0| +foo://20130101T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T0000Z/hello/uranus.txt|0| +foo://20130101T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|0218271ecf447dade73fa9b20ca806ac +foo://20130101T0000Z/hello/worlds/earth.txt|hello/earth.txt|64a40fb18eeb19581b1a3b9a7fd00b77 +foo://20130101T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|ad79c5ad231e48d9986e7b32ef78ecfb +foo://20130101T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130101T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|cac4e4ff00049b6656e77e53612fbb2d +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|d5c0f47b697168163badd9b58c263726 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|f4657ee8b08b153d0d2a3ded6b943e72 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|600f9b25d93c8d4f3e4666da5e1c1be8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|339ec467c56ce5b50deb2f5ef9442a64 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|118dcd67a2d96968bd5a552a15f109a8 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|1d8a5bf446b565ca4fe461ed89bc2143 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|9790ab68d589f0a3c93a4f93eb8d0979 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|9402ad6c5ede0bd21eccaa71cca635b6 +foo://20130101T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|a860bc83f223a37ddd3560ce9d9cf128 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|0842b4e54f373cc6daf6a72ef3bacef5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|e07d1be49e0116266ddd0b5b52eca0f5 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|64e75062f6ca23f78f489286cdf1befb +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|b2813c2febf24d6b9c460d6e91d5584a +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|83ddd3b9422015989eb33f9f213cb1d3 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|0ef7af3d861cb997d9318410686ddeb2 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|055509281d7b83f60b867f39d817ad72 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|6c56036bfc0f7fbf855e67ebe632db04 +foo://20130101T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|eeaa51883a8b1ae0acaaecfa1ff85090 +foo://20130101T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|5a7a6e3b1b7976a07bac7fc0f8890e19 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|4b8aa78c4db8f5a17852821bc5856fd1 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|82ef697d04d3e175a6653d5047e4379e +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|c9bfb5f29d4c94bbd6cfeaddac383535 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|9e120d5961a129fcf7e16baf2c9c2d35 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b21bd2b81d440e9d1dbf82db12334bb4 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|d37b95e99a7f9647be60bfc4176cca9a +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|050e316a433bc33923d61052b17afb14 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|7e79965ca3fa019c9bec8bdd4a745067 +foo://20130101T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|9ee1dfff7161007ff211325b909775af +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-1.txt|39184686037c4f089065df1cea2243db +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-2.txt|86603e313db52af9313b49ab8b7d03a2 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-3.txt|eb9a6efe551ae81ab0867087b60b9456 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-4.txt|04276542f5259f022f85e99094364d26 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-5.txt|76df6a7b327a0a873e9d99708ee31e53 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-6.txt|fc1a89e038a378ce58e3d90921e478b4 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-7.txt|c097220252b6623d69815f0d405832e0 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-8.txt|447d11a036427f2271edc11287a3be84 +foo://20130101T0000Z/hello/worlds/stars/|20130101T0000Z-star-9.txt|b3d436b8a13089629c196ccbae5a3dcd +foo://20130101T0000Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-1.txt|9def5f332e2904c360a5c88cb1abbbfb +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-2.txt|e4295cbe0bd6b64331b097eb6e4baf8d +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-3.txt|437f605878bee78144f1fdb42d839289 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-4.txt|7a50b985d13f98b81cd2e4afbb7f1e29 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-5.txt|7d2a84842e1321f1cddceeb94329d778 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-6.txt|d93899e3495e231c1dc8c42b8240ceb5 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-7.txt|6622fd0c176f91d5107946db1c98b9bc +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-8.txt|ed91e4d43abc966d9b1c4d5ccb49af55 +foo://20130101T0000Z/hello/worlds/unknown/stuff.pax|hello/20130101T0000Z-stuff-9.txt|3266b847bb54c55c87f3329bead70801 +foo://20130101T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out b/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out new file mode 100644 index 0000000000..e86d46b7bc --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T1200Z-1.out @@ -0,0 +1,66 @@ +foo://20130101T1200Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130101T1200Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/jupiter.txt|0| +foo://20130101T1200Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/saturn.txt|0| +foo://20130101T1200Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T1200Z/hello/uranus.txt|0| +foo://20130101T1200Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|e621a29f24a71f23b3780e2bb0117bb9 +foo://20130101T1200Z/hello/worlds/earth.txt|hello/earth.txt|788bad396b603b9783f9e7bcafa6c797 +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|94c7fccf08f8b7959d884eaff7312227 +foo://20130101T1200Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130101T1200Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|f6674a8308b4d43791f9c52e07e1fa7c +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|f9c382939532a88c769a4221b1e22503 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|c48fcde460349f75ac39e850354ec628 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|052aebda2673e7a0e155486563b90f39 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|b68a019e7d41306580c5dcb78553040b +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|ad22c9d8f490d06fb5bc67cc5a169240 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|9a698b7c5d5118f4d1d3eb176b54c023 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|66bb8f516bafed7658826592ba352eb5 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|1dea09e75c6ce4461078a7ded7252e55 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|e1f7d05200a141edf26cfc4b1156e798 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|8a98f0c613e771f5d331a2d34c2895da +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|8cad6bcdf4d22e1dfe1d35fa03c6f5cb +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|275ab56c89605dd66078b80834253baa +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|dfe0cff95c79512d1bbc1d87c74cd36a +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|a90b5e210b99eb5f922a6ee672f5565b +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|1663ba51e68369147d92569515134353 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|6afc5fc439021fcf4089c282da667816 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|ba860ddc208de7748145144eb7d83b62 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|7557bf0caf752172b29fc74afea77ce8 +foo://20130101T1200Z/hello/worlds/saturn.txt|hello/saturn.txt|6bb3a3d31e658b6f33cc74aa29a15275 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-1.txt|d184f4517e6210d7c0f076127441572f +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-2.txt|d1222f82c23304d5d9c52f6e7e973522 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-3.txt|38fd825c464f691d9cdc0f45456fe4ac +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-4.txt|6c501adbd95198f88e756b6b6db59116 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-5.txt|44d8c92a12c4b7c9eb278f52ced5e234 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-6.txt|4e6150d0e385688e71b6b3947d500adf +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-7.txt|4cc45c56b453604e197fc79f39cd484c +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-8.txt|45dd9c8b7454e9aa007616224a196c6e +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-9.txt|0e2a93472b0faaf22a0b790679e102e1 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-1.txt|9c0756f2e12955569750f9143e75f1a6 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-2.txt|9a017867af5ffc2156b426bdffaaf444 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-3.txt|7416e2c7a8577dbdce044c809e173a4d +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-4.txt|e6659108a95d8197f6167ec23a217179 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-5.txt|b043475305dc7e845a01935eb8ae6363 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-6.txt|1869ae024acdd3c797d6c8e530d89888 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-7.txt|4512974ee6609d1c23079163e8f16120 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-8.txt|c24a8e117d5de41b79af875eb382eb7f +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-9.txt|113ba9f4a4e3bd7d82b7a317b9bfec9e +foo://20130101T1200Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-1.txt|f64291009d30056102802980bf80e1c9 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-2.txt|c94c43b3e3c3dc55dfbca86890902816 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-3.txt|d4959cb344dd139b33766337e00f9d85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-4.txt|571c83c56b247d8d95fbd4225016e2b4 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-5.txt|98343306a5ff665121a754f2fedbce1b +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-6.txt|989d6be0f0f83840d7959df50cb25ee5 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-7.txt|fa121b4fb2a8e3bcf99f0d758a885682 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-8.txt|2e9be382c5071a6d0032ca166ead2d39 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-9.txt|99c4e7f4d54851314a8a6aea2d1d4a02 +foo://20130101T1200Z/hello/worlds/uranus.txt|hello/uranus.txt|47afc5ee013b505bcb6f8eb244ddd42a diff --git a/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out b/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out new file mode 100644 index 0000000000..b53941af70 --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130101T1200Z-2.out @@ -0,0 +1,66 @@ +foo://20130101T1200Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/jupiter.txt|0| +foo://20130101T1200Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130101T1200Z/hello/t-ref share/cycle/20130101T1200Z/hello/saturn.txt|0| +foo://20130101T1200Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130101T1200Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130101T1200Z/hello/uranus.txt|0| +foo://20130101T1200Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|e621a29f24a71f23b3780e2bb0117bb9 +foo://20130101T1200Z/hello/worlds/earth.txt|hello/earth.txt|788bad396b603b9783f9e7bcafa6c797 +foo://20130101T1200Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|94c7fccf08f8b7959d884eaff7312227 +foo://20130101T1200Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130101T1200Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|f6674a8308b4d43791f9c52e07e1fa7c +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|f9c382939532a88c769a4221b1e22503 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|c48fcde460349f75ac39e850354ec628 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|052aebda2673e7a0e155486563b90f39 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|b68a019e7d41306580c5dcb78553040b +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|ad22c9d8f490d06fb5bc67cc5a169240 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|9a698b7c5d5118f4d1d3eb176b54c023 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|66bb8f516bafed7658826592ba352eb5 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|1dea09e75c6ce4461078a7ded7252e55 +foo://20130101T1200Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|e1f7d05200a141edf26cfc4b1156e798 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|8a98f0c613e771f5d331a2d34c2895da +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|8cad6bcdf4d22e1dfe1d35fa03c6f5cb +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|275ab56c89605dd66078b80834253baa +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|dfe0cff95c79512d1bbc1d87c74cd36a +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|a90b5e210b99eb5f922a6ee672f5565b +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|1663ba51e68369147d92569515134353 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|6afc5fc439021fcf4089c282da667816 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|ba860ddc208de7748145144eb7d83b62 +foo://20130101T1200Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|7557bf0caf752172b29fc74afea77ce8 +foo://20130101T1200Z/hello/worlds/saturn.txt|hello/saturn.txt|6bb3a3d31e658b6f33cc74aa29a15275 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-1.txt|d184f4517e6210d7c0f076127441572f +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-2.txt|d1222f82c23304d5d9c52f6e7e973522 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-3.txt|38fd825c464f691d9cdc0f45456fe4ac +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-4.txt|6c501adbd95198f88e756b6b6db59116 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-5.txt|44d8c92a12c4b7c9eb278f52ced5e234 +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-6.txt|4e6150d0e385688e71b6b3947d500adf +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-7.txt|4cc45c56b453604e197fc79f39cd484c +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-8.txt|45dd9c8b7454e9aa007616224a196c6e +foo://20130101T1200Z/hello/worlds/spaceships/|hello/spaceship-9.txt|0e2a93472b0faaf22a0b790679e102e1 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-1.txt|9c0756f2e12955569750f9143e75f1a6 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-2.txt|9a017867af5ffc2156b426bdffaaf444 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-3.txt|7416e2c7a8577dbdce044c809e173a4d +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-4.txt|e6659108a95d8197f6167ec23a217179 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-5.txt|b043475305dc7e845a01935eb8ae6363 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-6.txt|1869ae024acdd3c797d6c8e530d89888 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-7.txt|4512974ee6609d1c23079163e8f16120 +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-8.txt|c24a8e117d5de41b79af875eb382eb7f +foo://20130101T1200Z/hello/worlds/stars/|20130101T1200Z-star-9.txt|113ba9f4a4e3bd7d82b7a317b9bfec9e +foo://20130101T1200Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-1.txt|f64291009d30056102802980bf80e1c9 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-2.txt|c94c43b3e3c3dc55dfbca86890902816 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-3.txt|d4959cb344dd139b33766337e00f9d85 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-4.txt|571c83c56b247d8d95fbd4225016e2b4 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-5.txt|98343306a5ff665121a754f2fedbce1b +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-6.txt|989d6be0f0f83840d7959df50cb25ee5 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-7.txt|fa121b4fb2a8e3bcf99f0d758a885682 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-8.txt|2e9be382c5071a6d0032ca166ead2d39 +foo://20130101T1200Z/hello/worlds/unknown/stuff.pax|hello/20130101T1200Z-stuff-9.txt|99c4e7f4d54851314a8a6aea2d1d4a02 +foo://20130101T1200Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out b/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out new file mode 100644 index 0000000000..91f590e58a --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130102T0000Z-1.out @@ -0,0 +1,66 @@ +foo://20130102T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 1)) foo put %(target)s %(sources)s|1| +foo://20130102T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/jupiter.txt|0| +foo://20130102T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/saturn.txt|0| +foo://20130102T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130102T0000Z/hello/uranus.txt|0| +foo://20130102T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|8ee65f3e701e07c0c3ad586dc58a948c +foo://20130102T0000Z/hello/worlds/earth.txt|hello/earth.txt|d912f30f89b5d40f0ffa434e9e9714a8 +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|7a4d5d5aba14ba1e059086ac6e0e34d0 +foo://20130102T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1321819860.0:size=31 +foo://20130102T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|52be17cb5032ab1ee2b3a34407e4f78f +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|bf410362cfc403e2542bdcb1ae46acce +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|8ba02aef72f42796d591e8858178bf0e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|2989e4dbaa92e00fb7a05cc2db97468d +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|4ad3cf583dc9618d4986ff27d8fafdfd +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|233105195d5f897efaa46f2e0483d24e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|c63b821746a19144b2b507ef8744d1af +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|656ec00852d9d90f929a689b1dfe1f81 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|7798ae798fffcb529a8729cf5f5ba128 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|bdaecbceb631bac47bd2749bd20acab5 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|d1bf2cfa960311278a3aea9afaf60fca +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|bbebf2273dab1dc56b104f6373c1b249 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|f4f61419fc7464ce8b132f8b9729740e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|0df8d342722232d196dd1049e78b3e77 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|dea2da39535cfc5af553f04402bbfc0e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|980aac5825abde73528a0c823309fc64 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|97ef84d6a5bfab093523e20e09aa5137 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|a53efd68f81f975bd317f1e39b39f7f3 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|ff0fbf9027e58543bacc15e5b7d289a8 +foo://20130102T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|2610a66e838d75c745d8a64ff8541812 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|9612fa0c9617830d044e9e402f74d834 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|8010aa72a3d5fe2ef4c4bf9d38751de3 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|83cf0703fc41171eade4510496574063 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|1ee60607b68bd42f5df90b586f5ca0ce +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b0cc0a54e9354bba052b40dc7cd6cb3f +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|2d53a0b073804ee611bf86ec73a09589 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|f8fc808d5003015b6d3643e8825cc214 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|85a07828e07ab8aa8afa6f8262081403 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|de6b8c8c53d0d91c960644a18129cb0c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-1.txt|8839564acc51805ff31ff04ae5d968fa +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-2.txt|c499c20d00e4608a1917e3029b53b53e +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-3.txt|864b4d48a5c392c346bae087d0fc2b1b +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-4.txt|7a29d25e015216980de1f209b8ab11f5 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-5.txt|b8f755ca24d6021b4d2b7d0d5c7e7b1c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-6.txt|16bd791b242665fcb42a2457dca44dc1 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-7.txt|f0b3afb481ce3d593d9fc9820cef0456 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-8.txt|bd89739f414b8cb132bc7c1e9dcdf185 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-9.txt|326c18a6f265aabd6fe26e8c69dcad60 +foo://20130102T0000Z/hello/worlds/try.nl|try.nl|2185bc8a21ae6a853bd74b01ad25af85 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-1.txt|a69330fee599fe039f6cb1d467f70393 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-2.txt|7b1e0742b9df08316e24fcabb957b482 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-3.txt|c8b60d5d4bf9b42c411c07c853952283 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-4.txt|ea557c22f3a35303b19dcdfd45dacd01 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-5.txt|468cd05e391cbdaf628af7283a407381 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-6.txt|75dcc0652cf14ea3cab8a88335a0918c +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-7.txt|6d5d10d843d9d04b975bbef7bfe908e5 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-8.txt|ca0fb6bef96ec3e9dadaea8fc799bd54 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-9.txt|8d988c43da1dd7df887ec5a9d4acd3e6 +foo://20130102T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|e5079393fefe526d4074c2dcc8ef2462 diff --git a/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out b/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out new file mode 100644 index 0000000000..56d245981e --- /dev/null +++ b/t/rose-task-run/48-app-arch-db-20130102T0000Z-2.out @@ -0,0 +1,66 @@ +foo://20130102T0000Z/hello/worlds/dark-matter.txt||FOO_RC=$((2 - 2)) foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/earth.txt||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|tar.zst|foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/jupiter.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/jupiter.txt|0| +foo://20130102T0000Z/hello/worlds/neptune-1.txt||foo put %(target)s %(sources)s|0|sed 's/Hello/Greet/' %(in)s >%(out)s +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|tar.zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/saturn.txt||foo put %(target)s %(sources)s; touch -r share/cycle/20130102T0000Z/hello/t-ref share/cycle/20130102T0000Z/hello/saturn.txt|0| +foo://20130102T0000Z/hello/worlds/spaceships/|zst|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/stars/||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/try.nl||foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|pax|foo put %(target)s %(sources)s|0| +foo://20130102T0000Z/hello/worlds/uranus.txt||foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/20130102T0000Z/hello/uranus.txt|0| +foo://20130102T0000Z/hello/worlds/dark-matter.txt|hello/dark-matter.txt|8ee65f3e701e07c0c3ad586dc58a948c +foo://20130102T0000Z/hello/worlds/earth.txt|hello/earth.txt|d912f30f89b5d40f0ffa434e9e9714a8 +foo://20130102T0000Z/hello/worlds/jupiter-moons.tar.zst|hello/jupiter-1.txt|7a4d5d5aba14ba1e059086ac6e0e34d0 +foo://20130102T0000Z/hello/worlds/jupiter.txt|hello/jupiter.txt|source=$ROSE_DATAC/hello/jupiter.txt:mtime=1356034320.0:size=31 +foo://20130102T0000Z/hello/worlds/neptune-1.txt|hello/neptune-1.txt|52be17cb5032ab1ee2b3a34407e4f78f +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/crocodile.txt|bf410362cfc403e2542bdcb1ae46acce +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/elephant.txt|8ba02aef72f42796d591e8858178bf0e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/goose.txt|2989e4dbaa92e00fb7a05cc2db97468d +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/animals/tiger.txt|4ad3cf583dc9618d4986ff27d8fafdfd +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/daisy.txt|233105195d5f897efaa46f2e0483d24e +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/holly.txt|c63b821746a19144b2b507ef8744d1af +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/iris.txt|656ec00852d9d90f929a689b1dfe1f81 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/jasmine.txt|7798ae798fffcb529a8729cf5f5ba128 +foo://20130102T0000Z/hello/worlds/organisms.tar.zst|hello/organisms/plants/lily.txt|bdaecbceb631bac47bd2749bd20acab5 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-1.txt|d1bf2cfa960311278a3aea9afaf60fca +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-2.txt|bbebf2273dab1dc56b104f6373c1b249 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-3.txt|f4f61419fc7464ce8b132f8b9729740e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-4.txt|0df8d342722232d196dd1049e78b3e77 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-5.txt|dea2da39535cfc5af553f04402bbfc0e +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-6.txt|980aac5825abde73528a0c823309fc64 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-7.txt|97ef84d6a5bfab093523e20e09aa5137 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-8.txt|a53efd68f81f975bd317f1e39b39f7f3 +foo://20130102T0000Z/hello/worlds/planet-n.tar.zst|hello/planet-9.txt|ff0fbf9027e58543bacc15e5b7d289a8 +foo://20130102T0000Z/hello/worlds/saturn.txt|hello/saturn.txt|2610a66e838d75c745d8a64ff8541812 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-1.txt|9612fa0c9617830d044e9e402f74d834 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-2.txt|8010aa72a3d5fe2ef4c4bf9d38751de3 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-3.txt|83cf0703fc41171eade4510496574063 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-4.txt|1ee60607b68bd42f5df90b586f5ca0ce +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-5.txt|b0cc0a54e9354bba052b40dc7cd6cb3f +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-6.txt|2d53a0b073804ee611bf86ec73a09589 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-7.txt|f8fc808d5003015b6d3643e8825cc214 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-8.txt|85a07828e07ab8aa8afa6f8262081403 +foo://20130102T0000Z/hello/worlds/spaceships/|hello/spaceship-9.txt|de6b8c8c53d0d91c960644a18129cb0c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-1.txt|8839564acc51805ff31ff04ae5d968fa +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-2.txt|c499c20d00e4608a1917e3029b53b53e +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-3.txt|864b4d48a5c392c346bae087d0fc2b1b +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-4.txt|7a29d25e015216980de1f209b8ab11f5 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-5.txt|b8f755ca24d6021b4d2b7d0d5c7e7b1c +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-6.txt|16bd791b242665fcb42a2457dca44dc1 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-7.txt|f0b3afb481ce3d593d9fc9820cef0456 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-8.txt|bd89739f414b8cb132bc7c1e9dcdf185 +foo://20130102T0000Z/hello/worlds/stars/|20130102T0000Z-star-9.txt|326c18a6f265aabd6fe26e8c69dcad60 +foo://20130102T0000Z/hello/worlds/try.nl|try.nl|d6a814e1eda7d8c2b232d16a2d81dc1f +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-1.txt|a69330fee599fe039f6cb1d467f70393 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-2.txt|7b1e0742b9df08316e24fcabb957b482 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-3.txt|c8b60d5d4bf9b42c411c07c853952283 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-4.txt|ea557c22f3a35303b19dcdfd45dacd01 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-5.txt|468cd05e391cbdaf628af7283a407381 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-6.txt|75dcc0652cf14ea3cab8a88335a0918c +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-7.txt|6d5d10d843d9d04b975bbef7bfe908e5 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-8.txt|ca0fb6bef96ec3e9dadaea8fc799bd54 +foo://20130102T0000Z/hello/worlds/unknown/stuff.pax|hello/20130102T0000Z-stuff-9.txt|8d988c43da1dd7df887ec5a9d4acd3e6 +foo://20130102T0000Z/hello/worlds/uranus.txt|hello/uranus.txt|a4856e19e074ba4baf25714d08003aac diff --git a/t/rose-task-run/48-app-arch-find-foo.out b/t/rose-task-run/48-app-arch-find-foo.out new file mode 100644 index 0000000000..e770df9790 --- /dev/null +++ b/t/rose-task-run/48-app-arch-find-foo.out @@ -0,0 +1,61 @@ +foo/20130101T0000Z/hello/worlds/dark-matter.txt +foo/20130101T0000Z/hello/worlds/earth.txt +foo/20130101T0000Z/hello/worlds/jupiter-moons.tar.zst +foo/20130101T0000Z/hello/worlds/jupiter.txt +foo/20130101T0000Z/hello/worlds/neptune-1.txt +foo/20130101T0000Z/hello/worlds/organisms.tar.zst +foo/20130101T0000Z/hello/worlds/planet-n.tar.zst +foo/20130101T0000Z/hello/worlds/saturn.txt +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130101T0000Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130101T0000Z/hello/worlds/try.nl +foo/20130101T0000Z/hello/worlds/unknown/stuff.pax +foo/20130101T0000Z/hello/worlds/uranus.txt +foo/20130101T1200Z/hello/worlds/dark-matter.txt +foo/20130101T1200Z/hello/worlds/earth.txt +foo/20130101T1200Z/hello/worlds/jupiter-moons.tar.zst +foo/20130101T1200Z/hello/worlds/jupiter.txt +foo/20130101T1200Z/hello/worlds/mars.txt.zst +foo/20130101T1200Z/hello/worlds/neptune-1.txt +foo/20130101T1200Z/hello/worlds/organisms.tar.zst +foo/20130101T1200Z/hello/worlds/planet-n.tar.zst +foo/20130101T1200Z/hello/worlds/saturn.txt +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130101T1200Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130101T1200Z/hello/worlds/try.nl +foo/20130101T1200Z/hello/worlds/unknown/stuff.pax +foo/20130101T1200Z/hello/worlds/uranus.txt +foo/20130102T0000Z/hello/worlds/dark-matter.txt +foo/20130102T0000Z/hello/worlds/earth.txt +foo/20130102T0000Z/hello/worlds/jupiter-moons.tar.zst +foo/20130102T0000Z/hello/worlds/jupiter.txt +foo/20130102T0000Z/hello/worlds/neptune-1.txt +foo/20130102T0000Z/hello/worlds/organisms.tar.zst +foo/20130102T0000Z/hello/worlds/planet-n.tar.zst +foo/20130102T0000Z/hello/worlds/saturn.txt +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-1.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-2.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-3.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-4.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-5.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-6.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-7.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-8.txt.zst +foo/20130102T0000Z/hello/worlds/spaceships/spaceship-9.txt.zst +foo/20130102T0000Z/hello/worlds/try.nl +foo/20130102T0000Z/hello/worlds/unknown/stuff.pax +foo/20130102T0000Z/hello/worlds/uranus.txt diff --git a/t/rose-task-run/48-app-arch-planet-n.out b/t/rose-task-run/48-app-arch-planet-n.out new file mode 100644 index 0000000000..a7600f6bd4 --- /dev/null +++ b/t/rose-task-run/48-app-arch-planet-n.out @@ -0,0 +1,9 @@ +hello/planet-1.txt +hello/planet-2.txt +hello/planet-3.txt +hello/planet-4.txt +hello/planet-5.txt +hello/planet-6.txt +hello/planet-7.txt +hello/planet-8.txt +hello/planet-9.txt diff --git a/t/rose-task-run/48-app-arch-unknown-stuff.out b/t/rose-task-run/48-app-arch-unknown-stuff.out new file mode 100644 index 0000000000..0ee5c02929 --- /dev/null +++ b/t/rose-task-run/48-app-arch-unknown-stuff.out @@ -0,0 +1,9 @@ +hello/$CYCLE-stuff-1.txt +hello/$CYCLE-stuff-2.txt +hello/$CYCLE-stuff-3.txt +hello/$CYCLE-stuff-4.txt +hello/$CYCLE-stuff-5.txt +hello/$CYCLE-stuff-6.txt +hello/$CYCLE-stuff-7.txt +hello/$CYCLE-stuff-8.txt +hello/$CYCLE-stuff-9.txt diff --git a/t/rose-task-run/48-app-arch.t b/t/rose-task-run/48-app-arch.t new file mode 100755 index 0000000000..b70722c6f2 --- /dev/null +++ b/t/rose-task-run/48-app-arch.t @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test rose_arch built-in application. +#------------------------------------------------------------------------------- +. $(dirname $0)/test_header + + +#------------------------------------------------------------------------------- +tests 49 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export ROSE_CONF_PATH= +get_reg +TEST_KEY="${TEST_KEY_BASE}-install" +run_pass "$TEST_KEY" \ + cylc install \ + "$TEST_SOURCE_DIR/$TEST_KEY_BASE" \ + --workflow-name="$FLOW" \ + --no-run-name +TEST_KEY="${TEST_KEY_BASE}-play" +run_pass "$TEST_KEY" \ + cylc play \ + "$FLOW" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +cp ${TEST_KEY}.err ~/temp +cp ${TEST_KEY}.out ~/temp +#------------------------------------------------------------------------------- +# Results, good ones +TEST_KEY="$TEST_KEY_BASE-find-foo" +(cd $FLOW_RUN_DIR; find foo -type f | LANG=C sort) >"$TEST_KEY.out" +file_cmp "$TEST_KEY.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" "$TEST_KEY.out" +for CYCLE in 20130101T0000Z 20130101T1200Z 20130102T0000Z; do + TEST_KEY="$TEST_KEY_BASE-$CYCLE.out" + sed '/^\[INFO\] [=!+]/!d; + s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; + s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ + $FLOW_RUN_DIR/log/job/$CYCLE/archive/0*/job.out >"$TEST_KEY" + file_cmp "$TEST_KEY" "$TEST_KEY" $TEST_SOURCE_DIR/$TEST_KEY_BASE-$CYCLE.out + TEST_KEY="$TEST_KEY_BASE-planet-n" + tar -I zstd -tf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/planet-n.tar.zst | \ + LANG=C sort >"$TEST_KEY-$CYCLE.out" + file_cmp "$TEST_KEY-$CYCLE.out" \ + "$TEST_KEY-$CYCLE.out" "$TEST_SOURCE_DIR/$TEST_KEY.out" + TEST_KEY="$TEST_KEY_BASE-unknown-stuff" + tar -tf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/unknown/stuff.pax | \ + LANG=C sort >"$TEST_KEY-$CYCLE.out" + sed "s/\\\$CYCLE/$CYCLE/" "$TEST_SOURCE_DIR/$TEST_KEY.out" \ + >"$TEST_KEY-$CYCLE.out.expected" + file_cmp "$TEST_KEY-$CYCLE.out" \ + "$TEST_KEY-$CYCLE.out" "$TEST_KEY-$CYCLE.out.expected" + TEST_KEY="$TEST_KEY_BASE-db" + for TRY in 1 2; do + FILE=$FLOW_RUN_DIR/work/$CYCLE/archive/rose-arch-db-$TRY.out + sed "s?\\\$ROSE_DATAC?$FLOW_RUN_DIR/share/cycle/$CYCLE?" \ + "$TEST_SOURCE_DIR/$TEST_KEY-$CYCLE-$TRY.out" >$FILE.expected + file_cmp "$TEST_KEY-$CYCLE.out" $FILE.expected $FILE + done + for KEY in dark-matter.txt jupiter.txt try.nl uranus.txt; do + TEST_KEY="$TEST_KEY_BASE-$CYCLE-grep-$KEY-foo-log-2" + file_grep "$TEST_KEY" $KEY $FLOW_RUN_DIR/foo.log.$CYCLE.2 + done + if test $(wc -l <$FLOW_RUN_DIR/foo.log.$CYCLE.2) -eq 4; then + pass "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" + else + fail "$TEST_KEY_BASE-$CYCLE-foo-log-2-wc-l" + fi + TEST_KEY="$TEST_KEY_BASE-$CYCLE-neptune-1.txt" + file_cmp "$TEST_KEY" \ + $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/neptune-1.txt <<__TXT__ +[$CYCLE] Greet Triton +__TXT__ + TEST_KEY="$TEST_KEY_BASE-$CYCLE-jupiter-moons.tar.zst" + tar -xf $FLOW_RUN_DIR/foo/$CYCLE/hello/worlds/jupiter-moons.tar.zst -O \ + >"$TEST_KEY.out" + file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__TXT__ +[$CYCLE] Greet Io +__TXT__ +done +#------------------------------------------------------------------------------- +# Results, bad ones +CYCLE=20130101T1200Z +TEST_KEY="$TEST_KEY_BASE-bad-archive-1" +FILE_PREFIX="$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}1/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/hello/worlds/planet-n.tar.zst: bad command-format: foo put %(target)s %(source)s: KeyError: 'source' +[FAIL] ! foo://20130101T1200Z/hello/worlds/planet-n.tar.zst [compress=tar.zst] +[FAIL] ! hello/planet-1.txt (hello/planet-1.txt) +[FAIL] ! hello/planet-2.txt (hello/planet-2.txt) +[FAIL] ! hello/planet-3.txt (hello/planet-3.txt) +[FAIL] ! hello/planet-4.txt (hello/planet-4.txt) +[FAIL] ! hello/planet-5.txt (hello/planet-5.txt) +[FAIL] ! hello/planet-6.txt (hello/planet-6.txt) +[FAIL] ! hello/planet-7.txt (hello/planet-7.txt) +[FAIL] ! hello/planet-8.txt (hello/planet-8.txt) +[FAIL] ! hello/planet-9.txt (hello/planet-9.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-2" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}2/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] source=None: missing configuration error: 'source' +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-3" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}3/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] source=$UNBOUND_PLANET-[1-9].txt: configuration value error: [UNDEFINED ENVIRONMENT VARIABLE] UNBOUND_PLANET +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-4" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}4/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] [arch:$UNKNOWN_DARK_PLANETS.tar.zst]=: configuration value error: [UNDEFINED ENVIRONMENT VARIABLE] UNKNOWN_DARK_PLANETS +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-5" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}5/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ +[FAIL] [arch:inner.tar.zst]source=hello/mercurry.txt: configuration value error: [Errno 2] No such file or directory: '${FLOW_RUN_DIR}/share/cycle/20130101T1200Z/hello/mercurry.txt' +[FAIL] ! foo://20130101T1200Z/hello/worlds/inner.tar.zst [compress=tar.zst] +[FAIL] ! hello/venus.txt (hello/venus.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-6" +# NOTE: /BASH_XTRACEFD/d is to remove any Cylc errors of the form: +# BASH_XTRACEFD: 19: invalid value for trace file descriptor +sed \ + -e '/^\[FAIL\] /!d; s/ \[compress.*\]$//' \ + -e '/BASH_XTRACEFD/d' \ + "$TEST_KEY.err" \ + "${FILE_PREFIX}6/01/job.err" >"$TEST_KEY.err" +file_cmp "$TEST_KEY.err" "$TEST_KEY.err" <<__ERR__ +[FAIL] foo push foo://20130101T1200Z/hello/worlds/mars.txt.zst $FLOW_RUN_DIR/share/cycle/20130101T1200Z/hello/mars.txt # return-code=1, stderr= +[FAIL] foo: push: unknown action +[FAIL] ! foo://20130101T1200Z/hello/worlds/mars.txt.zst +[FAIL] ! hello/mars.txt (hello/mars.txt) +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-7" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}7/01/job.err" >"${TEST_KEY}.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/planet-n.tar.zst: bad rename-format: %(planet?maybedwarfplanet???)s: KeyError: 'planet?maybedwarfplanet???' +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-8" +sed '/^\[FAIL\] /!d' "${FILE_PREFIX}8/01/job.err" >"${TEST_KEY}.err" +# Iron out differences between error messages in Python versions: +sed -i 's/PatternError:/error:/g' "$TEST_KEY.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<'__ERR__' +[FAIL] foo://20130101T1200Z/planet-n.tar.zst: bad rename-parser: planet-(?P[MVEJSUN]\w+.txt: error: missing ), unterminated subpattern at position 7 +__ERR__ +TEST_KEY="$TEST_KEY_BASE-bad-archive-9" +sed '/^\[INFO\] [=!+]/!d; + s/\(t(init)=\)[^Z]*Z/\1YYYY-mm-DDTHH:MM:SSZ/; + s/\(dt(\(tran\|arch\))=\)[^s]*s/\1SSSSs/g' \ + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.out" >"$TEST_KEY.out" +file_cmp "$TEST_KEY.out" \ + "$TEST_SOURCE_DIR/$TEST_KEY_BASE-bad-9.out" "$TEST_KEY.out" +sed -e '/^\[FAIL\] /!d' \ + -e 's/^\(\[FAIL\] my-bad-command\) .*\( # return-code=1, stderr=\)$/\1\2/' \ + -e '/^\[FAIL\] \[my-bad-command\]/d' \ + -e 's/ \[compress.*]$//' \ + -e '/BASH_XTRACEFD/d' \ + "$FLOW_RUN_DIR/log/job/$CYCLE/archive_bad_9/01/job.err" >"$TEST_KEY.err" +file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <<__ERR__ +[FAIL] ! foo://20130101T1200Z/planet-n.tar.zst +[FAIL] ! hello/dark-matter.txt (hello/dark-matter.txt) +[FAIL] ! hello/earth.txt (hello/earth.txt) +[FAIL] ! hello/jupiter-moon-1.txt (hello/jupiter-moon-1.txt) +[FAIL] ! hello/jupiter.txt (hello/jupiter.txt) +[FAIL] ! hello/mars.txt (hello/mars.txt) +[FAIL] ! hello/mercury.txt (hello/mercury.txt) +[FAIL] ! hello/neptune-1.txt (hello/neptune-1.txt) +[FAIL] ! hello/planet-1.txt (hello/planet-1.txt) +[FAIL] ! hello/planet-2.txt (hello/planet-2.txt) +[FAIL] ! hello/planet-3.txt (hello/planet-3.txt) +[FAIL] ! hello/planet-4.txt (hello/planet-4.txt) +[FAIL] ! hello/planet-5.txt (hello/planet-5.txt) +[FAIL] ! hello/planet-6.txt (hello/planet-6.txt) +[FAIL] ! hello/planet-7.txt (hello/planet-7.txt) +[FAIL] ! hello/planet-8.txt (hello/planet-8.txt) +[FAIL] ! hello/planet-9.txt (hello/planet-9.txt) +[FAIL] ! hello/saturn.txt (hello/saturn.txt) +[FAIL] ! hello/spaceship-1.txt (hello/spaceship-1.txt) +[FAIL] ! hello/spaceship-2.txt (hello/spaceship-2.txt) +[FAIL] ! hello/spaceship-3.txt (hello/spaceship-3.txt) +[FAIL] ! hello/spaceship-4.txt (hello/spaceship-4.txt) +[FAIL] ! hello/spaceship-5.txt (hello/spaceship-5.txt) +[FAIL] ! hello/spaceship-6.txt (hello/spaceship-6.txt) +[FAIL] ! hello/spaceship-7.txt (hello/spaceship-7.txt) +[FAIL] ! hello/spaceship-8.txt (hello/spaceship-8.txt) +[FAIL] ! hello/spaceship-9.txt (hello/spaceship-9.txt) +[FAIL] ! hello/uranus.txt (hello/uranus.txt) +[FAIL] ! hello/venus.txt (hello/venus.txt) +[FAIL] my-bad-command # return-code=1, stderr= +__ERR__ + +#------------------------------------------------------------------------------- +purge +exit 0 diff --git a/t/rose-task-run/48-app-arch/app/archive/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive/rose-app.conf new file mode 100644 index 0000000000..6a85fd52cb --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive/rose-app.conf @@ -0,0 +1,69 @@ +mode=rose_arch + +[env] +FOO_SESSION=$ROSE_TASK_CYCLE_TIME.$CYLC_TASK_TRY_NUMBER +MY_UNKNOWN=unknown +MY_STUFF=stuff + +[file:$ROSE_DATAC/try.nl] +source=namelist:try + +[namelist:try] +n=$CYLC_TASK_TRY_NUMBER + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:earth.txt] +source=hello/earth.txt + +[arch:try.nl] +source=try.nl + +[arch:planet-n.tar.zst] +source=hello/planet-[1-9].txt + +[arch:spaceships/] +compress=zst +source=hello/spaceship-[1-9].txt + +[arch:stars/] +rename-format=%(cycle)s-%(name)s +source=star-[1-9].txt + +[arch:${MY_UNKNOWN}/${MY_STUFF}.pax] +rename-format=hello/%(cycle)s-%(name_head)s%(name_tail)s +rename-parser=^(?Pstuff)ing(?P-[1-9]\.txt)$ +source=stuffing-*.txt + +[arch:dark-matter.txt] +command-format=FOO_RC=$((2 - $CYLC_TASK_TRY_NUMBER)) foo put %(target)s %(sources)s +source=hello/dark-matter.txt + +[arch:organisms.tar.zst] +source=hello/organisms/ + +[arch:jupiter.txt] +command-format=foo put %(target)s %(sources)s; touch -r share/cycle/$ROSE_TASK_CYCLE_TIME/hello/t-ref share/cycle/$ROSE_TASK_CYCLE_TIME/hello/jupiter.txt +source=hello/jupiter.txt +update-check=mtime+size + +[arch:saturn.txt] +command-format=foo put %(target)s %(sources)s; touch -r share/cycle/$ROSE_TASK_CYCLE_TIME/hello/t-ref share/cycle/$ROSE_TASK_CYCLE_TIME/hello/saturn.txt +source=hello/saturn.txt + +[arch:uranus.txt] +command-format=foo put %(target)s %(sources)s; echo 'Hello Uranus and Moons' >share/cycle/$ROSE_TASK_CYCLE_TIME/hello/uranus.txt +source=hello/uranus.txt + +[arch:jupiter-moons.tar.zst] +source=hello/jupiter-moon-*.txt +rename-parser=^(?Phello/jupiter)-moon(?P-[1-9]\.txt)$ +rename-format=%(name_head)s%(name_tail)s +source-edit-format=sed 's/Hello/Greet/' %(in)s >%(out)s + +[arch:neptune-1.txt] +source=hello/neptune-1.txt +source-edit-format=sed 's/Hello/Greet/' %(in)s >%(out)s diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf new file mode 100644 index 0000000000..27467972c4 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_1/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(source)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:planet-n.tar.zst] +source=hello/planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf new file mode 100644 index 0000000000..f44b882db0 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_2/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +sources=planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf new file mode 100644 index 0000000000..2f12889168 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_3/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=$UNBOUND_PLANET-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf new file mode 100644 index 0000000000..a50f97c185 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_4/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(source)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:$UNKNOWN_DARK_PLANETS.tar.zst] +source=hello/planet-[1-9].txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf new file mode 100644 index 0000000000..368c8f05e1 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_5/rose-app.conf @@ -0,0 +1,13 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:mars.txt.zst] +source=hello/mars.txt + +# Too much curry last night +[arch:inner.tar.zst] +source=hello/venus.txt hello/mercurry.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf new file mode 100644 index 0000000000..5dfdee811f --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_6/rose-app.conf @@ -0,0 +1,9 @@ +mode=rose_arch + +[arch] +command-format=foo push %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/hello/worlds/ + +[arch:mars.txt.zst] +source=hello/mars.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf new file mode 100644 index 0000000000..b1f132118d --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_7/rose-app.conf @@ -0,0 +1,12 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +# The following format will fail: +rename-format=%(planet?maybedwarfplanet???)s +rename-parser=planet-(?P[MVEJSUN]\w+).txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf new file mode 100644 index 0000000000..272a17a8a8 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_8/rose-app.conf @@ -0,0 +1,12 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +rename-format=%(planet)s +# The following will fail Python regex compilation: +rename-parser=planet-(?P[MVEJSUN]\w+.txt diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command b/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command new file mode 100755 index 0000000000..7abc5524da --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_9/bin/my-bad-command @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +THIS=$(basename $0) +echo "[$THIS] $*" >&2 +exit 1 diff --git a/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf b/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf new file mode 100644 index 0000000000..8716d955dd --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/archive_bad_9/rose-app.conf @@ -0,0 +1,10 @@ +mode=rose_arch + +[arch] +command-format=foo put %(target)s %(sources)s +source-prefix=$ROSE_DATAC/ +target-prefix=foo://$ROSE_TASK_CYCLE_TIME/ + +[arch:planet-n.tar.zst] +source=hello/*.txt +source-edit-format=my-bad-command %(in)s %(out)s diff --git a/t/rose-task-run/48-app-arch/app/install/bin/my-install b/t/rose-task-run/48-app-arch/app/install/bin/my-install new file mode 100755 index 0000000000..4db149c402 --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/install/bin/my-install @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -eu +PREFIX= +if [[ ${ROSE_TASK_CYCLE_TIME:-} ]]; then + PREFIX="[$ROSE_TASK_CYCLE_TIME] " +fi +mkdir -p $ROSE_DATAC/hello +echo "${PREFIX}Hello Earth" >$ROSE_DATAC/hello/earth.txt +echo "${PREFIX}Hello Dark Matter" >$ROSE_DATAC/hello/dark-matter.txt +echo "${PREFIX}Hello Mars" >$ROSE_DATAC/hello/mars.txt +echo "${PREFIX}Hello Venus" >$ROSE_DATAC/hello/venus.txt +echo "${PREFIX}Hello mercury" >$ROSE_DATAC/hello/mercury.txt +for I in $(seq 1 9); do + echo "${PREFIX}Hello Planet $I" >$ROSE_DATAC/hello/planet-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Spaceship $I" >$ROSE_DATAC/hello/spaceship-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Star $I" >$ROSE_DATAC/star-$I.txt +done +for I in $(seq 1 9); do + echo "${PREFIX}Hello Stuff $I" >$ROSE_DATAC/stuffing-$I.txt +done +mkdir -p $ROSE_DATAC/hello/organisms/{plants,animals} +for NAME in lily jasmine holly iris daisy; do + echo "${PREFIX}Hello $NAME" >$ROSE_DATAC/hello/organisms/plants/$NAME.txt +done +for NAME in elephant tiger goose crocodile; do + echo "${PREFIX}Hello $NAME" >$ROSE_DATAC/hello/organisms/animals/$NAME.txt +done +touch $ROSE_DATAC/hello/t-ref +touch -t 201212202012.00 $ROSE_DATAC/hello/t-ref +echo "${PREFIX}Hello jupiter" >$ROSE_DATAC/hello/jupiter.txt +touch -t 201111202011.00 $ROSE_DATAC/hello/jupiter.txt +echo "${PREFIX}Hello saturn" >$ROSE_DATAC/hello/saturn.txt +touch -t 201111202011.00 $ROSE_DATAC/hello/saturn.txt +echo "${PREFIX}Hello uranus" >$ROSE_DATAC/hello/uranus.txt +echo "${PREFIX}Hello Io" >$ROSE_DATAC/hello/jupiter-moon-1.txt +echo "${PREFIX}Hello Triton" >$ROSE_DATAC/hello/neptune-1.txt diff --git a/t/rose-task-run/48-app-arch/app/install/rose-app.conf b/t/rose-task-run/48-app-arch/app/install/rose-app.conf new file mode 100644 index 0000000000..4f2ceb8dbe --- /dev/null +++ b/t/rose-task-run/48-app-arch/app/install/rose-app.conf @@ -0,0 +1,2 @@ +[command] +default=my-install diff --git a/t/rose-task-run/48-app-arch/bin/foo b/t/rose-task-run/48-app-arch/bin/foo new file mode 100755 index 0000000000..532940fe39 --- /dev/null +++ b/t/rose-task-run/48-app-arch/bin/foo @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +set -eu +FOO=$(basename $0) +FOO_SESSION=${FOO_SESSION:-0} +FOO_RC=${FOO_RC:-0} +echo "$FOO $* # rc=$FOO_RC" 1>>"$FOO.log.$FOO_SESSION" +ACTION=$1 +shift 1 +if [[ $ACTION != 'put' ]]; then + echo "foo: $ACTION: unknown action" >&2 + exit 1 +fi +TARGET=${1#foo://} +mkdir -p $(dirname foo/$TARGET) +shift 1 +rsync -a "$@" foo/$TARGET +exit $FOO_RC diff --git a/t/rose-task-run/48-app-arch/flow.cylc b/t/rose-task-run/48-app-arch/flow.cylc new file mode 100644 index 0000000000..0787739379 --- /dev/null +++ b/t/rose-task-run/48-app-arch/flow.cylc @@ -0,0 +1,49 @@ +#!jinja2 +{% set bad_nums=["1", "2", "3", "4", "5", "6", "7", "8", "9"] %} +[cylc] + UTC mode=True + [[events]] + abort on timeout=True + timeout=PT1M +[scheduling] + initial cycle point=20130101 + final cycle point=20130102 + [[dependencies]] + [[[T12]]] + graph=""" +{% for bad_num in bad_nums %} +install => archive_bad_{{bad_num}} +{% endfor %} +""" + [[[T00, T12]]] + graph=""" +install => archive +archive[-PT12H] => archive +""" + +[runtime] + [[root]] + script=""" +RC=0 +rose task-run --debug || RC=$? +""" + [[[job]]] + execution retry delays=PT1S + execution time limit=PT1M + [[archive]] + post-script=""" +{ +sqlite3 .rose-arch.db 'SELECT * FROM targets;' | sort +sqlite3 .rose-arch.db 'SELECT * FROM sources;' | sort +} >rose-arch-db-$CYLC_TASK_TRY_NUMBER.out +((RC==0)) +""" + [[ARCHIVE_BAD]] + script = """rose task-run || true; RC=$?""" + [[[job]]] + execution retry delays=P0Y +{% for bad_num in bad_nums %} + [[archive_bad_{{bad_num}}]] + inherit=ARCHIVE_BAD +{% endfor %} + [[install]] diff --git a/t/rose-task-run/48-app-arch/rose-suite.conf b/t/rose-task-run/48-app-arch/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/rose-task-run/49-app-arch-zstd-mp.t b/t/rose-task-run/49-app-arch-zstd-mp.t new file mode 100644 index 0000000000..eba331e14a --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp.t @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# Copyright (C) British Crown (Met Office) & Contributors. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# Test "rose_arch" built-in application, archive with optional sources. +#------------------------------------------------------------------------------- +. "$(dirname "$0")/test_header" + + +#------------------------------------------------------------------------------- +tests 4 +#------------------------------------------------------------------------------- +# Run the suite, and wait for it to complete +export CYLC_CONF_PATH= +export ROSE_CONF_PATH= + +get_reg +run_pass "${TEST_KEY_BASE}-install" \ + cylc install \ + "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" \ + --workflow-name="${FLOW}" \ + --no-run-name +run_pass "${TEST_KEY_BASE}-play" \ + cylc play \ + "${FLOW}" \ + --abort-if-any-task-fails \ + --host=localhost \ + --no-detach \ + --debug +#------------------------------------------------------------------------------- +TEST_KEY="${TEST_KEY_BASE}-job.status" +file_grep "${TEST_KEY}-archive-01" \ + 'CYLC_JOB_EXIT=SUCCEEDED' \ + "${FLOW_RUN_DIR}/log/job/1/archive/01/job.status" +TEST_KEY="${TEST_KEY_BASE}-find" +(cd "${FLOW_RUN_DIR}/share/backup" && find . -type f) | sort >"${TEST_KEY}.out" +file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<'__FIND__' +./archive.d/2016.txt.zst +./archive.d/whatever.tar.zst +__FIND__ +TEST_KEY="${TEST_KEY_BASE}-zstd-command" +#------------------------------------------------------------------------------- +purge +exit 0 \ No newline at end of file diff --git a/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf new file mode 100644 index 0000000000..a4a65ac5b1 --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp/app/archive/rose-app.conf @@ -0,0 +1,19 @@ +mode=rose_arch + +[file:$ROSE_SUITE_DIR/share/backup/archive.d/] +mode=mkdir + +[arch] +command-format=cp -pr %(sources)s %(target)s +target-prefix=$ROSE_SUITE_DIR/share/backup/ + +[arch:archive.d/2016.txt.zst] +compress=zstd +compress-threads=4 +source-prefix=work/1/$ROSE_TASK_NAME/ +source=2016.txt.zst + +[arch:archive.d/whatever.tar.zst] +compress-threads=0 +source-prefix=work/1/$ROSE_TASK_NAME/ +source=whatever.tar.zst diff --git a/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc new file mode 100644 index 0000000000..7551ffebda --- /dev/null +++ b/t/rose-task-run/49-app-arch-zstd-mp/flow.cylc @@ -0,0 +1,21 @@ +#!jinja2 +[cylc] + UTC mode=True + [[events]] + timeout=PT1M + abort on timeout=True +[scheduling] + [[dependencies]] + graph=archive + +[runtime] + [[archive]] + script=""" +zstd --rm -T4 -f - <<<'MMXVI' >'2016.txt.zst' +mkdir 'whatever' +echo "I don't care." >'whatever/idontcare.txt' +echo "You may be right." >'whatever/youmayberight.txt' +tar -I "zstd --rm -T0" -cf 'whatever.tar.zst' 'whatever' +rm -fr 'whatever' +rose task-run --debug +""" diff --git a/t/rose-task-run/49-app-arch-zstd-mp/rose-suite.conf b/t/rose-task-run/49-app-arch-zstd-mp/rose-suite.conf new file mode 100644 index 0000000000..e69de29bb2