diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16c6bf2ac1..3488135725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,7 +147,7 @@ jobs: - name: Install Rose working-directory: rose run: | - pip install ."[tests,docs,rose-edit${{ (startsWith(matrix.os, 'ubuntu') && ',graph,rosa') || '' }}]" + pip install ."[tests,docs,rose-edit,graph,rosa,disco]" yarn install - name: Install Cylc @@ -259,7 +259,7 @@ jobs: - name: Install Rose run: | - pip install -e .[docs,graph,edit] --upgrade-strategy='only-if-needed' + pip install -e .[docs,graph,edit,disco] --upgrade-strategy='only-if-needed' - name: Install Cylc uses: cylc/release-actions/install-cylc-components@v1 diff --git a/changes.d/2985.feat.md b/changes.d/2985.feat.md new file mode 100644 index 0000000000..4019793c1e --- /dev/null +++ b/changes.d/2985.feat.md @@ -0,0 +1,3 @@ +rosie: The `rosie disco` server has been restored restored to Rose 2. + +It is now available as the pip package `rose[disco]` or the conda-forge package `metomi-rose`. diff --git a/conda-environment.yml b/conda-environment.yml index b78c9cfe80..d5f3448e62 100644 --- a/conda-environment.yml +++ b/conda-environment.yml @@ -12,10 +12,10 @@ dependencies: - metomi-isodatetime >=1!3.0,<1!4.0 - psutil >=5.6.0 - requests - - sqlalchemy >=1,<2 - # - tornado (rosie disco is temporally disabled) # output: metomi-rose - pygraphviz >1.0,!=1.8 - pygobject >=3.50.0 - gtk3 >=3.24.43 + - sqlalchemy >=1,<2 + - tornado diff --git a/metomi/rose/rose.py b/metomi/rose/rose.py index 7f6d7c5961..5c37057c85 100644 --- a/metomi/rose/rose.py +++ b/metomi/rose/rose.py @@ -113,8 +113,6 @@ def iter_entry_points(name: str): 'This command has been replaced by: "cylc stop".', ('rose', 'suite-stop'): 'This command has been replaced by: "cylc stop".', - ('rosie', 'disco'): - 'Rosie Disco has been disabled pending fixes at a later release.', ('rosie', 'go'): 'This command has been removed.', } diff --git a/metomi/rosie/ws.py b/metomi/rosie/ws.py index 5038117dec..6feded76ed 100644 --- a/metomi/rosie/ws.py +++ b/metomi/rosie/ws.py @@ -38,13 +38,13 @@ from pathlib import Path import pwd import signal +import sys from time import sleep import jinja2 from tornado.ioloop import IOLoop, PeriodicCallback import tornado.log import tornado.web -import tornado.wsgi from metomi.isodatetime.data import get_timepoint_from_seconds_since_unix_epoch from metomi.rose import __version__ as ROSE_VERSION @@ -104,7 +104,13 @@ def __init__(self, service_root_mode=False, *args, **kwargs): ) db_url_map = {} - for key, node in rose_conf.get(["rosie-db"]).value.items(): + rosie_db = rose_conf.get(["rosie-db"]) + if not rosie_db or not rosie_db.value: + sys.exit( + 'Please configure one or more rosie prefix in' + ' "rose.conf[rosie-db]".' + ) + for key, node in rosie_db.value.items(): if key.startswith("db.") and key[3:]: db_url_map[key[3:]] = node.value self.db_url_map = db_url_map diff --git a/metomi/rosie/ws_client_cli.py b/metomi/rosie/ws_client_cli.py index dda80019e6..e7807b44af 100644 --- a/metomi/rosie/ws_client_cli.py +++ b/metomi/rosie/ws_client_cli.py @@ -154,7 +154,12 @@ def list_local_suites(): * `X` means that the suite is checked out but is corrupted. ''', ).add_my_options( - "no_headers", "prefixes", "print_format", "reverse", "sort", "user" + "no_headers", + "prefixes", + "print_format", + "reverse", + "sort", + "user", ) opt_parser.modify_option( 'verbosity', @@ -390,7 +395,7 @@ def _display_maps(opts, ws_client, dict_rows, url=None): if "%" + key in opts.print_format: keylist.append(key) - if not opts.no_headers: + if getattr(opts, 'no_pretty_mode', True) and not opts.no_headers: dummy_row = {} for key in all_keys: dummy_row[key] = key @@ -415,8 +420,21 @@ def _display_maps(opts, ws_client, dict_rows, url=None): if url is not None: report(URLEvent(url + "\n"), prefix="") else: - cols = [x.replace('%', '') for x in opts.print_format.split()] - _rows = [[_dict[col] for col in cols] for _dict in dict_rows[2:]] + cols = [ + col[1:] + for col in opts.print_format.split() + # NOTE: strip off anything which isn't a valid field + if col[0] == '%' + ] + _rows = [ + # NOTE: tolerate missing fields - column format used as placeholder + # (e.g, "access-list" might not be defined for all rows) + # NOTE: convert to string (e.g, "access-list" is a list) + [str(_dict.get(col, f"%{col}")) for col in cols] + for _dict in dict_rows + ] + if opts.no_headers: + cols = None # turn header row off in table try: print(table(_rows, header=cols, max_width=terminal_cols)) except UnicodeDecodeError: diff --git a/pyproject.toml b/pyproject.toml index 53c9cf227a..9f88545a6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ dependencies = [ "metomi-isodatetime==1!3.*", "psutil>=5.6.0", "requests", - "sqlalchemy==1.*", ] [project.urls] @@ -96,22 +95,22 @@ tutorials = [ "cylc-flow[tutorials]>=8.6.1" ] rosa = [] -# disco = [ -# TODO: rosie disco has been disabled due to the removal of WSGI support at -# Tornado 6. -# tornado -# ] edit = [ "pygobject>=3.50.0", ] +disco = [ + "sqlalchemy==1.*", + "tornado", +] all = [ "metomi-rose[lint]", "metomi-rose[docs]", "metomi-rose[graph]", "metomi-rose[tests]", + "metomi-rose[tutorials]", "metomi-rose[rosa]", "metomi-rose[edit]", - "metomi-rose[tutorials]", + "metomi-rose[disco]", ] [project.scripts] @@ -145,7 +144,7 @@ rosa = "metomi.rose.rose:rosa" "checkout" = "metomi.rosie.vc:checkout" "create" = "metomi.rosie.vc:create" "delete" = "metomi.rosie.vc:delete" -# "disco" = "metomi.rosie.ws:main [disco]" +"disco" = "metomi.rosie.ws:main [disco]" "graph" = "metomi.rosie.graph:main [graph]" "hello" = "metomi.rosie.ws_client_cli:hello" "id" = "metomi.rosie.suite_id:main" diff --git a/sphinx/installation.rst b/sphinx/installation.rst index 1c65195052..1fcf248ad4 100644 --- a/sphinx/installation.rst +++ b/sphinx/installation.rst @@ -92,6 +92,22 @@ Rose with the ``graph`` optional dependency:: hosting services such as GitHub. Note Git is not automatically installed by the metomi-rose conda. +Rosie +""""" + +If installing via ``pip``, you will need to install the ``rosie`` optional +dependency:: + + $ pip install metomi-rose[rosie] + +Rosa +"""" + +If installing via ``pip``, you will need to install the ``rosa`` optional +dependency:: + + $ pip install metomi-rose[rosa] + Configuring Rose ---------------- diff --git a/t/lib/bash/test_header b/t/lib/bash/test_header index 0b03e2a0b2..07a5915f39 100644 --- a/t/lib/bash/test_header +++ b/t/lib/bash/test_header @@ -402,7 +402,7 @@ rose_ws_init() { TEST_ROSE_WS_PORT="${PORT}" if port_is_busy "${TEST_ROSE_WS_PORT}"; then pass "${TEST_KEY}" - TEST_ROSE_WS_URL="http://${HOSTNAME}:${TEST_ROSE_WS_PORT}/${NS}-${UTIL}" + TEST_ROSE_WS_URL="http://$(hostname -f):${TEST_ROSE_WS_PORT}/${NS}-${UTIL}" else fail "${TEST_KEY}" rose_ws_kill diff --git a/t/rosie-disco/00-basic.t b/t/rosie-disco/00-basic.t index 65a59f47ee..216f094a33 100755 --- a/t/rosie-disco/00-basic.t +++ b/t/rosie-disco/00-basic.t @@ -24,8 +24,22 @@ if ! python3 -c 'import tornado, sqlalchemy' 2>/dev/null; then skip_all '"tornado" or "sqlalchemy" not installed' fi #------------------------------------------------------------------------------- -tests 23 +tests 27 #------------------------------------------------------------------------------- +# Test without config +TEST_KEY="${TEST_KEY_BASE}-no-config" +export ROSE_CONF_PATH=$PWD +run_fail "${TEST_KEY}-1" rosie disco start +cat >rose.conf <<__ROSE_CONF__ +[rosie-id] +__ROSE_CONF__ +run_fail "${TEST_KEY}-2" rosie disco start +file_cmp "${TEST_KEY}-1.err" "${TEST_KEY}-1.err" <<__STDERR__ +Please configure one or more rosie prefix in "rose.conf[rosie-db]". +__STDERR__ +file_cmp "${TEST_KEY}-the-same" "${TEST_KEY}-1.out" "${TEST_KEY}-2.out" +#------------------------------------------------------------------------------- +# create valid config mkdir svn svnadmin create svn/foo SVN_URL=file://$PWD/svn/foo @@ -39,14 +53,12 @@ local-copy-root=$PWD/roses prefix-default=foo prefix-location.foo=$SVN_URL __ROSE_CONF__ -export ROSE_CONF_PATH=$PWD rosa db-create -q #------------------------------------------------------------------------------- rose_ws_init 'rosie' 'disco' if [[ -z "${TEST_ROSE_WS_PORT}" ]]; then exit 1 fi - URL_FOO="${TEST_ROSE_WS_URL}/foo/" URL_FOO_S="${URL_FOO}search?" URL_FOO_Q="${URL_FOO}query?" diff --git a/t/rosie-lookup/00-basic.t b/t/rosie-lookup/00-basic.t index 6e59569596..f28d0ace9e 100755 --- a/t/rosie-lookup/00-basic.t +++ b/t/rosie-lookup/00-basic.t @@ -26,7 +26,12 @@ if ! python3 -c 'import tornado, sqlalchemy' 2>/dev/null; then skip_all '"tornado" or "sqlalchemy" not installed' fi -tests 75 +if [[ "$OSTYPE" == darwin* ]]; then + # tests not yet passing on GH Mac OS runner + # see: https://github.com/metomi/rose/pull/2985 + skip_all 'Does not work on MacOS' +fi +tests 91 #------------------------------------------------------------------------------- # Setup Rose site/user configuration for the tests. export TZ='UTC' @@ -145,7 +150,7 @@ while port_is_busy "${PORT}"; do done cat >conf/opt/rose-port.conf <<__ROSE_CONF__ [rosie-id] -prefix-ws.foo=http://${HOSTNAME}:${PORT}/foo +prefix-ws.foo=http://$(hostname -f):${PORT}/foo __ROSE_CONF__ rosie disco 'start' "${PORT}" \ 0<'/dev/null' 1>'rosie-disco.out' 2>'rosie-disco.err' & @@ -160,79 +165,79 @@ fi #------------------------------------------------------------------------------- TEST_KEY=$TEST_KEY_BASE-bad-prefix -run_fail "$TEST_KEY" timeout 10 rosie lookup --prefix=bar poetry +run_fail "$TEST_KEY" timeout 10 rosie lookup --no-pretty --prefix=bar poetry file_cmp "$TEST_KEY.out" "$TEST_KEY.out" foo-aa001/trunk@2 roses poetry Roses are Red,... foo-aa000/trunk@4 iris eye pad Should have gone to ... -url: http://$HOSTNAME:$PORT/foo/query?q=and+description+contains+a&all_revs=1 +url: http://$(hostname -f):$PORT/foo/query?q=and+description+contains+a&all_revs=1 __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" foo-aa001/trunk@2 roses poetry Roses are Red,... = foo-aa001/trunk@3 roses poetry Roses are Red, Violets are Blue,... -url: http://$HOSTNAME:$PORT/foo/query?q=and+title+contains+a&q=and+project+eq+poetry&all_revs=1 +url: http://$(hostname -f):$PORT/foo/query?q=and+title+contains+a&q=and+project+eq+poetry&all_revs=1 __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" foo-aa001/trunk@2 roses poetry Roses are Red,... = foo-aa001/trunk@3 roses poetry Roses are Red, Violets are Blue,... = foo-aa002/trunk@5 aphids eat roses Eat all the roses! -url: http://$HOSTNAME:$PORT/foo/query?q=and+title+contains+Roses&q=or+owner+eq+roses&all_revs=1 +url: http://$(hostname -f):$PORT/foo/query?q=and+title+contains+Roses&q=or+owner+eq+roses&all_revs=1 __OUT__ file_cmp "$TEST_KEY.err" "$TEST_KEY.err" │ foo-aa001/trunk@2 │ roses │ poetry │ Roses are Red,... │ +├───────┼───────────────────┼────────┼───────────┼─────────────────────────────┤ +│ = │ foo-aa001/trunk@3 │ roses │ poetry │ Roses are Red, Violets are │ +│ │ │ │ │ Blue,... │ +└───────┴───────────────────┴────────┴───────────┴─────────────────────────────┘ +__OUT__ +file_cmp "$TEST_KEY.err" "$TEST_KEY.err" '/dev/null' -rm -f ~/.metomi/rosie-disco-${HOSTNAME:-0.0.0.0}-${PORT}* +rm -f ~/.metomi/rosie-disco-$(hostname -f)-${PORT}* exit diff --git a/t/rosie-lookup/01-multi.t b/t/rosie-lookup/01-multi.t index 3aa97b0e22..94d08a0e8b 100755 --- a/t/rosie-lookup/01-multi.t +++ b/t/rosie-lookup/01-multi.t @@ -17,13 +17,18 @@ # You should have received a copy of the GNU General Public License # along with Rose. If not, see . #------------------------------------------------------------------------------- -# Basic multi-source tests for "rosie lookup". +# Basic multi-source tests for " rosie lookup". #------------------------------------------------------------------------------- . $(dirname $0)/test_header #------------------------------------------------------------------------------- if ! python3 -c 'import tornado, sqlalchemy' 2>/dev/null; then skip_all '"tornado" or "sqlalchemy" not installed' fi +if [[ "$OSTYPE" == darwin* ]]; then + # tests not yet passing on GH Mac OS runner + # see: https://github.com/metomi/rose/pull/2985 + skip_all 'Does not work on MacOS' +fi tests 18 #------------------------------------------------------------------------------- # Setup Rose site/user configuration for the tests. @@ -94,8 +99,8 @@ while port_is_busy "${PORT}"; do done cat >'conf/opt/rose-port.conf' <<__ROSE_CONF__ [rosie-id] -prefix-ws.bar=http://${HOSTNAME}:${PORT}/bar -prefix-ws.foo=http://${HOSTNAME}:${PORT}/foo +prefix-ws.bar=http://$(hostname -f):${PORT}/bar +prefix-ws.foo=http://$(hostname -f):${PORT}/foo __ROSE_CONF__ rosie disco 'start' "${PORT}" \ 0<'/dev/null' 1>'rosie-disco.out' 2>'rosie-disco.err' & @@ -112,34 +117,34 @@ set +e #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}-search-both" -run_pass "${TEST_KEY}" rosie lookup 'bus' +run_pass "${TEST_KEY}" rosie lookup --no-pretty 'bus' file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<__OUT__ local suite owner project title bar-aa000/trunk@1 beth bus Down by the bus station -url: http://${HOSTNAME}:${PORT}/bar/search?s=bus +url: http://$(hostname -f):${PORT}/bar/search?s=bus local suite owner project title foo-aa000/trunk@1 billy bus Wheels on the bus -url: http://${HOSTNAME}:${PORT}/foo/search?s=bus +url: http://$(hostname -f):${PORT}/foo/search?s=bus __OUT__ file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" 'conf/opt/rose-default.conf' <<__ROSE_CONF__ [rosie-id] prefixes-ws-default=foo __ROSE_CONF__ -run_pass "${TEST_KEY}" rosie lookup 'bus' +run_pass "${TEST_KEY}" rosie lookup --no-pretty 'bus' rm 'conf/opt/rose-default.conf' file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<__OUT__ local suite owner project title foo-aa000/trunk@1 billy bus Wheels on the bus -url: http://${HOSTNAME}:${PORT}/foo/search?s=bus +url: http://$(hostname -f):${PORT}/foo/search?s=bus __OUT__ file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" '/dev/null' -rm -f ~/.metomi/rosie-disco-${HOSTNAME:-0.0.0.0}-${PORT}* +rm -f ~/.metomi/rosie-disco-$(hostname -f)-${PORT}* exit diff --git a/t/rosie-lookup/02-unicode.t b/t/rosie-lookup/02-unicode.t index cce4ab9dcc..a5b7a1fe41 100755 --- a/t/rosie-lookup/02-unicode.t +++ b/t/rosie-lookup/02-unicode.t @@ -79,7 +79,7 @@ while port_is_busy "${PORT}"; do done cat >'conf/opt/rose-port.conf' <<__ROSE_CONF__ [rosie-id] -prefix-ws.foo=http://${HOSTNAME}:${PORT}/foo +prefix-ws.foo=http://$(hostname -f):${PORT}/foo __ROSE_CONF__ rosie disco 'start' "${PORT}" \ 0<'/dev/null' 1>'rosie-disco.out' 2>'rosie-disco.err' & @@ -96,15 +96,15 @@ set +e #------------------------------------------------------------------------------- TEST_KEY="${TEST_KEY_BASE}" -run_pass "${TEST_KEY}" rosie lookup 'euro' +run_pass "${TEST_KEY}" rosie lookup --no-pretty 'euro' file_cmp "${TEST_KEY}.out" "${TEST_KEY}.out" <<__OUT__ local suite owner project title foo-aa000/trunk@1 billy euro The euro symbol € requires unicode -url: http://${HOSTNAME}:${PORT}/foo/search?s=euro +url: http://$(hostname -f):${PORT}/foo/search?s=euro __OUT__ file_cmp "${TEST_KEY}.err" "${TEST_KEY}.err" <'/dev/null' #------------------------------------------------------------------------------- kill "${ROSA_WS_PID}" wait 2>'/dev/null' -rm -f ~/.metomi/rosie-disco-${HOSTNAME:-0.0.0.0}-${PORT}* +rm -f "~/.metomi/rosie-disco-$(hostname -f)-${PORT}"* exit