From 0f760f63eb70dd5832e4961097bb712eb98efdfe Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Wed, 27 May 2020 17:23:56 -0500 Subject: [PATCH 01/16] bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8e750a7..d401442 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = htmap -version = 0.6.0 +version = 0.7.0 author = Josh Karpel author_email = josh.karpel@gmail.com description = High-Throughput Computing in Python, powered by HTCondor From 86f16be7df9f8fa143b8922e5ca9c9fab57a680f Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Wed, 27 May 2020 17:24:46 -0500 Subject: [PATCH 02/16] make failed map submission safer --- htmap/maps.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/htmap/maps.py b/htmap/maps.py index d5128d4..b535d24 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -946,8 +946,18 @@ def _submit(self, components: Optional[Iterable[int]] = None) -> None: # if we fail to write the cluster id for any reason, abort the submit try: htio.append_cluster_id(self._map_dir, new_cluster_id) - except BaseException as e: - condor.get_schedd().act(htcondor.JobAction.Remove, f"ClusterId=={new_cluster_id}") + except BaseException as write_exception: + logger.exception( + f"Failed to write new cluster id {new_cluster_id} for map {self.tag}, aborting submission" + ) + try: + condor.get_schedd().act(htcondor.JobAction.Remove, f"ClusterId=={new_cluster_id}") + except BaseException as remove_exception: + logger.exception( + f"Was not able to abort submission of cluster id {new_cluster_id} for map {self.tag}" + ) + raise remove_exception + raise write_exception logger.debug( f"Submitted {len(sliced_itemdata)} components (out of {self._num_components}) from map {self.tag}" From 2c7b3610b557ca8a23ff1d9c75dc0f587ec8bb81 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Wed, 8 Jul 2020 09:27:02 -0500 Subject: [PATCH 03/16] loosen dependency version requirements --- MANIFEST.in | 1 - setup.cfg | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 72a3300..72ca36c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1 @@ include LICENSE README.md -include requirements.txt diff --git a/setup.cfg b/setup.cfg index d401442..e6f2438 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,10 +30,10 @@ python_requires = >=3.6 include_package_data = True install_requires = htcondor >= 8.8 - cloudpickle ~= 1.4 - toml ~= 0.10 - tqdm ~= 4.46 - click ~= 7.0 + cloudpickle >= 1.4 + toml >= 0.10 + tqdm >= 4.46 + click >= 7.0 click-didyoumean halo importlib-metadata ~= 1.0; python_version < "3.8" From ec970cb913b63bdc6a4be5aa26acead66ec63c3d Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Fri, 14 Aug 2020 10:52:30 -0500 Subject: [PATCH 04/16] Tests pass, html stable produced --- htmap/maps.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 2 ++ 2 files changed, 64 insertions(+) diff --git a/htmap/maps.py b/htmap/maps.py index 6c3b723..3ee1dde 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -142,6 +142,68 @@ def load(cls, tag: str) -> "Map": return cls(tag=tag, map_dir=map_dir,) + def _repr_html_(self): + + table = self._repr_html_table() + grid = "" + + both = [ + "", + "", + "", + "", + "", + "
", + table, + "", + grid, + "
", + ] + return "\n".join(both) + + def _repr_html_table(self): + sc = collections.Counter(self.component_statuses) + row: Dict[str, Union[str, int, float]] = {"tag": self.tag} + + for status in state.ComponentStatus.display_statuses(): + row[status.value.lower()] = sc[status] + + tag = row["tag"] + held = row["held"] + errored = row["errored"] + idle = row["idle"] + running = row["running"] + completed = row["completed"] + + local_data = utils.num_bytes_to_str(self.local_data) + max_memory = utils.num_bytes_to_str(max(self.memory_usage) * 1024 * 1024) + max_runtime = str(max(self.runtime)) + total_runtime = str(sum(self.runtime, datetime.timedelta())) + + table = [ + "", + " ", + " ", + " ", + " ", + " ".format( + tag, + held, + errored, + idle, + running, + completed, + local_data, + max_memory, + max_runtime, + total_runtime, + ), + " ", + "
TAG HELD ERRORED IDLE RUNNING COMPLETED Local Data Max Memory Max Runtime Total Runtime
{} {} {} {} {} {} {} {} {} {}
", + ] + + return "\n".join(table) + def __repr__(self): return f"{self.__class__.__name__}(tag = {self.tag})" diff --git a/setup.cfg b/setup.cfg index c7efa4d..ef98376 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,8 @@ install_requires = cloudpickle>=1.4 halo>=0.0.30 htcondor>=8.8 + ipywidgets + jupyterlab>2 toml>=0.10 tqdm>=4.46 importlib-metadata>=1.0;python_version < "3.8" From b5615b4653e025a247200ab715df4a8a610eacf7 Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Mon, 17 Aug 2020 12:27:53 -0500 Subject: [PATCH 05/16] Uodated maps for html representation divided into header and grid --- htmap/maps.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/htmap/maps.py b/htmap/maps.py index 3ee1dde..48c2b55 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -161,7 +161,10 @@ def _repr_html_(self): ] return "\n".join(both) - def _repr_html_table(self): + def _repr_header_(self): + return " TAG HELD ERRORED IDLE RUNNING COMPLETED Local Data Max Memory Max Runtime Total Runtime " + + def _repr_grid_(self): sc = collections.Counter(self.component_statuses) row: Dict[str, Union[str, int, float]] = {"tag": self.tag} @@ -180,24 +183,27 @@ def _repr_html_table(self): max_runtime = str(max(self.runtime)) total_runtime = str(sum(self.runtime, datetime.timedelta())) + return " {} {} {} {} {} {} {} {} {} {} ".format( + tag, + held, + errored, + idle, + running, + completed, + local_data, + max_memory, + max_runtime, + total_runtime, + ) + + def _repr_html_table(self): table = [ "", " ", - " ", + " {}".format(self._repr_header_()), " ", " ", - " ".format( - tag, - held, - errored, - idle, - running, - completed, - local_data, - max_memory, - max_runtime, - total_runtime, - ), + " {}".format(self._repr_grid_()), " ", "
TAG HELD ERRORED IDLE RUNNING COMPLETED Local Data Max Memory Max Runtime Total Runtime
{} {} {} {} {} {} {} {} {} {}
", ] From af663d3b615e9124d3e569e8f6a57b713a2f7f74 Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Thu, 20 Aug 2020 00:27:07 -0500 Subject: [PATCH 06/16] changes made based on PR --- htmap/maps.py | 63 +++++++++++++++++++-------------------------------- setup.cfg | 1 - 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/htmap/maps.py b/htmap/maps.py index ea892c5..9ac58f8 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -143,67 +143,50 @@ def load(cls, tag: str) -> "Map": return cls(tag=tag, map_dir=map_dir,) def _repr_html_(self): - - table = self._repr_html_table() - grid = "" - - both = [ - "", - "", - "", - "", - "", - "
", - table, - "", - grid, - "
", - ] - return "\n".join(both) + return self._repr_html_table() def _repr_header_(self): return " TAG HELD ERRORED IDLE RUNNING COMPLETED Local Data Max Memory Max Runtime Total Runtime " def _repr_grid_(self): sc = collections.Counter(self.component_statuses) - row: Dict[str, Union[str, int, float]] = {"tag": self.tag} - - for status in state.ComponentStatus.display_statuses(): - row[status.value.lower()] = sc[status] - tag = row["tag"] - held = row["held"] - errored = row["errored"] - idle = row["idle"] - running = row["running"] - completed = row["completed"] + tag = self.tag + held = sc["held"] + errored = sc["errored"] + idle = sc["idle"] + running = sc["running"] + completed = sc["completed"] local_data = utils.num_bytes_to_str(self.local_data) max_memory = utils.num_bytes_to_str(max(self.memory_usage) * 1024 * 1024) max_runtime = str(max(self.runtime)) total_runtime = str(sum(self.runtime, datetime.timedelta())) - return " {} {} {} {} {} {} {} {} {} {} ".format( - tag, - held, - errored, - idle, - running, - completed, - local_data, - max_memory, - max_runtime, - total_runtime, + return "".join( + f" {h} " + for h in [ + tag, + held, + errored, + idle, + running, + completed, + local_data, + max_memory, + max_runtime, + total_runtime, + ] ) def _repr_html_table(self): table = [ "", " ", - " {}".format(self._repr_header_()), + " %s" % (self._repr_header_()), " ", " ", - " {}".format(self._repr_grid_()), + " %s" % (self._repr_grid_()), " ", "
", ] diff --git a/setup.cfg b/setup.cfg index ef98376..8eedb99 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,6 @@ install_requires = cloudpickle>=1.4 halo>=0.0.30 htcondor>=8.8 - ipywidgets jupyterlab>2 toml>=0.10 tqdm>=4.46 From 212d95a9d16e292d354fa3a8bb021a473c342206 Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Thu, 20 Aug 2020 10:04:34 -0500 Subject: [PATCH 07/16] Pair programming Josh PR fixes --- htmap/maps.py | 34 +++++++++++++++---------------- setup.cfg | 3 ++- tests/integration/test_widgets.py | 22 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 tests/integration/test_widgets.py diff --git a/htmap/maps.py b/htmap/maps.py index 9ac58f8..d3e78ee 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -146,32 +146,32 @@ def _repr_html_(self): return self._repr_html_table() def _repr_header_(self): - return " TAG HELD ERRORED IDLE RUNNING COMPLETED Local Data Max Memory Max Runtime Total Runtime " + return " TAG " + "".join( + f" {h} " + for h in [ + *state.ComponentStatus.display_statuses(), + "Local Data", + "Max Memory", + "Max Runtime", + "Total Runtime", + ] + ) def _repr_grid_(self): sc = collections.Counter(self.component_statuses) - tag = self.tag - held = sc["held"] - errored = sc["errored"] - idle = sc["idle"] - running = sc["running"] - completed = sc["completed"] - local_data = utils.num_bytes_to_str(self.local_data) max_memory = utils.num_bytes_to_str(max(self.memory_usage) * 1024 * 1024) max_runtime = str(max(self.runtime)) total_runtime = str(sum(self.runtime, datetime.timedelta())) - return "".join( + return f" {self.tag} " + "".join( f" {h} " for h in [ - tag, - held, - errored, - idle, - running, - completed, + *[ + sc[component_state] + for component_state in state.ComponentStatus.display_statuses() + ], local_data, max_memory, max_runtime, @@ -183,10 +183,10 @@ def _repr_html_table(self): table = [ "", " ", - " %s" % (self._repr_header_()), + f" {self._repr_header_()}", " ", " ", - " %s" % (self._repr_grid_()), + f" {self._repr_grid_()}", " ", "
", ] diff --git a/setup.cfg b/setup.cfg index 8eedb99..69deaa9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,6 @@ install_requires = cloudpickle>=1.4 halo>=0.0.30 htcondor>=8.8 - jupyterlab>2 toml>=0.10 tqdm>=4.46 importlib-metadata>=1.0;python_version < "3.8" @@ -58,6 +57,8 @@ docs = tests = codecov coverage + ipywidgets + jupyterlab>2 pre-commit pytest>=6 pytest-cov diff --git a/tests/integration/test_widgets.py b/tests/integration/test_widgets.py new file mode 100644 index 0000000..06f4bd4 --- /dev/null +++ b/tests/integration/test_widgets.py @@ -0,0 +1,22 @@ +# Copyright 2019 HTCondor Team, Computer Sciences Department, +# University of Wisconsin-Madison, WI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import htmap + + +def test_repr_html_contains_map_tag(map_that_never_finishes): + assert map_that_never_finishes.tag in map_that_never_finishes._repr_html_() From b993c02815573c73a95a10fe633ed9c68ae78b1e Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Thu, 20 Aug 2020 11:16:53 -0500 Subject: [PATCH 08/16] install nodejs and npm in the dev container and build the ipywidgets notebook and lab extensions --- docker/Dockerfile | 8 ++++++-- dr | 8 +++++++- setup.cfg | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b1fc2ce..3b81566 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,7 +34,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ # install HTCondor version specified in config RUN : \ && apt-get update \ - && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace \ + && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace nodejs npm \ && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ && locale-gen \ && wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ @@ -67,6 +67,10 @@ ENTRYPOINT ["/home/mapper/htmap/docker/entrypoint.sh"] CMD ["bash"] COPY --chown=mapper:mapper . /home/${USER}/htmap -RUN python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* +RUN : \ + && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* \ + && jupyter nbextension enable --py widgetsnbextension \ + && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ + && : WORKDIR /home/${USER}/htmap diff --git a/dr b/dr index d2af731..0779865 100755 --- a/dr +++ b/dr @@ -6,4 +6,10 @@ set -e docker build -t ${CONTAINER_TAG} --file docker/Dockerfile . -docker run -it --rm --mount type=bind,src="$PWD",dst=/home/mapper/htmap -p 8000:8000 ${CONTAINER_TAG} $@ +docker run \ + -it --rm \ + --mount type=bind,src="$PWD",dst=/home/mapper/htmap \ + -p 8000:8000 \ + -p 8888:8888 \ + ${CONTAINER_TAG} \ + $@ diff --git a/setup.cfg b/setup.cfg index 69deaa9..ba29b62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,7 +58,9 @@ tests = codecov coverage ipywidgets + ipywidgets jupyterlab>2 + jupyterlab>=2 pre-commit pytest>=6 pytest-cov From 8e00f4c17b14bccfc9a8820690056a17829afa1d Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Thu, 27 Aug 2020 09:50:46 -0500 Subject: [PATCH 09/16] Jupyter lab widget wrapped HTML code in ipywidget (#228) converted to widgets, aesthetic changes required --- docker/Dockerfile | 36 ++++++++++++++++++------------------ htmap/maps.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3b81566..0f5f113 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,26 +33,26 @@ ENV DEBIAN_FRONTEND=noninteractive \ # install utils and dependencies # install HTCondor version specified in config RUN : \ - && apt-get update \ - && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace nodejs npm \ - && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ - && locale-gen \ - && wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ - && echo "deb http://research.cs.wisc.edu/htcondor/debian/${HTCONDOR_VERSION}/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list \ - && apt-get -y update \ - && apt-get -y install --no-install-recommends htcondor \ - && apt-get -y clean \ - && rm -rf /var/lib/apt/lists/* \ - && : + && apt-get update \ + && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace nodejs npm \ + && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ + && locale-gen \ + && wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ + && echo "deb http://research.cs.wisc.edu/htcondor/debian/${HTCONDOR_VERSION}/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list \ + && apt-get -y update \ + && apt-get -y install --no-install-recommends htcondor \ + && apt-get -y clean \ + && rm -rf /var/lib/apt/lists/* \ + && : # create a user, set their PATH and PYTHONPATH ENV USER=mapper \ PATH="/home/mapper/.local/bin:${PATH}" \ PYTHONPATH="/home/mapper/htmap:${PYTHONPATH}" RUN : \ - && groupadd ${USER} \ - && useradd -m -g ${USER} ${USER} \ - && : + && groupadd ${USER} \ + && useradd -m -g ${USER} ${USER} \ + && : # switch to the user, don't need root anymore USER ${USER} @@ -68,9 +68,9 @@ CMD ["bash"] COPY --chown=mapper:mapper . /home/${USER}/htmap RUN : \ - && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* \ - && jupyter nbextension enable --py widgetsnbextension \ - && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - && : + && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* \ + && jupyter nbextension enable --py widgetsnbextension \ + && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ + && : WORKDIR /home/${USER}/htmap diff --git a/htmap/maps.py b/htmap/maps.py index d3e78ee..caf656e 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -165,8 +165,8 @@ def _repr_grid_(self): max_runtime = str(max(self.runtime)) total_runtime = str(sum(self.runtime, datetime.timedelta())) - return f" {self.tag} " + "".join( - f" {h} " + return f' {self.tag} ' + "".join( + f' {h} ' for h in [ *[ sc[component_state] @@ -181,7 +181,7 @@ def _repr_grid_(self): def _repr_html_table(self): table = [ - "", + '
', " ", f" {self._repr_header_()}", " ", @@ -193,6 +193,33 @@ def _repr_html_table(self): return "\n".join(table) + def get_completed_and_total(self): + sc = collections.Counter(self.component_statuses) + completed = sc[state.ComponentStatus.COMPLETED] + total = sum( + [sc[component_state] for component_state in state.ComponentStatus.display_statuses()] + ) + return completed, total + + def _ipython_display_(self, **kwargs): + from IPython.display import display + from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox, widgets + + table_widget = widgets.HTML(value=self._repr_html_(), layout=Layout(min_width="150px"),) + + completed, total = self.get_completed_and_total() + + progress_bar_widget = widgets.IntProgress( + value=completed, + min=0, + max=total, + step=1, + bar_style="", # 'success', 'info', 'warning', 'danger' or '' + orientation="horizontal", + ) + v_box = VBox([table_widget, progress_bar_widget]) + return v_box._ipython_display_(**kwargs) + def __repr__(self): return f"{self.__class__.__name__}(tag = {self.tag})" From 5fa3096a49c6c387303fc21859107e00c4cc1745 Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:20:06 -0500 Subject: [PATCH 10/16] Live update working --- htmap/maps.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/htmap/maps.py b/htmap/maps.py index caf656e..3e635d4 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -682,14 +682,38 @@ def job(x): } def status(self) -> str: - """Return a string containing the number of jobs in each status.""" - counts = collections.Counter(self.component_statuses) - stat = " | ".join( - f"{str(js)} = {counts[js]}" for js in state.ComponentStatus.display_statuses() - ) - msg = f"{self.__class__.__name__} {self.tag} ({len(self)} components): {stat}" + # """Return a string containing the number of jobs in each status.""" + # counts = collections.Counter(self.component_statuses) + # stat = " | ".join( + # f"{str(js)} = {counts[js]}" for js in state.ComponentStatus.display_statuses() + # ) + # msg = f"{self.__class__.__name__} {self.tag} ({len(self)} components): {stat}" + + # return utils.rstr(msg) + + from IPython.display import display + from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox, widgets - return utils.rstr(msg) + table_widget = widgets.HTML(value=self._repr_html_(), layout=Layout(min_width="150px"),) + + completed, total = self.get_completed_and_total() + + progress_bar_widget = widgets.IntProgress( + value=completed, + min=0, + max=total, + step=1, + bar_style="", # 'success', 'info', 'warning', 'danger' or '' + orientation="horizontal", + ) + v_box = VBox([table_widget, progress_bar_widget]) + display(v_box) + while True: + table_widget.value = self._repr_html_() + completed, total = self.get_completed_and_total() + progress_bar_widget.value = completed + time.sleep(1) + # return v_box._ipython_display_(**kwargs) @property def holds(self) -> Dict[int, holds.ComponentHold]: From 1ba660f2ffaca312ef7174450e8ce6c49fc30840 Mon Sep 17 00:00:00 2001 From: elin1231 <43862267+elin1231@users.noreply.github.com> Date: Fri, 28 Aug 2020 10:04:50 -0500 Subject: [PATCH 11/16] Updated to remove duplicate code. Working live updating for status and repr --- htmap/maps.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/htmap/maps.py b/htmap/maps.py index 3e635d4..23bebca 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -202,23 +202,7 @@ def get_completed_and_total(self): return completed, total def _ipython_display_(self, **kwargs): - from IPython.display import display - from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox, widgets - - table_widget = widgets.HTML(value=self._repr_html_(), layout=Layout(min_width="150px"),) - - completed, total = self.get_completed_and_total() - - progress_bar_widget = widgets.IntProgress( - value=completed, - min=0, - max=total, - step=1, - bar_style="", # 'success', 'info', 'warning', 'danger' or '' - orientation="horizontal", - ) - v_box = VBox([table_widget, progress_bar_widget]) - return v_box._ipython_display_(**kwargs) + self.status() def __repr__(self): return f"{self.__class__.__name__}(tag = {self.tag})" @@ -699,12 +683,7 @@ def status(self) -> str: completed, total = self.get_completed_and_total() progress_bar_widget = widgets.IntProgress( - value=completed, - min=0, - max=total, - step=1, - bar_style="", # 'success', 'info', 'warning', 'danger' or '' - orientation="horizontal", + value=completed, min=0, max=total, step=1, orientation="horizontal", ) v_box = VBox([table_widget, progress_bar_widget]) display(v_box) @@ -712,8 +691,8 @@ def status(self) -> str: table_widget.value = self._repr_html_() completed, total = self.get_completed_and_total() progress_bar_widget.value = completed + progress_bar_widget.max = total time.sleep(1) - # return v_box._ipython_display_(**kwargs) @property def holds(self) -> Dict[int, holds.ComponentHold]: From 5095a5410477a927c8e5fea822f351de72505636 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Mon, 31 Aug 2020 17:36:09 -0500 Subject: [PATCH 12/16] implement background thread for updating widgets; add widget to Map.wait; only display widgets when in a Jupyter session --- docker/Dockerfile | 2 +- htmap/mapping.py | 2 +- htmap/maps.py | 203 ++++++++++++++++++++++++++++++---------------- htmap/state.py | 3 +- htmap/utils.py | 16 ++++ setup.cfg | 7 +- 6 files changed, 158 insertions(+), 75 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0f5f113..96e812e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -68,7 +68,7 @@ CMD ["bash"] COPY --chown=mapper:mapper . /home/${USER}/htmap RUN : \ - && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* \ + && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs,widgets]" htcondor==${HTCONDOR_VERSION}.* \ && jupyter nbextension enable --py widgetsnbextension \ && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ && : diff --git a/htmap/mapping.py b/htmap/mapping.py index 3a64c10..fb21402 100644 --- a/htmap/mapping.py +++ b/htmap/mapping.py @@ -300,7 +300,7 @@ def create_map( tags.tag_file_path(tag).write_text(str(uid)) - m = maps.Map(tag=tag, map_dir=map_dir,) + m = maps.Map(tag=tag, map_dir=map_dir) if transient: m._make_transient() diff --git a/htmap/maps.py b/htmap/maps.py index 23bebca..8f0b07d 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -20,6 +20,7 @@ import inspect import logging import shutil +import threading import time import weakref from copy import copy @@ -70,6 +71,32 @@ def maps_by_tag() -> Dict[str, "Map"]: return {m.tag: m for m in MAPS} +def update_widgets(): + while True: + for map in MAPS.copy(): + try: + _, update = map._widget() + + if update is not None: + update() + except: + logger.exception("Widget update thread encountered error!") + + time.sleep(settings["WAIT_TIME"]) + + +WIDGET_UPDATE_THREAD = threading.Thread(target=update_widgets, daemon=True) + + +def start_widget_update_thread(): + try: + if not WIDGET_UPDATE_THREAD.is_alive(): + WIDGET_UPDATE_THREAD.start() + except RuntimeError: + # Someone else started the thread before we did, no worries + pass + + @_protect_map_after_remove class Map(collections.abc.Sequence): """ @@ -105,6 +132,8 @@ def __init__( self._stderr: MapStdErr = MapStdErr(self) self._output_files: MapOutputFiles = MapOutputFiles(self) + self._cached_widget = (None, None) + MAPS.add(self) @property @@ -140,12 +169,89 @@ def load(cls, tag: str) -> "Map": logger.debug(f"Loaded map {tag} from {map_dir}") - return cls(tag=tag, map_dir=map_dir,) + return cls(tag=tag, map_dir=map_dir) + + def __repr__(self): + return f"{self.__class__.__name__}(tag={self.tag})" + + def status(self): + """Display a string containing the number of jobs in each status.""" + counts = collections.Counter(self.component_statuses) + stat = " | ".join( + f"{str(js)} = {counts[js]}" for js in state.ComponentStatus.display_statuses() + ) + plain = f"{self.__class__.__name__} {self.tag} ({len(self)} components): {stat}" + + if not utils.is_jupyter(): + print(plain) + return + + from IPython.display import display + + widget, _ = self._widget() + + if widget is not None: + display(widget) + return + + data = {"text/plain": plain, "text/html": self._repr_html_()} + + display(data, raw=True) + + def _ipython_display_(self, **kwargs): + self.status() + + def _widget(self): + try: + from ipywidgets import Layout, VBox, widgets + except ImportError: + return self._cached_widget + + if self._cached_widget != (None, None): + return self._cached_widget + + table = widgets.HTML(value=self._repr_html_(), layout=Layout(min_width="150px")) + + pbar = widgets.IntProgress( + value=0, min=0, max=len(self), orientation="horizontal", layout=Layout(width="90%"), + ) + widget = VBox([table, pbar]) + + def update(): + table.value = self._repr_html_() + pbar.value = len(self.components_by_status().get(state.ComponentStatus.COMPLETED, [])) + + update() + + self._cached_widget = widget, update + + start_widget_update_thread() + + return self._cached_widget def _repr_html_(self): - return self._repr_html_table() + return self._html_table() + + def _html_table(self): + table = [ + # Hacked together by looking at the classes of the parent div in + # the version formatted by Jupyter... probably not very stable. + '
', + " ", + f" {self._html_table_header()}", + " ", + " ", + f" {self._html_table_body()}", + " ", + "
", + "", + ] - def _repr_header_(self): + return "\n".join(table) + + @staticmethod + def _html_table_header(): return " TAG " + "".join( f" {h} " for h in [ @@ -157,7 +263,7 @@ def _repr_header_(self): ] ) - def _repr_grid_(self): + def _html_table_body(self): sc = collections.Counter(self.component_statuses) local_data = utils.num_bytes_to_str(self.local_data) @@ -179,34 +285,6 @@ def _repr_grid_(self): ] ) - def _repr_html_table(self): - table = [ - '', - " ", - f" {self._repr_header_()}", - " ", - " ", - f" {self._repr_grid_()}", - " ", - "
", - ] - - return "\n".join(table) - - def get_completed_and_total(self): - sc = collections.Counter(self.component_statuses) - completed = sc[state.ComponentStatus.COMPLETED] - total = sum( - [sc[component_state] for component_state in state.ComponentStatus.display_statuses()] - ) - return completed, total - - def _ipython_display_(self, **kwargs): - self.status() - - def __repr__(self): - return f"{self.__class__.__name__}(tag = {self.tag})" - def __gt__(self, other): return self.tag > other.tag @@ -284,7 +362,7 @@ def is_active(self) -> bool: def wait( self, timeout: utils.Timeout = None, - show_progress_bar: bool = False, + show_progress_bar: Optional[bool] = None, holds_ok: bool = False, errors_ok: bool = False, ) -> None: @@ -302,6 +380,8 @@ def wait( If ``None``, wait forever. show_progress_bar If ``True``, a progress bar will be displayed. + If ``None`` (the default), a progress bar will be displayed if you + are running Python interactively (e.g., in a REPL or Jupyter session). holds_ok If ``True``, will not raise exceptions if components are held. errors_ok @@ -310,11 +390,22 @@ def wait( start_time = time.time() timeout = utils.timeout_to_seconds(timeout) + if show_progress_bar is None and utils.is_interactive_session(): + show_progress_bar = True + try: + pbar = None if show_progress_bar: - pbar = tqdm(desc=self.tag, total=len(self), unit="component", ascii=True,) + # TODO: what if no widget + widget, update = self._widget() + if utils.is_jupyter() and widget is not None: + from IPython.display import display + + display(widget) + else: + pbar = tqdm(desc=self.tag, total=len(self), unit="component", ascii=True,) - previous_pbar_len = 0 + previous_pbar_len = 0 ok_statuses = {state.ComponentStatus.COMPLETED} if holds_ok: @@ -324,10 +415,15 @@ def wait( while True: num_incomplete = sum(cs not in ok_statuses for cs in self.component_statuses) + if show_progress_bar: - pbar_len = self._num_components - num_incomplete - pbar.update(pbar_len - previous_pbar_len) - previous_pbar_len = pbar_len + if pbar: + pbar_len = self._num_components - num_incomplete + pbar.update(pbar_len - previous_pbar_len) + previous_pbar_len = pbar_len + else: + update() + if num_incomplete == 0: break @@ -346,7 +442,7 @@ def wait( time.sleep(settings["WAIT_TIME"]) finally: - if show_progress_bar: + if show_progress_bar and pbar: pbar.close() def _wait_for_component(self, component: int, timeout: utils.Timeout = None) -> None: @@ -665,35 +761,6 @@ def job(x): status: tuple(sorted(components)) for status, components in status_to_components.items() } - def status(self) -> str: - # """Return a string containing the number of jobs in each status.""" - # counts = collections.Counter(self.component_statuses) - # stat = " | ".join( - # f"{str(js)} = {counts[js]}" for js in state.ComponentStatus.display_statuses() - # ) - # msg = f"{self.__class__.__name__} {self.tag} ({len(self)} components): {stat}" - - # return utils.rstr(msg) - - from IPython.display import display - from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox, widgets - - table_widget = widgets.HTML(value=self._repr_html_(), layout=Layout(min_width="150px"),) - - completed, total = self.get_completed_and_total() - - progress_bar_widget = widgets.IntProgress( - value=completed, min=0, max=total, step=1, orientation="horizontal", - ) - v_box = VBox([table_widget, progress_bar_widget]) - display(v_box) - while True: - table_widget.value = self._repr_html_() - completed, total = self.get_completed_and_total() - progress_bar_widget.value = completed - progress_bar_widget.max = total - time.sleep(1) - @property def holds(self) -> Dict[int, holds.ComponentHold]: """ diff --git a/htmap/state.py b/htmap/state.py index fcbe69c..cd6c4b0 100644 --- a/htmap/state.py +++ b/htmap/state.py @@ -111,8 +111,9 @@ def _event_log_path(self): def _read_events(self): with self._event_reader_lock: # no thread can be in here at the same time as another if self._event_reader is None: - logger.debug(f"Created event log reader for map {self.map.tag}") + self._event_log_path.touch(exist_ok=True) self._event_reader = htcondor.JobEventLog(self._event_log_path.as_posix()).events(0) + logger.debug(f"Created event log reader for map {self.map.tag}") with utils.Timer() as timer: handled_events = self._handle_events() diff --git a/htmap/utils.py b/htmap/utils.py index ee66ce7..943624c 100644 --- a/htmap/utils.py +++ b/htmap/utils.py @@ -239,6 +239,22 @@ def is_interactive_session() -> bool: ) +def is_jupyter() -> bool: + # https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook/24937408 + # This seems quite fragile, but it also seems hard to determine otherwise... + # I would not be shocked if this breaks in the future. + try: + shell = get_ipython().__class__.__name__ + if shell == "ZMQInteractiveShell": + return True # Jupyter notebook or qtconsole + elif shell == "TerminalInteractiveShell": + return False # Terminal running IPython + else: + return False # Something else... + except NameError: + return False # Probably standard Python interpreter + + def enable_debug_logging(): logger = logging.getLogger("htmap") logger.setLevel(logging.DEBUG) diff --git a/setup.cfg b/setup.cfg index ba29b62..dc33f51 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,10 +57,6 @@ docs = tests = codecov coverage - ipywidgets - ipywidgets - jupyterlab>2 - jupyterlab>=2 pre-commit pytest>=6 pytest-cov @@ -69,6 +65,9 @@ tests = pytest-timeout pytest-watch pytest-xdist +widgets = + ipywidgets + jupyterlab>=2 [options.package_data] * = From 3072125335c593f1b4f1bf9d9aaeee85e052de04 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 24 Jun 2021 20:42:08 -0500 Subject: [PATCH 13/16] Drop --use-feature=2020-resolver flag because it's now the default --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 96e812e..c70eddc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -68,7 +68,7 @@ CMD ["bash"] COPY --chown=mapper:mapper . /home/${USER}/htmap RUN : \ - && python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs,widgets]" htcondor==${HTCONDOR_VERSION}.* \ + && python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs,widgets]" htcondor==${HTCONDOR_VERSION}.* \ && jupyter nbextension enable --py widgetsnbextension \ && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ && : From d1af00e2840e8d1ed5239ca3b3ae7a26a237f417 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 24 Jun 2021 20:50:55 -0500 Subject: [PATCH 14/16] Don't install the jupyterlab-manager labextension -- it requires nodejs >= 12 but this distro only has 10 --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c70eddc..ed0b4d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,7 +70,8 @@ COPY --chown=mapper:mapper . /home/${USER}/htmap RUN : \ && python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs,widgets]" htcondor==${HTCONDOR_VERSION}.* \ && jupyter nbextension enable --py widgetsnbextension \ - && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ + # && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ + # ^^^ disabled because it requires nodejs >= 12 and this container only has 10 && : WORKDIR /home/${USER}/htmap From 502f24e4b8e3d9fd6fe8d3a25dcb41327a3ed03c Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 4 Nov 2022 16:49:31 -0500 Subject: [PATCH 15/16] Add jupyterlab-manager back -- Debian bullseye has Node 12 --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 66ba258..1f80ae7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -68,8 +68,7 @@ RUN : \ fi \ && python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs,widgets]" "$requirement" \ && jupyter nbextension enable --py widgetsnbextension \ - # && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ - # ^^^ disabled because it requires nodejs >= 12 and this container only has 10 \ + && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ && : WORKDIR /home/${USER}/htmap From 17a027a0bdf64bc4cc5725ef56351f40886edd90 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 4 Nov 2022 16:59:54 -0500 Subject: [PATCH 16/16] Add a build arg to disable NodeJS --- docker/Dockerfile | 4 +++- docker/install-htcondor.sh | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f80ae7..df316d6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,6 +21,8 @@ FROM python:${PYTHON_VERSION} # build config ARG HTCONDOR_VERSION=9.0 +ARG DISABLE_NODEJS= + # switch to root to do root-level config USER root @@ -68,7 +70,7 @@ RUN : \ fi \ && python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs,widgets]" "$requirement" \ && jupyter nbextension enable --py widgetsnbextension \ - && jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ + && [ "X${DISABLE_NODEJS}" != X ] || jupyter labextension install --minimize=False @jupyter-widgets/jupyterlab-manager \ && : WORKDIR /home/${USER}/htmap diff --git a/docker/install-htcondor.sh b/docker/install-htcondor.sh index 9419449..071632c 100755 --- a/docker/install-htcondor.sh +++ b/docker/install-htcondor.sh @@ -12,7 +12,10 @@ fi export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace nodejs npm +apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace +if [[ ! $DISABLE_NODEJS ]]; then + apt-get -y install --no-install-recommends nodejs npm +fi echo "en_US.UTF-8 UTF-8" > /etc/locale.gen locale-gen wget -qO - "https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-${HTCONDOR_VERSION}-Key" | apt-key add -