From 06bc25cf91c05be432cc2bd0c87daca6d250626a Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Fri, 5 Sep 2025 15:45:06 +0000 Subject: [PATCH 01/20] for #141 add n_average and start on save feature --- slicops/package_data/sliclet/screen.yaml | 19 +++++++++++++ slicops/sliclet/screen.py | 34 +++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index 7c07650..209d46f 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -24,6 +24,18 @@ fields: Gaussian: gaussian "Super Gaussian": super_gaussian value: gaussian + n_average: + prototype: Enum + constraints: + choices: + - 1 + - 2 + - 3 + - 4 + - 5 + ui: + label: Number of images to average + value: 1 plot: prototype: Dict ui: @@ -50,6 +62,11 @@ fields: ui: css_kind: danger label: Stop + save_button: + prototype: Button + ui: + css_kind: outline-info + label: Save ui_layout: - cols: @@ -58,10 +75,12 @@ ui_layout: - beam_path - camera - pv + - n_average - cell_group: - start_button - stop_button - single_button + - save_button - css: col-sm-9 col-xxl-7 rows: - plot diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index f8e7cd5..d41b564 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -6,6 +6,7 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp +import numpy import pykern.pkconfig import pykern.util import queue @@ -38,6 +39,9 @@ ("single_button.ui.visible", False), ("start_button.ui.visible", False), ("stop_button.ui.visible", False), + ("save_button.ui.visible", False), + ("n_average.ui.visible", False), + ("save_button.ui.enabled", False), ) + _BUTTONS_DISABLE _DEVICE_ENABLE = ( @@ -45,6 +49,8 @@ ("single_button.ui.visible", True), ("start_button.ui.visible", True), ("stop_button.ui.visible", True), + ("save_button.ui.visible", True), + ("n_average.ui.visible", True), ("single_button.ui.enabled", True), ("stop_button.ui.enabled", False), ("start_button.ui.enabled", True), @@ -72,6 +78,10 @@ def on_change_beam_path(self, txn, value, **kwargs): def on_change_curve_fit_method(self, txn, **kwargs): self.__update_plot(txn) + def on_click_save_button(self, txn, **kwargs): + #TODO(pjm): create and save hdf5 file to $PYHSICS_DATA + pass + def on_click_single_button(self, txn, **kwargs): self.__single_button = True self.__set_acquire(txn, True) @@ -84,6 +94,10 @@ def on_click_stop_button(self, txn, **kwargs): def handle_init(self, txn): self.__device = None + self.__images = PKDict( + frames=[], + average=None, + ) self.__monitors = PKDict() self.__single_button = False txn.multi_set(("beam_path.constraints.choices", slicops.device_db.beam_paths())) @@ -191,8 +205,7 @@ def __handle_acquire(self, acquire): def __handle_image(self, image): with self.lock_for_update() as txn: - if self.__update_plot(txn) and self.__single_button: - # self.__single_button = False + if self.__update_plot(txn, txn.field_get("n_average")) and self.__single_button: self.__set_acquire(txn, False) txn.multi_set( ("single_button.ui.enabled", True), @@ -226,17 +239,30 @@ def __set_acquire(self, txn, acquire): # if status is in: # enable buttons # - def __update_plot(self, txn): + def __update_plot(self, txn, n_average=None): if not self.__device: return False if (i := self.__monitors.image.prev_value()) is None or not i.size: return False + if n_average is None: + if self.__images.average is None: + self.__images.average = i + else: + self.__images.frames.append(i) + if len(self.__images.frames) < n_average: + return False + self.__images.average = numpy.mean(numpy.array(self.__images.frames), axis=0) + self.__images.frames = [] if not txn.group_get("plot", "ui", "visible"): txn.multi_set(_PLOT_ENABLE) txn.field_set( "plot", - slicops.plot.fit_image(i, txn.field_get("curve_fit_method")), + slicops.plot.fit_image(self.__images.average, txn.field_get("curve_fit_method")), ) + if not txn.group_get("save_button", "ui", "enabled"): + txn.multi_set( + ("save_button.ui.enabled", True), + ) return True def __user_alert(self, txn, fmt, *args): From dabf193ae529990bbf3fc5744d737f1fc358b412 Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Fri, 5 Sep 2025 23:05:12 +0000 Subject: [PATCH 02/20] for #141 implement saving profile results as hdf5 --- slicops/sliclet/screen.py | 130 +++++++++++++++++++++++++++++++------- 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index d41b564..7be48cf 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -4,10 +4,13 @@ :license: http://github.com/slaclab/slicops/LICENSE """ +from datetime import datetime from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp +import h5py import numpy import pykern.pkconfig +import pykern.pkio import pykern.util import queue import slicops.device @@ -79,8 +82,14 @@ def on_change_curve_fit_method(self, txn, **kwargs): self.__update_plot(txn) def on_click_save_button(self, txn, **kwargs): - #TODO(pjm): create and save hdf5 file to $PYHSICS_DATA - pass + self.__image.save( + PKDict( + [ + (k, txn.field_get(k)) + for k in ("pv", "camera", "curve_fit_method", "pv") + ] + ), + ) def on_click_single_button(self, txn, **kwargs): self.__single_button = True @@ -94,10 +103,7 @@ def on_click_stop_button(self, txn, **kwargs): def handle_init(self, txn): self.__device = None - self.__images = PKDict( - frames=[], - average=None, - ) + self.__image = _ImageSet() self.__monitors = PKDict() self.__single_button = False txn.multi_set(("beam_path.constraints.choices", slicops.device_db.beam_paths())) @@ -205,7 +211,10 @@ def __handle_acquire(self, acquire): def __handle_image(self, image): with self.lock_for_update() as txn: - if self.__update_plot(txn, txn.field_get("n_average")) and self.__single_button: + if ( + self.__update_plot(txn, txn.field_get("n_average")) + and self.__single_button + ): self.__set_acquire(txn, False) txn.multi_set( ("single_button.ui.enabled", True), @@ -242,27 +251,20 @@ def __set_acquire(self, txn, acquire): def __update_plot(self, txn, n_average=None): if not self.__device: return False - if (i := self.__monitors.image.prev_value()) is None or not i.size: - return False if n_average is None: - if self.__images.average is None: - self.__images.average = i + if not self.__image.ready(): + return False else: - self.__images.frames.append(i) - if len(self.__images.frames) < n_average: + if (i := self.__monitors.image.prev_value()) is None or not i.size: + return False + # TODO(pjm): timestamp should be EPICS timestamp included with image data + if not self.__image.add(i, datetime.now(), n_average): return False - self.__images.average = numpy.mean(numpy.array(self.__images.frames), axis=0) - self.__images.frames = [] if not txn.group_get("plot", "ui", "visible"): txn.multi_set(_PLOT_ENABLE) - txn.field_set( - "plot", - slicops.plot.fit_image(self.__images.average, txn.field_get("curve_fit_method")), - ) + txn.field_set("plot", self.__image.fit(txn.field_get("curve_fit_method"))) if not txn.group_get("save_button", "ui", "enabled"): - txn.multi_set( - ("save_button.ui.enabled", True), - ) + txn.multi_set(("save_button.ui.enabled", True)) return True def __user_alert(self, txn, fmt, *args): @@ -272,6 +274,89 @@ def __user_alert(self, txn, fmt, *args): CLASS = Screen +class _ImageSet: + """Collects images for averaging""" + + def __init__(self): + self._complete = None + self._frames = [] + + def add(self, image, timestamp, n_average): + """Add an image to the set. Returns True if ready""" + assert n_average + if len(self._frames) >= n_average: + # n_average changed while collecting frames + self._frames = self._frames[: n_average - 1] + self._frames.append( + PKDict( + image=image, + timestamp=timestamp, + ) + ) + if len(self._frames) == n_average: + self._complete = self._frames + self._frames = [] + return True + return False + + def fit(self, curve_fit_method): + assert self.ready() + if len(self._complete) == 1: + i = self._complete[0].image + else: + i = numpy.mean(numpy.array([v.image for v in self._complete]), axis=0) + return slicops.plot.fit_image(i, curve_fit_method) + + def ready(self): + return bool(self._complete) + + def save(self, metadata): + """Creates a hdf5 file with the structure: + /image Group + /frames Dataset {n_average, ysize, xsize} + /mean Dataset {ysize, xsize} + /timestamps Dataset {n_average} + /x Group + /fit Dataset {xsize} + /profile Dataset {xsize} + /y Group + /fit Dataset {ysize} + /profile Dataset {ysize} + /meta Group (beam_path, camera, pv, curve_fit_method) + """ + assert self.ready() + i = self.fit(metadata.curve_fit_method) + with h5py.File(self._file_path(metadata), "w") as hf: + g = hf.create_group("meta") + for f in metadata: + g.attrs[f] = metadata[f] + g = hf.create_group("image") + g.create_dataset("mean", data=i.raw_pixels) + for dim in ("x", "y"): + g2 = g.create_group(dim) + g2.create_dataset("profile", data=i[dim].lineout) + if not i[dim].fit.results: + continue + for f in i[dim].fit.results: + g2.attrs[f] = i[dim].fit.results[f] + g2.create_dataset("fit", data=i[dim].fit.fit_line) + g.create_dataset( + "frames", data=numpy.array([v.image for v in self._complete]) + ) + g.create_dataset( + "timestamps", data=[v.timestamp.timestamp() for v in self._complete] + ) + + def _file_path(self, metadata): + t = self._complete[-1].timestamp + # TODO(pjm): not sure about filename or whether it should be nested in directories + n = f"Screen-{metadata.camera}-{t.strftime('%Y-%m-%d-%f')}.h5" + p = pykern.pkio.py_path(_cfg.save_file_path).join(n) + if p.exists(): + p.remove() + return str(p) + + class _Monitor: # TODO(robnagler) handle more values besides plot def __init__(self, accessor, handler): @@ -356,6 +441,7 @@ def _init(): beam_path=("DEV_BEAM_PATH", str, "dev beam path name"), camera=("DEV_CAMERA", str, "dev camera name"), ), + save_file_path=("/tmp", str, "root path for screen save files"), ) From 268aa28c466e8bd1927edd6b2d42be221172992c Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Tue, 9 Sep 2025 23:17:26 +0000 Subject: [PATCH 03/20] refactor --- slicops/ctx.py | 3 + slicops/pkcli/ioc.py | 2 +- slicops/plot.py | 98 ++++++++++++++++++++++ slicops/sliclet/__init__.py | 24 +++++- slicops/sliclet/screen.py | 156 ++++++++---------------------------- 5 files changed, 160 insertions(+), 123 deletions(-) diff --git a/slicops/ctx.py b/slicops/ctx.py index ed2c9e7..1c3f172 100644 --- a/slicops/ctx.py +++ b/slicops/ctx.py @@ -165,6 +165,9 @@ def _update(old, new): def group_get(self, field, group, attr=None): return self.__ctx.fields[field].group_get(group, attr) + def multi_get(self, fields): + return PKDict(k: self.__field(k).value_get() for k in fields) + def multi_set(self, *args): def _args(): if len(args) > 1: diff --git a/slicops/pkcli/ioc.py b/slicops/pkcli/ioc.py index 4b9dd48..a62ed24 100644 --- a/slicops/pkcli/ioc.py +++ b/slicops/pkcli/ioc.py @@ -25,7 +25,7 @@ def _pvgroup(config): return PKDict( # Need to hardwire the defaults, because ioc_arg_parser uses # argparse globally which causes a mess with argh (which uses argparse) - pvdb=_PVGroup(config, db_yaml, macros={}, prefix="").pvdb, + pvdb=pkdp(_PVGroup(config, db_yaml, macros={}, prefix="").pvdb), interfaces=["0.0.0.0"], module_name="caproto.asyncio.server", log_pv_names=False, diff --git a/slicops/plot.py b/slicops/plot.py index 1317577..993864c 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -6,9 +6,107 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp +import collections import numpy +import pykern.pkio import scipy.optimize +class ImageSet: + """Fits images, possibly averaging. + + Can take arbitrary meta data, e.g. pv and it will be written by + `save_file`. + + Args: + meta (PKDict): n_average, camera, curve_fit_method, pv + + """ + + def __init__(self, meta): + self.meta = meta + self._frames = collections.deque(meta.n_average) + self._timestamps = collections.deque(meta.n_average) + self._fit = None + + def add_frame(self, frame, timestamp): + """Add and update fit + + Args: + frame (ndarray): new image + timestamp (datetime): time of frame + Returns: + PKDict: frame and fit or None if not enough frames + """ + + def _mean(self): + if self._frames.maxlen == 1: + return self._frames[-1] + return numpy.mean(self._frames, axis=0) + + self._frames.append(frame) + self._timestamps.append(timestamp) + if self._ready(): + self._fit = fit_image(_mean(), self.meta.curve_fit_method) + return self._fit + + def save_file(self, dir_path): + #TODO(robnagler) the naming is a bit goofy, possibly frames/{images,timestamps} and analysis. + """Creates a hdf5 file with the structure:: + /image Group + /frames Dataset {n_average, ysize, xsize} + /mean Dataset {ysize, xsize} + /timestamps Dataset {n_average} + /x Group + /fit Dataset {xsize} + /profile Dataset {xsize} + /y Group + /fit Dataset {ysize} + /profile Dataset {ysize} + /meta Group (camera, curve_fit_method, n_average, etc.) + + Args: + dir_path (py.path): directory + """ + + def _image_dim(meta_group, dim): + g = meta_group.create_group(dim) + f = self._fit[dim] + g.create_dataset("profile", data=f.lineout) + if not f.fit.results: + return + g.attrs.update(f.results) + g.create_dataset("fit", data=f.fit.fit_line) + + def _meta(h5_file): + g = h5_file.create_group("meta") + g.attrs.update(self.meta) + g.create_dataset("frames", data=self._frames) + g.create_dataset("timestamps", data=self._timestamps) + + def _path(): + # TODO(robnagler) centralize timestamp format + t = self._timestamps[-1] + return dir_path.join( + t.strftime("%Y-%m"), + f"{t.strftime('%Y%m%d%H%M%S')}-{self.meta.camera}.h5", + ) + + def _writer(path): + with h5py.File(path, "w") as f: + _meta(f) + g = f.create_group("image") + g.dataset("mean", self._fit.raw_pixels) + _image_dim(g, "x") + _image_dim(g, "y") + + if not self._ready(): + return False + pykern.pkio.atomic_write(_path(), writer=_writer) + return True + + def _ready(self): + return len(self._frames) == self._frames.maxlen + def fit_image(image, method): """Attemp an analytical fit for the sum along the x and y dimensions diff --git a/slicops/sliclet/__init__.py b/slicops/sliclet/__init__.py index 2a6028e..a8f2e03 100644 --- a/slicops/sliclet/__init__.py +++ b/slicops/sliclet/__init__.py @@ -11,6 +11,9 @@ import enum import importlib import inspect +import pykern.pkconst +import pykern.pkconfig +import pykern.pkio import pykern.util import queue import re @@ -112,6 +115,9 @@ def lock_for_update(self, log_op=None): pkdlog("ERROR {}", d) self.__put_work(_Work.error, PKDict(desc=d)) + def save_file_path(self): + return _cfg.save_file_root.join(self.__class__.__name__).ensure(dir=True) + def session_end(self): self.__put_work(_Work.session_end, None) @@ -207,12 +213,28 @@ def _work_start(self, unused): return True +def _cfg_py_path(value): + if isinstance(value, str): + if not os.path.isabs(value): + pykern.pkconfig.raise_error(f"not an absolute path={value}") + return pykern.pkio.py_path(value) + if not isinstance(value, pykern.pkconst.PY_PATH_LOCAL_TYPE): + pykern.pkconfig.raise_error(f"not a py.path type={type(value)} value={value}") + return value + + def _init(): global _cfg - from pykern import pkconfig + + def _path(): + rv = (_cfg_py_path, "root path for screen save files") + if pykern.pkconfig.in_dev_mode(): + return (pykern.util.dev_run_dir("save").ensure(dir=True),) + rv + return pykern.pkconfig.Required(rv) _cfg = pkconfig.init( default=("screen", str, "default sliclet"), + save_file_root=_path(), ) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 7be48cf..524691c 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -9,8 +9,8 @@ from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp import h5py import numpy +import pykern.pkcompat import pykern.pkconfig -import pykern.pkio import pykern.util import queue import slicops.device @@ -72,24 +72,22 @@ class Screen(slicops.sliclet.Base): def handle_destroy(self): self.__device_destroy() + def on_change_beam_path(self, txn, value, **kwargs): + self.__beam_path_change(txn, value) + def on_change_camera(self, txn, value, **kwargs): self.__device_change(txn, value) - def on_change_beam_path(self, txn, value, **kwargs): - self.__beam_path_change(txn, value) + def on_change_curve_fit_method(self, txn, value, **kwargs): + # TODO(robnagler) optimize with ImageSet.update_curve_fit_method() + self.__new_image_set(txn) - def on_change_curve_fit_method(self, txn, **kwargs): - self.__update_plot(txn) + def on_change_n_average(self, txn, value, **kwargs): + # TODO(robnagler) optimize with ImageSet.update_n_average() + self.__new_image_set(txn) def on_click_save_button(self, txn, **kwargs): - self.__image.save( - PKDict( - [ - (k, txn.field_get(k)) - for k in ("pv", "camera", "curve_fit_method", "pv") - ] - ), - ) + self.__image_set.save_file(self.save_file_path()) def on_click_single_button(self, txn, **kwargs): self.__single_button = True @@ -103,7 +101,6 @@ def on_click_stop_button(self, txn, **kwargs): def handle_init(self, txn): self.__device = None - self.__image = _ImageSet() self.__monitors = PKDict() self.__single_button = False txn.multi_set(("beam_path.constraints.choices", slicops.device_db.beam_paths())) @@ -159,6 +156,7 @@ def __device_change(self, txn, camera): def __device_destroy(self, txn=None): if not self.__device: return + self.__image_set = None self.__single_button = False for m in self.__monitors.values(): m.destroy() @@ -193,6 +191,7 @@ def _monitors(): self.__user_alert(txn, "unable to connect to camera={} error={}", camera, e) return txn.multi_set(_DEVICE_ENABLE + (("pv.value", self.__device.meta.pv_prefix),)) + self.__new_image_set(txn) def __handle_acquire(self, acquire): with self.lock_for_update() as txn: @@ -210,17 +209,35 @@ def __handle_acquire(self, acquire): self.__single_button = False def __handle_image(self, image): - with self.lock_for_update() as txn: + def _plot(txn): + if not self.__device: + return False + if (i := self.__monitors.image.prev_value()) is None or not i.size: + return False if ( - self.__update_plot(txn, txn.field_get("n_average")) - and self.__single_button - ): + p := self.__image_set.add_frame(image, pykern.pkcompat.utcnow()) + ) is None: + return False + if not txn.group_get("plot", "ui", "visible"): + txn.multi_set(_PLOT_ENABLE) + txn.field_set("plot", p) + if not txn.group_get("save_button", "ui", "enabled"): + txn.multi_set(("save_button.ui.enabled", True)) + return True + + with self.lock_for_update() as txn: + if _plot(txn) and self.__single_button: self.__set_acquire(txn, False) txn.multi_set( ("single_button.ui.enabled", True), ("start_button.ui.enabled", True), ) + def __new_image_set(self, txtn): + self.__image_set = slicops.plot.ImageSet( + txn.multi_get(("beam_path", "camera", "curve_fit_method", "n_average", "pv")), + ) + def __set_acquire(self, txn, acquire): if not self.__device: # buttons already disabled @@ -248,25 +265,6 @@ def __set_acquire(self, txn, acquire): # if status is in: # enable buttons # - def __update_plot(self, txn, n_average=None): - if not self.__device: - return False - if n_average is None: - if not self.__image.ready(): - return False - else: - if (i := self.__monitors.image.prev_value()) is None or not i.size: - return False - # TODO(pjm): timestamp should be EPICS timestamp included with image data - if not self.__image.add(i, datetime.now(), n_average): - return False - if not txn.group_get("plot", "ui", "visible"): - txn.multi_set(_PLOT_ENABLE) - txn.field_set("plot", self.__image.fit(txn.field_get("curve_fit_method"))) - if not txn.group_get("save_button", "ui", "enabled"): - txn.multi_set(("save_button.ui.enabled", True)) - return True - def __user_alert(self, txn, fmt, *args): pkdlog("TODO: USER ALERT: " + fmt, *args) @@ -274,89 +272,6 @@ def __user_alert(self, txn, fmt, *args): CLASS = Screen -class _ImageSet: - """Collects images for averaging""" - - def __init__(self): - self._complete = None - self._frames = [] - - def add(self, image, timestamp, n_average): - """Add an image to the set. Returns True if ready""" - assert n_average - if len(self._frames) >= n_average: - # n_average changed while collecting frames - self._frames = self._frames[: n_average - 1] - self._frames.append( - PKDict( - image=image, - timestamp=timestamp, - ) - ) - if len(self._frames) == n_average: - self._complete = self._frames - self._frames = [] - return True - return False - - def fit(self, curve_fit_method): - assert self.ready() - if len(self._complete) == 1: - i = self._complete[0].image - else: - i = numpy.mean(numpy.array([v.image for v in self._complete]), axis=0) - return slicops.plot.fit_image(i, curve_fit_method) - - def ready(self): - return bool(self._complete) - - def save(self, metadata): - """Creates a hdf5 file with the structure: - /image Group - /frames Dataset {n_average, ysize, xsize} - /mean Dataset {ysize, xsize} - /timestamps Dataset {n_average} - /x Group - /fit Dataset {xsize} - /profile Dataset {xsize} - /y Group - /fit Dataset {ysize} - /profile Dataset {ysize} - /meta Group (beam_path, camera, pv, curve_fit_method) - """ - assert self.ready() - i = self.fit(metadata.curve_fit_method) - with h5py.File(self._file_path(metadata), "w") as hf: - g = hf.create_group("meta") - for f in metadata: - g.attrs[f] = metadata[f] - g = hf.create_group("image") - g.create_dataset("mean", data=i.raw_pixels) - for dim in ("x", "y"): - g2 = g.create_group(dim) - g2.create_dataset("profile", data=i[dim].lineout) - if not i[dim].fit.results: - continue - for f in i[dim].fit.results: - g2.attrs[f] = i[dim].fit.results[f] - g2.create_dataset("fit", data=i[dim].fit.fit_line) - g.create_dataset( - "frames", data=numpy.array([v.image for v in self._complete]) - ) - g.create_dataset( - "timestamps", data=[v.timestamp.timestamp() for v in self._complete] - ) - - def _file_path(self, metadata): - t = self._complete[-1].timestamp - # TODO(pjm): not sure about filename or whether it should be nested in directories - n = f"Screen-{metadata.camera}-{t.strftime('%Y-%m-%d-%f')}.h5" - p = pykern.pkio.py_path(_cfg.save_file_path).join(n) - if p.exists(): - p.remove() - return str(p) - - class _Monitor: # TODO(robnagler) handle more values besides plot def __init__(self, accessor, handler): @@ -441,7 +356,6 @@ def _init(): beam_path=("DEV_BEAM_PATH", str, "dev beam path name"), camera=("DEV_CAMERA", str, "dev camera name"), ), - save_file_path=("/tmp", str, "root path for screen save files"), ) From ff7abf11ef3b32943e84a2e585f98579be4fb6ba Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 10 Sep 2025 23:33:47 +0000 Subject: [PATCH 04/20] only send frames every n_average --- slicops/plot.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/slicops/plot.py b/slicops/plot.py index 993864c..e311eaf 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -6,11 +6,11 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp -import collections import numpy import pykern.pkio import scipy.optimize + class ImageSet: """Fits images, possibly averaging. @@ -24,9 +24,9 @@ class ImageSet: def __init__(self, meta): self.meta = meta - self._frames = collections.deque(meta.n_average) - self._timestamps = collections.deque(meta.n_average) - self._fit = None + self._frames = [] + self._timestamps = [] + self._prev = None def add_frame(self, frame, timestamp): """Add and update fit @@ -45,12 +45,18 @@ def _mean(self): self._frames.append(frame) self._timestamps.append(timestamp) - if self._ready(): - self._fit = fit_image(_mean(), self.meta.curve_fit_method) - return self._fit + if len(self._frames) == self.meta.n_average: + self._prev = PKDict( + fit=fit_image(_mean(), self.meta.curve_fit_method), + frames=self._frames, + timestamps=self._timestamps, + ) + self._frames = [] + self._timestamps =[] + return self._prev.fit def save_file(self, dir_path): - #TODO(robnagler) the naming is a bit goofy, possibly frames/{images,timestamps} and analysis. + # TODO(robnagler) the naming is a bit goofy, possibly frames/{images,timestamps} and analysis. """Creates a hdf5 file with the structure:: /image Group /frames Dataset {n_average, ysize, xsize} @@ -70,7 +76,7 @@ def save_file(self, dir_path): def _image_dim(meta_group, dim): g = meta_group.create_group(dim) - f = self._fit[dim] + f = self._prev.fit[dim] g.create_dataset("profile", data=f.lineout) if not f.fit.results: return @@ -80,8 +86,8 @@ def _image_dim(meta_group, dim): def _meta(h5_file): g = h5_file.create_group("meta") g.attrs.update(self.meta) - g.create_dataset("frames", data=self._frames) - g.create_dataset("timestamps", data=self._timestamps) + g.create_dataset("frames", data=self._prev.frames) + g.create_dataset("timestamps", data=self._prev.timestamps) def _path(): # TODO(robnagler) centralize timestamp format @@ -95,17 +101,14 @@ def _writer(path): with h5py.File(path, "w") as f: _meta(f) g = f.create_group("image") - g.dataset("mean", self._fit.raw_pixels) + g.dataset("mean", self._prev.fit.raw_pixels) _image_dim(g, "x") _image_dim(g, "y") - if not self._ready(): - return False pykern.pkio.atomic_write(_path(), writer=_writer) - return True def _ready(self): - return len(self._frames) == self._frames.maxlen + return def fit_image(image, method): From 848da4ba9aa71ec3f592b67be950df7512ad65ab Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 10 Sep 2025 23:38:36 +0000 Subject: [PATCH 05/20] tests pass --- slicops/ctx.py | 2 +- slicops/plot.py | 4 ++-- slicops/sliclet/__init__.py | 2 +- slicops/sliclet/screen.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/slicops/ctx.py b/slicops/ctx.py index 1c3f172..6ff7c26 100644 --- a/slicops/ctx.py +++ b/slicops/ctx.py @@ -166,7 +166,7 @@ def group_get(self, field, group, attr=None): return self.__ctx.fields[field].group_get(group, attr) def multi_get(self, fields): - return PKDict(k: self.__field(k).value_get() for k in fields) + return PKDict((k, self.__field(k).value_get()) for k in fields) def multi_set(self, *args): def _args(): diff --git a/slicops/plot.py b/slicops/plot.py index e311eaf..d53a908 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -38,8 +38,8 @@ def add_frame(self, frame, timestamp): PKDict: frame and fit or None if not enough frames """ - def _mean(self): - if self._frames.maxlen == 1: + def _mean(): + if self.meta.n_average == 1: return self._frames[-1] return numpy.mean(self._frames, axis=0) diff --git a/slicops/sliclet/__init__.py b/slicops/sliclet/__init__.py index a8f2e03..5cc8349 100644 --- a/slicops/sliclet/__init__.py +++ b/slicops/sliclet/__init__.py @@ -232,7 +232,7 @@ def _path(): return (pykern.util.dev_run_dir("save").ensure(dir=True),) + rv return pykern.pkconfig.Required(rv) - _cfg = pkconfig.init( + _cfg = pykern.pkconfig.init( default=("screen", str, "default sliclet"), save_file_root=_path(), ) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 524691c..78b9fd5 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -233,7 +233,7 @@ def _plot(txn): ("start_button.ui.enabled", True), ) - def __new_image_set(self, txtn): + def __new_image_set(self, txn): self.__image_set = slicops.plot.ImageSet( txn.multi_get(("beam_path", "camera", "curve_fit_method", "n_average", "pv")), ) From 48b3a7d66560a1f902f9032b5d0e17fa0ed75958 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 10 Sep 2025 23:43:33 +0000 Subject: [PATCH 06/20] seems to work --- slicops/plot.py | 17 +++++++++-------- slicops/sliclet/__init__.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/slicops/plot.py b/slicops/plot.py index d53a908..21dc8fe 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -45,14 +45,15 @@ def _mean(): self._frames.append(frame) self._timestamps.append(timestamp) - if len(self._frames) == self.meta.n_average: - self._prev = PKDict( - fit=fit_image(_mean(), self.meta.curve_fit_method), - frames=self._frames, - timestamps=self._timestamps, - ) - self._frames = [] - self._timestamps =[] + if len(self._frames) != self.meta.n_average: + return None + self._prev = PKDict( + fit=fit_image(_mean(), self.meta.curve_fit_method), + frames=self._frames, + timestamps=self._timestamps, + ) + self._frames = [] + self._timestamps =[] return self._prev.fit def save_file(self, dir_path): diff --git a/slicops/sliclet/__init__.py b/slicops/sliclet/__init__.py index 5cc8349..0d9a094 100644 --- a/slicops/sliclet/__init__.py +++ b/slicops/sliclet/__init__.py @@ -229,7 +229,7 @@ def _init(): def _path(): rv = (_cfg_py_path, "root path for screen save files") if pykern.pkconfig.in_dev_mode(): - return (pykern.util.dev_run_dir("save").ensure(dir=True),) + rv + return (pykern.util.dev_run_dir(_path).join("save").ensure(dir=True),) + rv return pykern.pkconfig.Required(rv) _cfg = pykern.pkconfig.init( From 60f072dc08fb5747541978062156129e8e432cf5 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 10 Sep 2025 23:52:46 +0000 Subject: [PATCH 07/20] timestamps not being saved properly: Object dtype dtype(O) has no native HDF5 --- slicops/plot.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/slicops/plot.py b/slicops/plot.py index 21dc8fe..0d75053 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -6,6 +6,7 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp +import h5py import numpy import pykern.pkio import scipy.optimize @@ -53,7 +54,7 @@ def _mean(): timestamps=self._timestamps, ) self._frames = [] - self._timestamps =[] + self._timestamps = [] return self._prev.fit def save_file(self, dir_path): @@ -88,15 +89,19 @@ def _meta(h5_file): g = h5_file.create_group("meta") g.attrs.update(self.meta) g.create_dataset("frames", data=self._prev.frames) - g.create_dataset("timestamps", data=self._prev.timestamps) + g.create_dataset( + "timestamps", data=(d.timestamp() for d in self._prev.timestamps) + ) def _path(): # TODO(robnagler) centralize timestamp format - t = self._timestamps[-1] - return dir_path.join( + t = self._prev.timestamps[-1] + rv = dir_path.join( t.strftime("%Y-%m"), f"{t.strftime('%Y%m%d%H%M%S')}-{self.meta.camera}.h5", ) + rv.dirpath().ensure(dir=True) + return rv def _writer(path): with h5py.File(path, "w") as f: From 5118ec710443b9a619f24647fe461767341b6820 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 11 Sep 2025 14:25:28 +0000 Subject: [PATCH 08/20] various fixes --- slicops/package_data/sliclet/screen.yaml | 10 +++++----- slicops/plot.py | 21 +++++++++++---------- slicops/sliclet/screen.py | 17 ++++++++--------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index 209d46f..c689c2f 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -24,8 +24,8 @@ fields: Gaussian: gaussian "Super Gaussian": super_gaussian value: gaussian - n_average: - prototype: Enum + images_to_average: + prototype: Integer constraints: choices: - 1 @@ -34,7 +34,7 @@ fields: - 4 - 5 ui: - label: Number of images to average + label: Images to average value: 1 plot: prototype: Dict @@ -66,7 +66,7 @@ fields: prototype: Button ui: css_kind: outline-info - label: Save + label: Save to File ui_layout: - cols: @@ -75,7 +75,7 @@ ui_layout: - beam_path - camera - pv - - n_average + - images_to_average - cell_group: - start_button - stop_button diff --git a/slicops/plot.py b/slicops/plot.py index 0d75053..84a5ede 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -19,7 +19,7 @@ class ImageSet: `save_file`. Args: - meta (PKDict): n_average, camera, curve_fit_method, pv + meta (PKDict): images_to_average, camera, curve_fit_method, pv """ @@ -40,13 +40,13 @@ def add_frame(self, frame, timestamp): """ def _mean(): - if self.meta.n_average == 1: + if self.meta.images_to_average == 1: return self._frames[-1] return numpy.mean(self._frames, axis=0) self._frames.append(frame) self._timestamps.append(timestamp) - if len(self._frames) != self.meta.n_average: + if len(self._frames) != self.meta.images_to_average: return None self._prev = PKDict( fit=fit_image(_mean(), self.meta.curve_fit_method), @@ -61,16 +61,16 @@ def save_file(self, dir_path): # TODO(robnagler) the naming is a bit goofy, possibly frames/{images,timestamps} and analysis. """Creates a hdf5 file with the structure:: /image Group - /frames Dataset {n_average, ysize, xsize} + /frames Dataset {images_to_average, ysize, xsize} /mean Dataset {ysize, xsize} - /timestamps Dataset {n_average} + /timestamps Dataset {images_to_average} /x Group /fit Dataset {xsize} /profile Dataset {xsize} /y Group /fit Dataset {ysize} /profile Dataset {ysize} - /meta Group (camera, curve_fit_method, n_average, etc.) + /meta Group (camera, curve_fit_method, images_to_average, etc.) Args: dir_path (py.path): directory @@ -82,7 +82,7 @@ def _image_dim(meta_group, dim): g.create_dataset("profile", data=f.lineout) if not f.fit.results: return - g.attrs.update(f.results) + g.attrs.update(f.fit.results) g.create_dataset("fit", data=f.fit.fit_line) def _meta(h5_file): @@ -90,7 +90,8 @@ def _meta(h5_file): g.attrs.update(self.meta) g.create_dataset("frames", data=self._prev.frames) g.create_dataset( - "timestamps", data=(d.timestamp() for d in self._prev.timestamps) + "timestamps", + data=[d.timestamp() for d in self._prev.timestamps], ) def _path(): @@ -98,7 +99,7 @@ def _path(): t = self._prev.timestamps[-1] rv = dir_path.join( t.strftime("%Y-%m"), - f"{t.strftime('%Y%m%d%H%M%S')}-{self.meta.camera}.h5", + f"{t.strftime('%Y%m%dT%H%M%SZ')}-{self.meta.camera}.h5", ) rv.dirpath().ensure(dir=True) return rv @@ -107,7 +108,7 @@ def _writer(path): with h5py.File(path, "w") as f: _meta(f) g = f.create_group("image") - g.dataset("mean", self._prev.fit.raw_pixels) + g.create_dataset("mean", data=self._prev.fit.raw_pixels) _image_dim(g, "x") _image_dim(g, "y") diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 78b9fd5..94fae17 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -24,9 +24,10 @@ _cfg = None _BUTTONS_DISABLE = ( + ("save_button.ui.enabled", False), ("single_button.ui.enabled", False), - ("stop_button.ui.enabled", False), ("start_button.ui.enabled", False), + ("stop_button.ui.enabled", False), ) _DEVICE_DISABLE = ( @@ -43,8 +44,7 @@ ("start_button.ui.visible", False), ("stop_button.ui.visible", False), ("save_button.ui.visible", False), - ("n_average.ui.visible", False), - ("save_button.ui.enabled", False), + ("images_to_average.ui.visible", False), ) + _BUTTONS_DISABLE _DEVICE_ENABLE = ( @@ -53,7 +53,7 @@ ("start_button.ui.visible", True), ("stop_button.ui.visible", True), ("save_button.ui.visible", True), - ("n_average.ui.visible", True), + ("images_to_average.ui.visible", True), ("single_button.ui.enabled", True), ("stop_button.ui.enabled", False), ("start_button.ui.enabled", True), @@ -82,8 +82,8 @@ def on_change_curve_fit_method(self, txn, value, **kwargs): # TODO(robnagler) optimize with ImageSet.update_curve_fit_method() self.__new_image_set(txn) - def on_change_n_average(self, txn, value, **kwargs): - # TODO(robnagler) optimize with ImageSet.update_n_average() + def on_change_images_to_average(self, txn, value, **kwargs): + # TODO(robnagler) optimize with ImageSet.update_images_to_average() self.__new_image_set(txn) def on_click_save_button(self, txn, **kwargs): @@ -221,8 +221,7 @@ def _plot(txn): if not txn.group_get("plot", "ui", "visible"): txn.multi_set(_PLOT_ENABLE) txn.field_set("plot", p) - if not txn.group_get("save_button", "ui", "enabled"): - txn.multi_set(("save_button.ui.enabled", True)) + txn.multi_set(("save_button.ui.enabled", True)) return True with self.lock_for_update() as txn: @@ -235,7 +234,7 @@ def _plot(txn): def __new_image_set(self, txn): self.__image_set = slicops.plot.ImageSet( - txn.multi_get(("beam_path", "camera", "curve_fit_method", "n_average", "pv")), + txn.multi_get(("beam_path", "camera", "curve_fit_method", "images_to_average", "pv")), ) def __set_acquire(self, txn, acquire): From e776c6cd04f9764d0f07a49f0f98645e9ebd751d Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 11 Sep 2025 17:46:54 +0000 Subject: [PATCH 09/20] fmt --- slicops/sliclet/screen.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 94fae17..4a66a0f 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -234,7 +234,9 @@ def _plot(txn): def __new_image_set(self, txn): self.__image_set = slicops.plot.ImageSet( - txn.multi_get(("beam_path", "camera", "curve_fit_method", "images_to_average", "pv")), + txn.multi_get( + ("beam_path", "camera", "curve_fit_method", "images_to_average", "pv") + ), ) def __set_acquire(self, txn, acquire): From ddc94994730f1cf57877daf032453e7ba70966f1 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Fri, 12 Sep 2025 22:20:51 +0000 Subject: [PATCH 10/20] Fix #161 service.ui_api.prod works with `//.*` - package_path supported properly for sliclets - ctx only supports a single (fconf) yaml file --- slicops/config.py | 2 +- slicops/ctx.py | 18 ++++-- slicops/pkcli/service.py | 63 ++++++++++++------- slicops/sliclet/__init__.py | 44 ++++++++++++- slicops/unit_util.py | 27 +++++++- .../simple.in/{ => sliclet}/input.yaml | 0 tests/sliclet_data/sliclet/unit.yaml | 10 +++ tests/sliclet_data/somepkg/__init__.py | 1 + .../sliclet_data/somepkg/sliclet/__init__.py | 1 + tests/sliclet_data/somepkg/sliclet/unit.py | 16 +++++ tests/sliclet_data/vue/index.html | 1 + tests/sliclet_test.py | 44 +++++++++++++ ui/src/components/VApp.vue | 4 +- 13 files changed, 198 insertions(+), 33 deletions(-) rename tests/ctx_data/simple.in/{ => sliclet}/input.yaml (100%) create mode 100644 tests/sliclet_data/sliclet/unit.yaml create mode 100644 tests/sliclet_data/somepkg/__init__.py create mode 100644 tests/sliclet_data/somepkg/sliclet/__init__.py create mode 100644 tests/sliclet_data/somepkg/sliclet/unit.py create mode 100644 tests/sliclet_data/vue/index.html create mode 100644 tests/sliclet_test.py diff --git a/slicops/config.py b/slicops/config.py index 27abe8b..9b2f70e 100644 --- a/slicops/config.py +++ b/slicops/config.py @@ -40,7 +40,7 @@ def cfg(): vue_port=(8008, pykern.pkasyncio.cfg_port, "port of Vue dev server"), ), package_path=( - tuple(["slicops"]), + ("slicops",), tuple, "Names of root packages that should be checked for codes and resources. Order is important, the first package with a matching code/resource will be used.", ), diff --git a/slicops/ctx.py b/slicops/ctx.py index ed2c9e7..73deda1 100644 --- a/slicops/ctx.py +++ b/slicops/ctx.py @@ -10,6 +10,7 @@ import pykern.fconf import pykern.pkresource import pykern.util +import slicops.config import slicops.field import slicops.ui_layout @@ -32,10 +33,19 @@ def _check_raw(got): step = "yaml" try: - r = pykern.fconf.parse_all( - path or pykern.pkresource.file_path("sliclet"), - glob=f"{name}*", - ) + n = f"sliclet/{name}.yaml" + r = pykern.fconf.Parser( + [ + ( + path.join(n) + if path + else pykern.pkresource.file_path( + n, + packages=slicops.config.cfg().package_path, + ) + ) + ] + ).result _check_raw(r) step = "fields" self.fields = self.__parse(r[step], PKDict(), slicops.field.prototypes()) diff --git a/slicops/pkcli/service.py b/slicops/pkcli/service.py index 9e599c6..483ddfe 100644 --- a/slicops/pkcli/service.py +++ b/slicops/pkcli/service.py @@ -24,54 +24,64 @@ def ui_api(self, tcp_port=None, prod=False): """ from pykern import pkconfig, pkresource from pykern.api import server - from slicops import config, ui_api, quest - from tornado import web + from slicops import config, quest, sliclet, ui_api - def _tcp_port(): - return ( - PKDict(tcp_port=pkconfig.parse_positive_int(tcp_port)) - if tcp_port - else PKDict() - ) + from tornado import web - def _uri_map(config): - if prod: - return [ - ( - # very specific so we control the name space - r"^/(assets/[^/.]+\.(?:css|js)|favicon.png|index.html|)$", - web.StaticFileHandler, - PKDict( - path=str(pkresource.file_path("vue")), - default_filename="index.html", - ), - ), - ] + def _dev_uri_map(config): return [ ( # send any non-api call to the proxy rf"^(?!{config.api_uri}).*", - ProxyHandler, + _ProxyHandler, PKDict( proxy_url=f"http://localhost:{config.vue_port}", ), ), ] + def _prod_uri_map(config): + d = PKDict( + # TODO(robnagler) package_path + path=str(pkresource.file_path("vue")), + default_filename="index.html", + ) + return [ + ( + # very specific so we control the name space + r"^/(assets/[^/.]+\.(?:css|js)|favicon.png|index.html|)$", + web.StaticFileHandler, + d, + ), + ( + # vue index.html is returned for sliclet URLs + rf"^/(?:$|{'|'.join(sliclet.names())}(?:$|/.*))", + _VueIndexHandler, + d, + ), + ] + + def _tcp_port(): + return ( + PKDict(tcp_port=pkconfig.parse_positive_int(tcp_port)) + if tcp_port + else PKDict() + ) + c = config.cfg().ui_api.copy() server.start( attr_classes=quest.attr_classes(), api_classes=ui_api.api_classes(), http_config=c.pkupdate( PKDict( - uri_map=_uri_map(c), + uri_map=_prod_uri_map(c) if prod else _dev_uri_map(c), **_tcp_port(), ) ), ) -class ProxyHandler(tornado.web.RequestHandler): +class _ProxyHandler(tornado.web.RequestHandler): def initialize(self, proxy_url, **kwargs): super(ProxyHandler, self).initialize(**kwargs) self.http_client = tornado.httpclient.AsyncHTTPClient() @@ -82,3 +92,8 @@ async def get(self): self.set_status(r.code) self.set_header("Content-Type", r.headers["Content-Type"]) self.write(r.body) + + +class _VueIndexHandler(tornado.web.StaticFileHandler): + def get_absolute_path(self, root, path, *args, **kwargs): + return super().get_absolute_path(root, self.default_filename) diff --git a/slicops/sliclet/__init__.py b/slicops/sliclet/__init__.py index 2a6028e..57eb641 100644 --- a/slicops/sliclet/__init__.py +++ b/slicops/sliclet/__init__.py @@ -11,9 +11,12 @@ import enum import importlib import inspect +import itertools +import pykern.pkinspect import pykern.util import queue import re +import slicops.config import slicops.ctx import slicops.field import threading @@ -30,11 +33,50 @@ class _Work(enum.IntEnum): _CTX_WRITE_ARGS = frozenset(["field_values"]) +_NAMES = None + def instance(name, queue): + def _import(name): + # TODO(robnagler) move to pykern, copied from sirepo.util + # NOTE: fixed a bug (s = None) + s = None + for p in slicops.config.cfg().package_path: + n = None + try: + n = f"{p}.sliclet.{name}" + return importlib.import_module(n) + except ModuleNotFoundError as e: + if n is not None and n != e.name: + # import is failing due to ModuleNotFoundError in a sub-import + # not the module we are looking for + raise + s = pkdexc() + pass + # gives more debugging info (perhaps more confusion) + if s: + pkdc(s) + raise AssertionError( + f"cannot find sliclet={name} in package_path={slicops.config.cfg().package_path}" + ) + if not name: name = _cfg.default - return importlib.import_module(f"slicops.sliclet.{name}").CLASS(name, queue) + return _import(name).CLASS(name, queue) + + +def names(): + + def _find(): + # TODO(robnagler) move to pykern, copied from sirepo + for p in slicops.config.cfg().package_path: + yield pykern.pkinspect.package_module_names(f"{p}.sliclet") + + global _NAMES + + if _NAMES is None: + _NAMES = tuple(sorted(itertools.chain.from_iterable(_find()))) + return _NAMES class Base: diff --git a/slicops/unit_util.py b/slicops/unit_util.py index 208c649..4e018cd 100644 --- a/slicops/unit_util.py +++ b/slicops/unit_util.py @@ -21,6 +21,9 @@ def __init__(self, sliclet, *args, **kwargs): self.__caproto = c super().__init__(*args, **kwargs) self.__update_q = asyncio.Queue() + self.__http_uri = ( + f"http://{self.http_config.tcp_ip}:{self.http_config.tcp_port}" + ) async def ctx_update(self): from pykern import pkunit @@ -39,6 +42,24 @@ async def ctx_field_set(self, **kwargs): self.__caller() await self.client.call_api("ui_ctx_write", PKDict(field_values=PKDict(kwargs))) + async def http_get(self, rel_uri): + from tornado import httpclient + + def _client(): + return httpclient.AsyncHTTPClient(force_instance=True) + + self.__caller() + return ( + await _client().fetch( + httpclient.HTTPRequest( + connect_timeout=_TIMEOUT, + method="GET", + request_timeout=_TIMEOUT, + url=self.__http_uri + rel_uri, + ), + ) + ).body + async def __aenter__(self): await super().__aenter__() asyncio.create_task(self.__subscribe()) @@ -69,8 +90,12 @@ def _server_config(self, *args, **kwargs): def _server_start(self, *args, **kwargs): from slicops.pkcli import service + from pykern.pkcollections import PKDict - service.Commands().ui_api() + k = PKDict() + if self.server_config.get("prod"): + k.prod = True + service.Commands().ui_api(**k) def __caller(self): from pykern import pkdebug, pkinspect diff --git a/tests/ctx_data/simple.in/input.yaml b/tests/ctx_data/simple.in/sliclet/input.yaml similarity index 100% rename from tests/ctx_data/simple.in/input.yaml rename to tests/ctx_data/simple.in/sliclet/input.yaml diff --git a/tests/sliclet_data/sliclet/unit.yaml b/tests/sliclet_data/sliclet/unit.yaml new file mode 100644 index 0000000..5f6bf21 --- /dev/null +++ b/tests/sliclet_data/sliclet/unit.yaml @@ -0,0 +1,10 @@ +fields: + one: + prototype: Integer + value: 1 + +ui_layout: + - cols: + - css: col-lg-6 + rows: + - one diff --git a/tests/sliclet_data/somepkg/__init__.py b/tests/sliclet_data/somepkg/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/tests/sliclet_data/somepkg/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/sliclet_data/somepkg/sliclet/__init__.py b/tests/sliclet_data/somepkg/sliclet/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/tests/sliclet_data/somepkg/sliclet/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/sliclet_data/somepkg/sliclet/unit.py b/tests/sliclet_data/somepkg/sliclet/unit.py new file mode 100644 index 0000000..85021c9 --- /dev/null +++ b/tests/sliclet_data/somepkg/sliclet/unit.py @@ -0,0 +1,16 @@ +""" + +:copyright: Copyright (c) 2025 The Board of Trustees of the Leland Stanford Junior University, through SLAC National Accelerator Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All Rights Reserved. +:license: http://github.com/slaclab/slicops/LICENSE +""" + +from pykern.pkcollections import PKDict +from pykern.pkdebug import pkdc, pkdlog, pkdp +import slicops.sliclet + + +class Unit(slicops.sliclet.Base): + pass + + +CLASS = Unit diff --git a/tests/sliclet_data/vue/index.html b/tests/sliclet_data/vue/index.html new file mode 100644 index 0000000..cf7711b --- /dev/null +++ b/tests/sliclet_data/vue/index.html @@ -0,0 +1 @@ +xyzzy diff --git a/tests/sliclet_test.py b/tests/sliclet_test.py new file mode 100644 index 0000000..4c681a5 --- /dev/null +++ b/tests/sliclet_test.py @@ -0,0 +1,44 @@ +"""Test sliclet + +:copyright: Copyright (c) 2025 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +import pytest + + +@pytest.mark.asyncio(loop_scope="module") +async def test_all(monkeypatch): + prev_file_path = None + + def _file_path(path, *args, **kwargs): + nonlocal prev_file_path + from pykern import pkinspect, pkdebug + + if path in ("vue", "sliclet/unit.yaml"): + return pkunit.data_dir().join(path) + return prev_file_path(path, caller_context=pkinspect.caller_module()) + + def _init(): + nonlocal prev_file_path + + import os, sys + + os.environ["SLICOPS_CONFIG_PACKAGE_PATH"] = "somepkg:slicops" + + from pykern import pkunit + + sys.path.insert(0, str(pkunit.data_dir())) + + from pykern import pkresource + + prev_file_path = pkresource.file_path + monkeypatch.setattr(pkresource, "file_path", _file_path) + + _init() + + from slicops import unit_util + from pykern import pkunit, pkdebug + + async with unit_util.SlicletSetup("unit", prod=True) as s: + pkunit.pkeq(b"xyzzy\n", await s.http_get("")) diff --git a/ui/src/components/VApp.vue b/ui/src/components/VApp.vue index c5175c0..b34551d 100644 --- a/ui/src/components/VApp.vue +++ b/ui/src/components/VApp.vue @@ -92,8 +92,8 @@ document.title = result.sliclet_title; if (! props.sliclet) { const u = '/' + result.sliclet_name; - if (route.path !== u) { - // avoid a possible redirect loop + // if default route or new app, just switch to it + if (route.path == '/' || ! (new RegExp(`^${u}(?:/|$)`)).test(route.path)) { router.replace(u); } } From 4ee173b5647730284b1305df55054c810da21252 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Fri, 12 Sep 2025 23:01:10 +0000 Subject: [PATCH 11/20] /screen does not work --- slicops/pkcli/service.py | 1 + tests/sliclet_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/slicops/pkcli/service.py b/slicops/pkcli/service.py index 483ddfe..7aa9f41 100644 --- a/slicops/pkcli/service.py +++ b/slicops/pkcli/service.py @@ -96,4 +96,5 @@ async def get(self): class _VueIndexHandler(tornado.web.StaticFileHandler): def get_absolute_path(self, root, path, *args, **kwargs): + pkdp(path) return super().get_absolute_path(root, self.default_filename) diff --git a/tests/sliclet_test.py b/tests/sliclet_test.py index 4c681a5..d659860 100644 --- a/tests/sliclet_test.py +++ b/tests/sliclet_test.py @@ -42,3 +42,4 @@ def _init(): async with unit_util.SlicletSetup("unit", prod=True) as s: pkunit.pkeq(b"xyzzy\n", await s.http_get("")) + pkunit.pkeq(b"xyzzy\n", await s.http_get("/screen")) From ade83f3850899471871941072b6229ff4c02281c Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Tue, 16 Sep 2025 17:53:49 +0000 Subject: [PATCH 12/20] for #141 rename save_button, improve layout --- slicops/package_data/sliclet/screen.yaml | 16 +++++++++------- slicops/sliclet/screen.py | 12 ++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index c689c2f..c5e9acb 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -25,7 +25,7 @@ fields: "Super Gaussian": super_gaussian value: gaussian images_to_average: - prototype: Integer + prototype: Enum constraints: choices: - 1 @@ -62,7 +62,7 @@ fields: ui: css_kind: danger label: Stop - save_button: + save_to_file: prototype: Button ui: css_kind: outline-info @@ -70,7 +70,7 @@ fields: ui_layout: - cols: - - css: col-sm-3 + - css: col-md-3 rows: - beam_path - camera @@ -80,14 +80,16 @@ ui_layout: - start_button - stop_button - single_button - - save_button - - css: col-sm-9 col-xxl-7 + - css: col-md-8 col-xxl-7 rows: - plot - cols: - - css: col-sm-3 + - css: col-md-3 rows: - curve_fit_method - - css: col-sm-3 + - css: col-md-3 rows: - color_map + - css: col-md-1 + rows: + - save_to_file diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 77273df..1183ff5 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -24,7 +24,6 @@ _cfg = None _BUTTONS_DISABLE = ( - ("save_button.ui.enabled", False), ("single_button.ui.enabled", False), ("start_button.ui.enabled", False), ("stop_button.ui.enabled", False), @@ -35,16 +34,17 @@ ("color_map.ui.visible", False), ("curve_fit_method.ui.enabled", False), ("curve_fit_method.ui.visible", False), + ("images_to_average.ui.visible", False), ("plot.ui.visible", False), # Useful to avoid large ctx sends ("plot.value", None), ("pv.ui.visible", False), ("pv.value", None), + ("save_to_file.ui.enabled", False), + ("save_to_file.ui.visible", False), ("single_button.ui.visible", False), ("start_button.ui.visible", False), ("stop_button.ui.visible", False), - ("save_button.ui.visible", False), - ("images_to_average.ui.visible", False), ) + _BUTTONS_DISABLE _DEVICE_ENABLE = ( @@ -52,7 +52,6 @@ ("single_button.ui.visible", True), ("start_button.ui.visible", True), ("stop_button.ui.visible", True), - ("save_button.ui.visible", True), ("images_to_average.ui.visible", True), ("single_button.ui.enabled", True), ("stop_button.ui.enabled", False), @@ -65,6 +64,8 @@ ("curve_fit_method.ui.enabled", True), ("curve_fit_method.ui.visible", True), ("plot.ui.visible", True), + ("save_to_file.ui.enabled", True), + ("save_to_file.ui.visible", True), ) @@ -86,7 +87,7 @@ def on_change_images_to_average(self, txn, value, **kwargs): # TODO(robnagler) optimize with ImageSet.update_images_to_average() self.__new_image_set(txn) - def on_click_save_button(self, txn, **kwargs): + def on_click_save_to_file(self, txn, **kwargs): self.__image_set.save_file(self.save_file_path()) def on_click_single_button(self, txn, **kwargs): @@ -222,7 +223,6 @@ def _plot(txn): if not txn.group_get("plot", "ui", "visible"): txn.multi_set(_PLOT_ENABLE) txn.field_set("plot", p) - txn.multi_set(("save_button.ui.enabled", True)) return True with self.lock_for_update() as txn: From 8ab186b8ed843c272658cc7ed1e478d97689a29a Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Tue, 16 Sep 2025 20:53:56 +0000 Subject: [PATCH 13/20] for #141 added plot test --- slicops/plot.py | 3 - tests/plot_test.py | 144 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 tests/plot_test.py diff --git a/slicops/plot.py b/slicops/plot.py index 84a5ede..9126225 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -114,9 +114,6 @@ def _writer(path): pykern.pkio.atomic_write(_path(), writer=_writer) - def _ready(self): - return - def fit_image(image, method): """Attemp an analytical fit for the sum along the x and y dimensions diff --git a/tests/plot_test.py b/tests/plot_test.py new file mode 100644 index 0000000..b98fe6c --- /dev/null +++ b/tests/plot_test.py @@ -0,0 +1,144 @@ +"""Test plot + +:copyright: Copyright (c) 2025 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +from pykern import pkunit + + +def test_imageset_save_to_file(): + from glob import glob + from pykern import pkio + import h5py + + i = _imageset() + with pkio.save_chdir(pkunit.work_dir()) as w: + i.imageset.save_file(w) + with h5py.File(glob("2024-09/*.h5")[0]) as h: + pkunit.pkeq( + h["/image/mean"][:].tolist(), + i.expected_mean, + ) + + +def test_imageset_stats(): + i = _imageset() + f = i.frame + pkunit.pkeq( + f.raw_pixels.tolist(), + i.expected_mean, + ) + pkunit.pkeq( + f.x.lineout.tolist(), + [0.5, 2.5, 12.5, 2.5, 0.5], + ) + pkunit.pkeq( + f.y.lineout.tolist(), + [3, 5, 7.5, 3], + ) + pkunit.pkeq(round(f.x.fit.results.sig, 2), 0.53) + pkunit.pkeq([round(v, 2) for v in f.x.fit.fit_line], [0.5, 2.5, 12.5, 2.5, 0.5]) + + +def _imageset(): + from datetime import datetime + from pykern.pkcollections import PKDict + from slicops.plot import ImageSet + import numpy + + def _image(values): + return numpy.array(values).reshape(4, 5) + + i = ImageSet( + PKDict( + images_to_average=2, + camera="Test", + curve_fit_method="gaussian", + pv="test", + ) + ) + pkunit.pkeq( + i.add_frame( + _image( + [ + 0, + 0, + 5, + 0, + 0, + 0, + 5, + 10, + 0, + 0, + 0, + 0, + 5, + 5, + 0, + 0, + 0, + 5, + 0, + 0, + ] + ), + datetime.strptime("2024-09-19 15:45:30", "%Y-%m-%d %H:%M:%S"), + ), + None, + ) + return PKDict( + imageset=i, + frame=i.add_frame( + _image( + [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + ] + ), + datetime.strptime("2024-09-19 15:45:31", "%Y-%m-%d %H:%M:%S"), + ), + expected_mean=_image( + [ + 0.5, + 0, + 2.5, + 0, + 0, + 0, + 2.5, + 5, + 0, + 0, + 0, + 0, + 2.5, + 2.5, + 0, + 0, + 0, + 2.5, + 0, + 0.5, + ] + ).tolist(), + ) From b7b85f9d3dd5ed298b5e3a8191e6b6249f409907 Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Tue, 16 Sep 2025 20:56:52 +0000 Subject: [PATCH 14/20] for #141 remove pkdp() --- slicops/pkcli/ioc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slicops/pkcli/ioc.py b/slicops/pkcli/ioc.py index a62ed24..4b9dd48 100644 --- a/slicops/pkcli/ioc.py +++ b/slicops/pkcli/ioc.py @@ -25,7 +25,7 @@ def _pvgroup(config): return PKDict( # Need to hardwire the defaults, because ioc_arg_parser uses # argparse globally which causes a mess with argh (which uses argparse) - pvdb=pkdp(_PVGroup(config, db_yaml, macros={}, prefix="").pvdb), + pvdb=_PVGroup(config, db_yaml, macros={}, prefix="").pvdb, interfaces=["0.0.0.0"], module_name="caproto.asyncio.server", log_pv_names=False, From 0a446624d2ebf4a4ac96bbb373cfc18c77724b06 Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Tue, 16 Sep 2025 21:02:25 +0000 Subject: [PATCH 15/20] for #141 remove unused imports --- slicops/sliclet/screen.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 1183ff5..f9e2f40 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -4,11 +4,8 @@ :license: http://github.com/slaclab/slicops/LICENSE """ -from datetime import datetime from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp -import h5py -import numpy import pykern.pkcompat import pykern.pkconfig import pykern.util From b6256e4f2becdc81ac68caca9c284baba235e1d4 Mon Sep 17 00:00:00 2001 From: Paul Moeller Date: Thu, 18 Sep 2025 19:25:04 +0000 Subject: [PATCH 16/20] fix #161 added work-around so prod sliclet route correctly --- slicops/package_data/sliclet/screen.yaml | 14 +++++++------- slicops/pkcli/service.py | 9 +++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index c5e9acb..a1a9812 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -70,7 +70,7 @@ fields: ui_layout: - cols: - - css: col-md-3 + - css: col-lg-3 rows: - beam_path - camera @@ -80,16 +80,16 @@ ui_layout: - start_button - stop_button - single_button - - css: col-md-8 col-xxl-7 + - css: col-lg-9 col-xxl-7 rows: - plot - cols: - - css: col-md-3 + - css: col-lg-3 rows: - curve_fit_method - - css: col-md-3 + - css: col-lg-3 rows: - color_map - - css: col-md-1 - rows: - - save_to_file + # - css: col-md-1 + # rows: + # - save_to_file diff --git a/slicops/pkcli/service.py b/slicops/pkcli/service.py index 9e599c6..d9295a4 100644 --- a/slicops/pkcli/service.py +++ b/slicops/pkcli/service.py @@ -46,6 +46,15 @@ def _uri_map(config): default_filename="index.html", ), ), + ( + # TODO(pjm): this should only route sliclet names + # route everything except the api_uri else to index.html + rf"^()(?!{config.api_uri}).*", + web.StaticFileHandler, + PKDict( + path=str(pkresource.file_path("vue")) + "/index.html", + ), + ), ] return [ ( From 55803adfba44db20af57751417153db317489cc2 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:00:03 +0000 Subject: [PATCH 17/20] seems to work for dev & prod --- slicops/pkcli/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/slicops/pkcli/service.py b/slicops/pkcli/service.py index 7aa9f41..ab603e2 100644 --- a/slicops/pkcli/service.py +++ b/slicops/pkcli/service.py @@ -47,6 +47,7 @@ def _prod_uri_map(config): default_filename="index.html", ) return [ + # NOTE: StaticFileHandler requires match returns a group ( # very specific so we control the name space r"^/(assets/[^/.]+\.(?:css|js)|favicon.png|index.html|)$", @@ -55,7 +56,7 @@ def _prod_uri_map(config): ), ( # vue index.html is returned for sliclet URLs - rf"^/(?:$|{'|'.join(sliclet.names())}(?:$|/.*))", + rf"^/($|(?:{'|'.join(sliclet.names())})(?:$|/.*))", _VueIndexHandler, d, ), @@ -83,7 +84,7 @@ def _tcp_port(): class _ProxyHandler(tornado.web.RequestHandler): def initialize(self, proxy_url, **kwargs): - super(ProxyHandler, self).initialize(**kwargs) + super().initialize(**kwargs) self.http_client = tornado.httpclient.AsyncHTTPClient() self.proxy_url = proxy_url @@ -96,5 +97,4 @@ async def get(self): class _VueIndexHandler(tornado.web.StaticFileHandler): def get_absolute_path(self, root, path, *args, **kwargs): - pkdp(path) return super().get_absolute_path(root, self.default_filename) From 9ac2c9304fd4b747f217af301e740400882e9683 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:01:18 +0000 Subject: [PATCH 18/20] gitignore --- slicops/package_data/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slicops/package_data/.gitignore b/slicops/package_data/.gitignore index 5970902..3ed971e 100644 --- a/slicops/package_data/.gitignore +++ b/slicops/package_data/.gitignore @@ -1,2 +1,2 @@ -vue/ +vue ng-build/ From 3d7176224336d67ecea9671d1ff6595991190838 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 4 Feb 2026 22:49:09 +0000 Subject: [PATCH 19/20] fix merge --- slicops/sliclet/screen.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 43bd70d..ddbf6bf 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -104,9 +104,6 @@ def on_change_camera(self, txn, value, **kwargs): def on_change_beam_path(self, txn, value, **kwargs): self.__beam_path_change(txn, value) - def on_change_camera(self, txn, value, **kwargs): - self.__device_change(txn, value) - def on_change_curve_fit_method(self, txn, value, **kwargs): # TODO(robnagler) optimize with ImageSet.update_curve_fit_method() self.__new_image_set(txn) From 8f42be35ff2780e2ae958216e995145396b73d17 Mon Sep 17 00:00:00 2001 From: Rob Nagler <5495179+robnagler@users.noreply.github.com> Date: Wed, 4 Feb 2026 22:54:07 +0000 Subject: [PATCH 20/20] ckp --- slicops/package_data/sliclet/screen.yaml | 2 +- slicops/sliclet/screen.py | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index 4b80efb..b1f096d 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -111,5 +111,5 @@ ui_layout: rows: - color_map - css: col-md-1 - rows: + rows: - save_to_file diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index ddbf6bf..e1628b2 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -250,23 +250,7 @@ def __handle_device_error(self, exc): self.put_exception(exc) def __handle_image(self, image): - def _plot(txn): - if not self.__device: - return False - if (i := self.__monitors.image.prev_value()) is None or not i.size: - return False - if ( - p := self.__image_set.add_frame(image, pykern.pkcompat.utcnow()) - ) is None: - return False - if not txn.group_get("plot", "ui", "visible"): - txn.multi_set(_PLOT_ENABLE) - txn.field_set("plot", p) - return True - with self.lock_for_update() as txn: - if _plot(txn) and self.__single_button: - self.__set_acquire(txn, False) self.__current_value["image"] = image if self.__update_plot(txn) and self.__single_button: # self.__single_button = False