From 4f626afdfe7480168f0c5d9844893f1b08a8fe37 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 27 Aug 2024 11:28:33 -0700 Subject: [PATCH 01/13] A couple comments were added to some code. --- mkidpipeline/steps/lincal.py | 7 ++++--- mkidpipeline/steps/wavecal.py | 5 +++-- pyproject.toml | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index 275b97e0..875893ab 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -9,7 +9,8 @@ """ import time import numpy as np -from progressbar import ProgressBar, NullBar +from progressbar import ProgressBar, NullBar # EDIT - To be replaced with tqdm +import tqdm import mkidpipeline.definitions as definitions from mkidcore.corelog import getLogger @@ -63,9 +64,9 @@ def apply(o: definitions.MKIDTimerange, config=None): lastpct = 0 if cfg.get('lincal.ncpu') > 1: - bar = NullBar() + bar = NullBar() # EDIT - Replace using tqdm else: - bar = ProgressBar(max_value=n_to_do) + bar = ProgressBar(max_value=n_to_do) # EDIT - Replace using tqdm #Not ram intensive ~250MB peak diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index f3b1a59e..e1ae6d35 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -6,7 +6,8 @@ import multiprocessing as mp import time from distutils.spawn import find_executable -import progressbar as pb +import progressbar as pb # EDIT - Change to using tqdm +import tqdm import scipy import astropy.constants @@ -891,7 +892,7 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: if initialize: - percentage = pb.Percentage() + percentage = pb.Percentage() # EDIT - Change to using tqdm bar = pb.Bar() timer = pb.Timer() eta = pb.ETA() diff --git a/pyproject.toml b/pyproject.toml index e092ec17..fbf96301 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "matplotlib>=3.5", "cycler>=0.11", "progressbar2>=4.0.0", + "tqdm>=4.66.5", "psutil>=5.0.0", "tables>=3.8.0", From 15e8bd2d1b1c35f599b7297991bbe7e5d93837e9 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 27 Aug 2024 11:28:33 -0700 Subject: [PATCH 02/13] A couple comments were added to some code. --- mkidpipeline/steps/lincal.py | 7 ++++--- mkidpipeline/steps/wavecal.py | 5 +++-- pyproject.toml | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index 275b97e0..875893ab 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -9,7 +9,8 @@ """ import time import numpy as np -from progressbar import ProgressBar, NullBar +from progressbar import ProgressBar, NullBar # EDIT - To be replaced with tqdm +import tqdm import mkidpipeline.definitions as definitions from mkidcore.corelog import getLogger @@ -63,9 +64,9 @@ def apply(o: definitions.MKIDTimerange, config=None): lastpct = 0 if cfg.get('lincal.ncpu') > 1: - bar = NullBar() + bar = NullBar() # EDIT - Replace using tqdm else: - bar = ProgressBar(max_value=n_to_do) + bar = ProgressBar(max_value=n_to_do) # EDIT - Replace using tqdm #Not ram intensive ~250MB peak diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index f3b1a59e..e1ae6d35 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -6,7 +6,8 @@ import multiprocessing as mp import time from distutils.spawn import find_executable -import progressbar as pb +import progressbar as pb # EDIT - Change to using tqdm +import tqdm import scipy import astropy.constants @@ -891,7 +892,7 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: if initialize: - percentage = pb.Percentage() + percentage = pb.Percentage() # EDIT - Change to using tqdm bar = pb.Bar() timer = pb.Timer() eta = pb.ETA() diff --git a/pyproject.toml b/pyproject.toml index d738e00e..3c7da39c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "matplotlib>=3.5", "cycler>=0.11", "progressbar2>=4.0.0", + "tqdm>=4.66.5", "psutil>=5.0.0", "tables>=3.8.0", From bd6041cc69abe29418d906c5f67b6df2e296ebc8 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 27 Aug 2024 14:44:40 -0700 Subject: [PATCH 03/13] Updating lincal.py to use tqdm for progress bar --- mkidpipeline/steps/lincal.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index 875893ab..db150488 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -9,8 +9,7 @@ """ import time import numpy as np -from progressbar import ProgressBar, NullBar # EDIT - To be replaced with tqdm -import tqdm +import tqdm # Got rid of progressbar import in favor of this. import mkidpipeline.definitions as definitions from mkidcore.corelog import getLogger @@ -63,15 +62,18 @@ def apply(o: definitions.MKIDTimerange, config=None): n_to_do = np.count_nonzero(~of.flagged(PROBLEM_FLAGS, all_flags=False)) lastpct = 0 + if cfg.get('lincal.ncpu') > 1: - bar = NullBar() # EDIT - Replace using tqdm - else: - bar = ProgressBar(max_value=n_to_do) # EDIT - Replace using tqdm + disable_bar = True + print('\nNo progress bar displayed due to multiple cpus used for lincal\n') + else: + disable_bar = False + #Not ram intensive ~250MB peak - for done, resid in bar(enumerate(of.resonators(exclude=PROBLEM_FLAGS))): + for done, resid in tqdm.tqdm(enumerate(of.resonators(exclude=PROBLEM_FLAGS)), disable = disable_bar): # Using tqdm here for a progress bar. indices = of.photonTable.get_where_list('resID==resid') if not indices.size: continue From 33dcd6b78953a7cb4d507e790acfb57aaa395c78 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 27 Aug 2024 16:30:19 -0700 Subject: [PATCH 04/13] Both lincal.py and wavecal.py updated to use tqdm for progress bar. --- mkidpipeline/steps/lincal.py | 2 +- mkidpipeline/steps/wavecal.py | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index db150488..df815277 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -65,7 +65,7 @@ def apply(o: definitions.MKIDTimerange, config=None): if cfg.get('lincal.ncpu') > 1: disable_bar = True - print('\nNo progress bar displayed due to multiple cpus used for lincal\n') + print('\nNo progress bar displayed due to multiple cpus used for lincal.\n') else: disable_bar = False diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index e1ae6d35..5d316948 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -6,7 +6,6 @@ import multiprocessing as mp import time from distutils.spawn import find_executable -import progressbar as pb # EDIT - Change to using tqdm import tqdm import scipy @@ -182,7 +181,6 @@ def __init__(self, configuration, solution_name='solution.npz', _shared_tables=N # define attributes self.solution_name = solution_name # solution name for saving self.progress = None # will be replaced with progress bar - self.progress_iteration = None # will be replaced with progress bar counter self._acquired = 0 # counter for number of pixel data sets acquired self._max_queue_size = 300 # max queue size for _parallel() method self._shared_tables = _shared_tables # shared photon tables @@ -892,20 +890,13 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: if initialize: - percentage = pb.Percentage() # EDIT - Change to using tqdm - bar = pb.Bar() - timer = pb.Timer() - eta = pb.ETA() - self.progress = pb.ProgressBar(widgets=[percentage, bar, ' (', timer, ') ', eta, ' '], - max_value=number).start() - self.progress_iteration = -1 + self.progress = tqdm.tqdm(total = number) elif finish: - self.progress_iteration += 1 - self.progress.update(self.progress_iteration) - self.progress.finish() + # Note that the updating convention has changed slightly. 1 is now added to the progress bar counter prior to each operation in the iterable rather than after each item finishes. I.e. the progress bar at "3" means that we are currently working on the 3rd item. + self.progress.update() + self.progress.close() else: - self.progress_iteration += 1 - self.progress.update(self.progress_iteration) + self.progress.update() def _setup(self, pixels, wavelengths): # check inputs From a0aba43920976c96e31daa4f8eaedfb8d67f953c Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 27 Aug 2024 17:01:22 -0700 Subject: [PATCH 05/13] Some pyproject.toml changes --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c7da39c..c6fb8334 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ "matplotlib>=3.5", "cycler>=0.11", - "progressbar2>=4.0.0", "tqdm>=4.66.5", "psutil>=5.0.0", From c0f0dde22fbad71600734de4e3803ad61a128b07 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Thu, 29 Aug 2024 15:40:30 -0700 Subject: [PATCH 06/13] Correction to definitions.py and temporary edit to wavecal.py --- mkidpipeline/definitions.py | 2 +- mkidpipeline/steps/wavecal.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mkidpipeline/definitions.py b/mkidpipeline/definitions.py index 50d64b1e..21a81bf4 100644 --- a/mkidpipeline/definitions.py +++ b/mkidpipeline/definitions.py @@ -364,7 +364,7 @@ def _metadata(self): try: d.update(dict(E_CXREFX=self.wcscal.conex_ref[0], E_CXREFY=self.wcscal.conex_ref[1], E_PREFX=self.wcscal.pixel_ref[0], E_PREFY=self.wcscal.pixel_ref[1], - E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale else self.photontable.query_header( + E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale is not None else self.photontable.query_header( 'E_PLTSCL'), E_DPDCX=self.wcscal.dp_dcx if self.wcscal.dp_dcx else self.photontable.query_header( 'E_DPDCX'), diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index 5d316948..aa6955c4 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -1,4 +1,8 @@ import os +import time # EDIT +time.sleep(5) +print('\n\nWavecal.py is running.') +time.sleep(5) import queue import warnings from logging import getLogger @@ -889,6 +893,9 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: + time.sleep(5) + print('\n\n\n\n\n\n', 'running _update_progres() ', '\n\n\n\n\n\n') # EDIT + time.sleep(5) if initialize: self.progress = tqdm.tqdm(total = number) elif finish: From a7c7cae1be7633464582b44c807a9bf99f3a38a4 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Fri, 30 Aug 2024 11:47:57 -0700 Subject: [PATCH 07/13] Reverted edit to wavecal.py --- mkidpipeline/steps/wavecal.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index aa6955c4..5d316948 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -1,8 +1,4 @@ import os -import time # EDIT -time.sleep(5) -print('\n\nWavecal.py is running.') -time.sleep(5) import queue import warnings from logging import getLogger @@ -893,9 +889,6 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: - time.sleep(5) - print('\n\n\n\n\n\n', 'running _update_progres() ', '\n\n\n\n\n\n') # EDIT - time.sleep(5) if initialize: self.progress = tqdm.tqdm(total = number) elif finish: From 4b6e8ff68ee7bc699afe66ef0bd0c7f1ae744d96 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Fri, 30 Aug 2024 12:04:40 -0700 Subject: [PATCH 08/13] Change to correct imports. --- mkidpipeline/steps/wavecal.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index 6eef4d0d..58675ede 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -5,13 +5,8 @@ import numpy as np import multiprocessing as mp import time -<<<<<<< HEAD -from distutils.spawn import find_executable import tqdm -======= import shutil -import progressbar as pb ->>>>>>> main import scipy import astropy.constants From 2b571273f675dad352eeadac9dc820d5040aeffd Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Wed, 4 Sep 2024 12:28:10 -0700 Subject: [PATCH 09/13] Fixed spacing in pyproject.toml and print statement in lincal.py --- mkidpipeline/steps/lincal.py | 2 +- pyproject.toml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index df815277..6b0f0b0b 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -65,7 +65,7 @@ def apply(o: definitions.MKIDTimerange, config=None): if cfg.get('lincal.ncpu') > 1: disable_bar = True - print('\nNo progress bar displayed due to multiple cpus used for lincal.\n') + getLogger(__name__).info('\nNo progress bar displayed due to multiple cpus used for lincal.\n') else: disable_bar = False diff --git a/pyproject.toml b/pyproject.toml index 02804c66..cfb5db06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,24 +16,20 @@ requires-python = "~=3.10" dependencies = [ "mkidcore>=1.9,<2.0", "setuptools>=72.2.0", - "numpy>=1.26", "mpmath>=1.0.0", "numexpr>=2.0.0", "lmfit>=1.3.0", "SharedArray>=3.0.0", - "astropy>=5.0", "astroquery>=0.4.7", "specutils>=1.0", "photutils>=1.13", "scikit-image>=0.20.0", "drizzle>=1.0", - "matplotlib>=3.5", "cycler>=0.11", "tqdm>=4.66.5", - "psutil>=5.0.0", "tables>=3.8.0", "ruamel.yaml>0.18", From ec1792eec0d6ea253fd93967ca76f01aa32685ab Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Mon, 9 Sep 2024 15:07:36 -0700 Subject: [PATCH 10/13] Replaced pkg_resources with importlib.resources for mkidpipeline --- mkidpipeline/mkidpipe.py | 4 ++-- mkidpipeline/samples.py | 10 ++++------ mkidpipeline/tests/h5speed.py | 8 ++++---- mkidpipeline/tests/output_validation.py | 4 ++-- mkidpipeline/tests/test_config.py | 4 ++-- mkidpipeline/tests/test_wavecal.py | 8 ++++---- mkidpipeline/utils/output_validation.py | 4 ++-- 7 files changed, 20 insertions(+), 22 deletions(-) diff --git a/mkidpipeline/mkidpipe.py b/mkidpipeline/mkidpipe.py index 0555002c..76f45f58 100644 --- a/mkidpipeline/mkidpipe.py +++ b/mkidpipeline/mkidpipe.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 +from importlib import resources as rs import argparse import os import sys -import pkg_resources as pkg from datetime import datetime import mkidpipeline.definitions as definitions @@ -71,7 +71,7 @@ def parser(): dest="logcfg", help="Run the pipeline on the outputs", type=str, - default=pkg.resource_filename("mkidpipeline", "./config/logging.yaml"), + default=rs.files('mkidpipeline').joinpath('./config/logging.yaml'), ) return parser diff --git a/mkidpipeline/samples.py b/mkidpipeline/samples.py index 377bf69e..cc9666a8 100644 --- a/mkidpipeline/samples.py +++ b/mkidpipeline/samples.py @@ -1,6 +1,5 @@ -import pkg_resources +from importlib import resources as rs from collections import defaultdict - import mkidpipeline.definitions as d import mkidpipeline.config as config @@ -47,8 +46,7 @@ def _namer(name='Thing'): # Speccal d.MKIDSpeccal(name=_namer('speccal'), data=d.MKIDObservation(name=_namer('star'), start=1602049166, duration=10, wavecal='wavecal0', - spectrum=pkg_resources.resource_filename('mkidpipeline', - 'data/sample_spectrum.txt')), + spectrum=rs.files('mkidpipeline').joinpath('data/sample_spectrum.txt')), aperture=('15h22m32.3', '30.32 deg', '200 mas')), # WCS cal @@ -68,7 +66,7 @@ def _namer(name='Thing'): header=dict(OBJECT="HIP 109427"), flatcal='flatcal0', speccal='', use='0,2,4-9', wcscal='wcscal0'), d.MKIDDither(name=_namer('dither'), - data=pkg_resources.resource_filename('mkidpipeline', 'data/dither_sample.log'), + data=rs.files('mkidpipeline').joinpath('data/dither_sample.log'), wavecal='wavecal0', flatcal='flatcal0', speccal='', use=(1,), wcscal='wcscal0'), d.MKIDDither(name=_namer('dither'), flatcal='', speccal='', wcscal='', wavecal='', @@ -140,4 +138,4 @@ def get_sample_output(dataset='default'): data.append(d.MKIDOutput(name=_namer('out'), data='dither0', min_wave='950 nm', max_wave='1375 nm', kind='movie', duration=10.0, movie_format='mp4', movie_runtime=3, movie_type='both')) - return data + return data \ No newline at end of file diff --git a/mkidpipeline/tests/h5speed.py b/mkidpipeline/tests/h5speed.py index 343eb266..5b488f92 100644 --- a/mkidpipeline/tests/h5speed.py +++ b/mkidpipeline/tests/h5speed.py @@ -6,7 +6,7 @@ import numpy as np import tables import time, timeit, copy, pickle, hashlib, re, collections, ast -from pkg_resources import resource_filename +from importlib import resources as rs import matplotlib.pyplot as plt import matplotlib @@ -268,7 +268,7 @@ def report(timesort=False): matplotlib.use('Qt5Agg') plt.ion() pipe.logtoconsole() - pipe.configure_pipeline(resource_filename('mkidpipeline', os.path.join('tests','h5speed_pipe.yml'))) + pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml'))) results = SpeedResults('/scratch/baileyji/mec/speedtest/recovery.pickle') phot_per_RrID = results.determine_mean_photperRID() @@ -365,8 +365,8 @@ def report(timesort=False): # from mkidpipeline.tests.h5speed import * pipe.logtoconsole(file='/scratch/baileyji/mec/speedtest/lastrun-pold.log') - pipe.configure_pipeline(resource_filename('mkidpipeline', os.path.join('tests','h5speed_pipe.yml'))) - d = pipe.load_data_description(resource_filename('mkidpipeline', os.path.join('tests','h5speed_data.yml'))) + pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml'))) + d = pipe.load_data_description(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_data.yml'))) # Basic checks print(numexpr.get_vml_version()) diff --git a/mkidpipeline/tests/output_validation.py b/mkidpipeline/tests/output_validation.py index 70a8afe9..061636b9 100644 --- a/mkidpipeline/tests/output_validation.py +++ b/mkidpipeline/tests/output_validation.py @@ -2,7 +2,7 @@ import mkidpipeline.config as config from astropy.io import fits import logging -import pkg_resources as pkg +from importlib import resources as rs import csv import astropy.units as u @@ -39,7 +39,7 @@ def fits_keys(o): :param o: :return: """ - with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f: + with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f: data = [row for row in csv.reader(f)] data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in data[1:]] diff --git a/mkidpipeline/tests/test_config.py b/mkidpipeline/tests/test_config.py index fefb583c..c7a1cd22 100644 --- a/mkidpipeline/tests/test_config.py +++ b/mkidpipeline/tests/test_config.py @@ -3,10 +3,10 @@ import mkidpipeline.config as config import mkidpipeline.pipeline as pipe import mkidcore.corelog -import pkg_resources as pkg +from importlib import resources as rs mkidcore.corelog.getLogger('mkidcore', setup=True, - configfile=pkg.resource_filename('mkidpipeline', './config/logging.yaml')) + configfile=rs.files('mkidpipeline').joinpath('./config/logging.yaml')) data = mkidpipeline.samples.get_sample_data('default') config.dump_dataconfig(data) diff --git a/mkidpipeline/tests/test_wavecal.py b/mkidpipeline/tests/test_wavecal.py index 7674fa08..907327c5 100644 --- a/mkidpipeline/tests/test_wavecal.py +++ b/mkidpipeline/tests/test_wavecal.py @@ -2,20 +2,20 @@ import mkidpipeline.definitions as definitions import mkidpipeline.config as config import mkidpipeline.pipeline as pipe -import pkg_resources as pkg +from importlib import resources as rs import mkidcore.corelog mkidcore.corelog.getLogger('mkidcore', setup=True, - configfile=pkg.resource_filename('mkidpipeline', './utils/logging.yaml')) + configfile=rs.files('mkidpipeline').joinpath('./utils/logging.yaml')) pipe.getLogger('mkidpipeline.steps.wavecal').setLevel('INFO') -config.configure_pipeline(pkg.resource_filename('mkidpipeline', 'pipe.yml')) +config.configure_pipeline(rs.files('mkidpipeline').joinpath('pipe.yml')) paths = dict(data='/darkdata/ScienceData/Subaru/', out='/work/tmp/tests/out', database='/work/tmp/tests/database', tmp='') config.update_paths(paths) config.make_paths() config.config.update('ncpu', 10) -dataset = definitions.MKIDObservingDataset(pkg.resource_filename('mkidpipeline', 'tests/test_data.yml')) +dataset = definitions.MKIDObservingDataset(rs.files('mkidpipeline').joinpath('tests/test_data.yml')) bin2hdf.buildtables(dataset.timeranges, remake=False) pipe.wavecal.fetch(dataset.wavecals, verbose=False) diff --git a/mkidpipeline/utils/output_validation.py b/mkidpipeline/utils/output_validation.py index e8252d14..b328ecb2 100644 --- a/mkidpipeline/utils/output_validation.py +++ b/mkidpipeline/utils/output_validation.py @@ -2,7 +2,7 @@ import mkidpipeline.config as config from astropy.io import fits from mkidcore.corelog import getLogger -import pkg_resources as pkg +from importlib import resources as rs import csv @@ -31,7 +31,7 @@ def fits_keys(o): :param o: :return: """ - with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f: + with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f: data = [row for row in csv.reader(f)] data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in data[1:]] From 11635e8a37f09b988fda4344edf5f8dab587d3f1 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Mon, 9 Sep 2024 15:09:36 -0700 Subject: [PATCH 11/13] Unfinished tqdm test and run_mkidpipeline.py file --- run_mkidpipeline.py | 14 ++++++++++++++ tests/test_tqdm.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 run_mkidpipeline.py create mode 100644 tests/test_tqdm.py diff --git a/run_mkidpipeline.py b/run_mkidpipeline.py new file mode 100644 index 00000000..de627d49 --- /dev/null +++ b/run_mkidpipeline.py @@ -0,0 +1,14 @@ + +# Run script for mkidpipeline +# Execute in command line with: python3 run_mkidpipeline.py +import subprocess +def run_mkidpipe(): + command = ['mkidpipe', '--make-dir', '--make-outputs'] + try: + # Run the command + subprocess.run(command, check=True) + print("Command executed successfully.") + except subprocess.CalledProcessError as e: + print(f"An error occurred while running the command: {e}") +if __name__ == '__main__': + run_mkidpipe() diff --git a/tests/test_tqdm.py b/tests/test_tqdm.py new file mode 100644 index 00000000..8c23f308 --- /dev/null +++ b/tests/test_tqdm.py @@ -0,0 +1,34 @@ +# Unit test for tqdm (used in lincal.py and wavecal.py) +# Note that I have seen the progress bar show up successfully, though only when manually changing verbose to verbose = True by default. + +import unittest +import os + +class TestTqdm(unittest.TestCase): + + def test_imports(self): + import mkidpipeline.steps.wavecal + import mkidpipeline.steps.lincal + import tqdm + + def test_progressbar(self): + # Test to see if progress bar shows up when passing data through + os.chdir(r'/home/dnearing/pipelineconfig/targets/HIP109427') + command1a = "echo 'BELOW IS THE PWD'" + command1b = 'pwd' + command1c = "echo 'ABOVE IS THE PWD'" + command2 = 'mkidpipe --make-outputs' + try: + # Run the command + os.system(command1a) + os.system(command1b) + os.system(command1c) + os.system(command2) + print("Command executed successfully.") + except: + print(f"An error occurred while running the command: {e}") + + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 723c00a4d348864e1b3a572d0e9d4e376e31e548 Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Tue, 10 Sep 2024 13:11:49 -0700 Subject: [PATCH 12/13] Modified test_tqdm.py --- tests/test_tqdm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_tqdm.py b/tests/test_tqdm.py index 8c23f308..b1468a2a 100644 --- a/tests/test_tqdm.py +++ b/tests/test_tqdm.py @@ -10,7 +10,8 @@ def test_imports(self): import mkidpipeline.steps.wavecal import mkidpipeline.steps.lincal import tqdm - + + ''' def test_progressbar(self): # Test to see if progress bar shows up when passing data through os.chdir(r'/home/dnearing/pipelineconfig/targets/HIP109427') @@ -26,8 +27,8 @@ def test_progressbar(self): os.system(command2) print("Command executed successfully.") except: - print(f"An error occurred while running the command: {e}") - + print(f"An error occurred while running the command. ") + ''' if __name__ == '__main__': From 196eec553afac7219522ce0ecdcb7a1b6c1b60dd Mon Sep 17 00:00:00 2001 From: Drew Nearing Date: Wed, 11 Sep 2024 13:08:28 -0700 Subject: [PATCH 13/13] Changed main.yaml in .github/workflows to have if statement under publish-package job. --- .github/workflows/main.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7e87a7d9..cc02403f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -46,6 +46,8 @@ jobs: name: upload release to PyPI runs-on: ubuntu-latest environment: publish-release + if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }} + permissions: # This permission is needed for private repositories. contents: read @@ -57,7 +59,6 @@ jobs: - uses: pdm-project/setup-pdm@v3 - name: Publish package distributions to PyPI - if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }} run: pdm publish required: # group all required workflows into one to avoid reconfiguring this in Actions settings