diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7e87a7d9..cc02403f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -46,6 +46,8 @@ jobs: name: upload release to PyPI runs-on: ubuntu-latest environment: publish-release + if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }} + permissions: # This permission is needed for private repositories. contents: read @@ -57,7 +59,6 @@ jobs: - uses: pdm-project/setup-pdm@v3 - name: Publish package distributions to PyPI - if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }} run: pdm publish required: # group all required workflows into one to avoid reconfiguring this in Actions settings diff --git a/mkidpipeline/definitions.py b/mkidpipeline/definitions.py index 50d64b1e..21a81bf4 100644 --- a/mkidpipeline/definitions.py +++ b/mkidpipeline/definitions.py @@ -364,7 +364,7 @@ def _metadata(self): try: d.update(dict(E_CXREFX=self.wcscal.conex_ref[0], E_CXREFY=self.wcscal.conex_ref[1], E_PREFX=self.wcscal.pixel_ref[0], E_PREFY=self.wcscal.pixel_ref[1], - E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale else self.photontable.query_header( + E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale is not None else self.photontable.query_header( 'E_PLTSCL'), E_DPDCX=self.wcscal.dp_dcx if self.wcscal.dp_dcx else self.photontable.query_header( 'E_DPDCX'), diff --git a/mkidpipeline/mkidpipe.py b/mkidpipeline/mkidpipe.py index 0555002c..76f45f58 100644 --- a/mkidpipeline/mkidpipe.py +++ b/mkidpipeline/mkidpipe.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 +from importlib import resources as rs import argparse import os import sys -import pkg_resources as pkg from datetime import datetime import mkidpipeline.definitions as definitions @@ -71,7 +71,7 @@ def parser(): dest="logcfg", help="Run the pipeline on the outputs", type=str, - default=pkg.resource_filename("mkidpipeline", "./config/logging.yaml"), + default=rs.files('mkidpipeline').joinpath('./config/logging.yaml'), ) return parser diff --git a/mkidpipeline/samples.py b/mkidpipeline/samples.py index 377bf69e..cc9666a8 100644 --- a/mkidpipeline/samples.py +++ b/mkidpipeline/samples.py @@ -1,6 +1,5 @@ -import pkg_resources +from importlib import resources as rs from collections import defaultdict - import mkidpipeline.definitions as d import mkidpipeline.config as config @@ -47,8 +46,7 @@ def _namer(name='Thing'): # Speccal d.MKIDSpeccal(name=_namer('speccal'), data=d.MKIDObservation(name=_namer('star'), start=1602049166, duration=10, wavecal='wavecal0', - spectrum=pkg_resources.resource_filename('mkidpipeline', - 'data/sample_spectrum.txt')), + spectrum=rs.files('mkidpipeline').joinpath('data/sample_spectrum.txt')), aperture=('15h22m32.3', '30.32 deg', '200 mas')), # WCS cal @@ -68,7 +66,7 @@ def _namer(name='Thing'): header=dict(OBJECT="HIP 109427"), flatcal='flatcal0', speccal='', use='0,2,4-9', wcscal='wcscal0'), d.MKIDDither(name=_namer('dither'), - data=pkg_resources.resource_filename('mkidpipeline', 'data/dither_sample.log'), + data=rs.files('mkidpipeline').joinpath('data/dither_sample.log'), wavecal='wavecal0', flatcal='flatcal0', speccal='', use=(1,), wcscal='wcscal0'), d.MKIDDither(name=_namer('dither'), flatcal='', speccal='', wcscal='', wavecal='', @@ -140,4 +138,4 @@ def get_sample_output(dataset='default'): data.append(d.MKIDOutput(name=_namer('out'), data='dither0', min_wave='950 nm', max_wave='1375 nm', kind='movie', duration=10.0, movie_format='mp4', movie_runtime=3, movie_type='both')) - return data + return data \ No newline at end of file diff --git a/mkidpipeline/steps/lincal.py b/mkidpipeline/steps/lincal.py index 275b97e0..6b0f0b0b 100644 --- a/mkidpipeline/steps/lincal.py +++ b/mkidpipeline/steps/lincal.py @@ -9,7 +9,7 @@ """ import time import numpy as np -from progressbar import ProgressBar, NullBar +import tqdm # Got rid of progressbar import in favor of this. import mkidpipeline.definitions as definitions from mkidcore.corelog import getLogger @@ -62,15 +62,18 @@ def apply(o: definitions.MKIDTimerange, config=None): n_to_do = np.count_nonzero(~of.flagged(PROBLEM_FLAGS, all_flags=False)) lastpct = 0 + if cfg.get('lincal.ncpu') > 1: - bar = NullBar() - else: - bar = ProgressBar(max_value=n_to_do) + disable_bar = True + getLogger(__name__).info('\nNo progress bar displayed due to multiple cpus used for lincal.\n') + else: + disable_bar = False + #Not ram intensive ~250MB peak - for done, resid in bar(enumerate(of.resonators(exclude=PROBLEM_FLAGS))): + for done, resid in tqdm.tqdm(enumerate(of.resonators(exclude=PROBLEM_FLAGS)), disable = disable_bar): # Using tqdm here for a progress bar. indices = of.photonTable.get_where_list('resID==resid') if not indices.size: continue diff --git a/mkidpipeline/steps/wavecal.py b/mkidpipeline/steps/wavecal.py index 72691542..58675ede 100644 --- a/mkidpipeline/steps/wavecal.py +++ b/mkidpipeline/steps/wavecal.py @@ -5,8 +5,8 @@ import numpy as np import multiprocessing as mp import time +import tqdm import shutil -import progressbar as pb import scipy import astropy.constants @@ -181,7 +181,6 @@ def __init__(self, configuration, solution_name='solution.npz', _shared_tables=N # define attributes self.solution_name = solution_name # solution name for saving self.progress = None # will be replaced with progress bar - self.progress_iteration = None # will be replaced with progress bar counter self._acquired = 0 # counter for number of pixel data sets acquired self._max_queue_size = 300 # max queue size for _parallel() method self._shared_tables = _shared_tables # shared photon tables @@ -891,20 +890,13 @@ def _assign_best_calibration_model(self, tried_models, pixel): def _update_progress(self, number=None, initialize=False, finish=False, verbose=True): if verbose: if initialize: - percentage = pb.Percentage() - bar = pb.Bar() - timer = pb.Timer() - eta = pb.ETA() - self.progress = pb.ProgressBar(widgets=[percentage, bar, ' (', timer, ') ', eta, ' '], - max_value=number).start() - self.progress_iteration = -1 + self.progress = tqdm.tqdm(total = number) elif finish: - self.progress_iteration += 1 - self.progress.update(self.progress_iteration) - self.progress.finish() + # Note that the updating convention has changed slightly. 1 is now added to the progress bar counter prior to each operation in the iterable rather than after each item finishes. I.e. the progress bar at "3" means that we are currently working on the 3rd item. + self.progress.update() + self.progress.close() else: - self.progress_iteration += 1 - self.progress.update(self.progress_iteration) + self.progress.update() def _setup(self, pixels, wavelengths): # check inputs diff --git a/mkidpipeline/tests/h5speed.py b/mkidpipeline/tests/h5speed.py index 343eb266..5b488f92 100644 --- a/mkidpipeline/tests/h5speed.py +++ b/mkidpipeline/tests/h5speed.py @@ -6,7 +6,7 @@ import numpy as np import tables import time, timeit, copy, pickle, hashlib, re, collections, ast -from pkg_resources import resource_filename +from importlib import resources as rs import matplotlib.pyplot as plt import matplotlib @@ -268,7 +268,7 @@ def report(timesort=False): matplotlib.use('Qt5Agg') plt.ion() pipe.logtoconsole() - pipe.configure_pipeline(resource_filename('mkidpipeline', os.path.join('tests','h5speed_pipe.yml'))) + pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml'))) results = SpeedResults('/scratch/baileyji/mec/speedtest/recovery.pickle') phot_per_RrID = results.determine_mean_photperRID() @@ -365,8 +365,8 @@ def report(timesort=False): # from mkidpipeline.tests.h5speed import * pipe.logtoconsole(file='/scratch/baileyji/mec/speedtest/lastrun-pold.log') - pipe.configure_pipeline(resource_filename('mkidpipeline', os.path.join('tests','h5speed_pipe.yml'))) - d = pipe.load_data_description(resource_filename('mkidpipeline', os.path.join('tests','h5speed_data.yml'))) + pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml'))) + d = pipe.load_data_description(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_data.yml'))) # Basic checks print(numexpr.get_vml_version()) diff --git a/mkidpipeline/tests/output_validation.py b/mkidpipeline/tests/output_validation.py index 70a8afe9..061636b9 100644 --- a/mkidpipeline/tests/output_validation.py +++ b/mkidpipeline/tests/output_validation.py @@ -2,7 +2,7 @@ import mkidpipeline.config as config from astropy.io import fits import logging -import pkg_resources as pkg +from importlib import resources as rs import csv import astropy.units as u @@ -39,7 +39,7 @@ def fits_keys(o): :param o: :return: """ - with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f: + with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f: data = [row for row in csv.reader(f)] data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in data[1:]] diff --git a/mkidpipeline/tests/test_config.py b/mkidpipeline/tests/test_config.py index fefb583c..c7a1cd22 100644 --- a/mkidpipeline/tests/test_config.py +++ b/mkidpipeline/tests/test_config.py @@ -3,10 +3,10 @@ import mkidpipeline.config as config import mkidpipeline.pipeline as pipe import mkidcore.corelog -import pkg_resources as pkg +from importlib import resources as rs mkidcore.corelog.getLogger('mkidcore', setup=True, - configfile=pkg.resource_filename('mkidpipeline', './config/logging.yaml')) + configfile=rs.files('mkidpipeline').joinpath('./config/logging.yaml')) data = mkidpipeline.samples.get_sample_data('default') config.dump_dataconfig(data) diff --git a/mkidpipeline/tests/test_wavecal.py b/mkidpipeline/tests/test_wavecal.py index 7674fa08..907327c5 100644 --- a/mkidpipeline/tests/test_wavecal.py +++ b/mkidpipeline/tests/test_wavecal.py @@ -2,20 +2,20 @@ import mkidpipeline.definitions as definitions import mkidpipeline.config as config import mkidpipeline.pipeline as pipe -import pkg_resources as pkg +from importlib import resources as rs import mkidcore.corelog mkidcore.corelog.getLogger('mkidcore', setup=True, - configfile=pkg.resource_filename('mkidpipeline', './utils/logging.yaml')) + configfile=rs.files('mkidpipeline').joinpath('./utils/logging.yaml')) pipe.getLogger('mkidpipeline.steps.wavecal').setLevel('INFO') -config.configure_pipeline(pkg.resource_filename('mkidpipeline', 'pipe.yml')) +config.configure_pipeline(rs.files('mkidpipeline').joinpath('pipe.yml')) paths = dict(data='/darkdata/ScienceData/Subaru/', out='/work/tmp/tests/out', database='/work/tmp/tests/database', tmp='') config.update_paths(paths) config.make_paths() config.config.update('ncpu', 10) -dataset = definitions.MKIDObservingDataset(pkg.resource_filename('mkidpipeline', 'tests/test_data.yml')) +dataset = definitions.MKIDObservingDataset(rs.files('mkidpipeline').joinpath('tests/test_data.yml')) bin2hdf.buildtables(dataset.timeranges, remake=False) pipe.wavecal.fetch(dataset.wavecals, verbose=False) diff --git a/mkidpipeline/utils/output_validation.py b/mkidpipeline/utils/output_validation.py index e8252d14..b328ecb2 100644 --- a/mkidpipeline/utils/output_validation.py +++ b/mkidpipeline/utils/output_validation.py @@ -2,7 +2,7 @@ import mkidpipeline.config as config from astropy.io import fits from mkidcore.corelog import getLogger -import pkg_resources as pkg +from importlib import resources as rs import csv @@ -31,7 +31,7 @@ def fits_keys(o): :param o: :return: """ - with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f: + with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f: data = [row for row in csv.reader(f)] data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in data[1:]] diff --git a/pyproject.toml b/pyproject.toml index 70106faa..cfb5db06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,24 +16,20 @@ requires-python = "~=3.10" dependencies = [ "mkidcore>=1.9,<2.0", "setuptools>=72.2.0", - "numpy>=1.26", "mpmath>=1.0.0", "numexpr>=2.0.0", "lmfit>=1.3.0", "SharedArray>=3.0.0", - "astropy>=5.0", "astroquery>=0.4.7", "specutils>=1.0", "photutils>=1.13", "scikit-image>=0.20.0", "drizzle>=1.0", - "matplotlib>=3.5", "cycler>=0.11", - "progressbar2>=4.0.0", - + "tqdm>=4.66.5", "psutil>=5.0.0", "tables>=3.8.0", "ruamel.yaml>0.18", diff --git a/run_mkidpipeline.py b/run_mkidpipeline.py new file mode 100644 index 00000000..de627d49 --- /dev/null +++ b/run_mkidpipeline.py @@ -0,0 +1,14 @@ + +# Run script for mkidpipeline +# Execute in command line with: python3 run_mkidpipeline.py +import subprocess +def run_mkidpipe(): + command = ['mkidpipe', '--make-dir', '--make-outputs'] + try: + # Run the command + subprocess.run(command, check=True) + print("Command executed successfully.") + except subprocess.CalledProcessError as e: + print(f"An error occurred while running the command: {e}") +if __name__ == '__main__': + run_mkidpipe() diff --git a/tests/test_tqdm.py b/tests/test_tqdm.py new file mode 100644 index 00000000..b1468a2a --- /dev/null +++ b/tests/test_tqdm.py @@ -0,0 +1,35 @@ +# Unit test for tqdm (used in lincal.py and wavecal.py) +# Note that I have seen the progress bar show up successfully, though only when manually changing verbose to verbose = True by default. + +import unittest +import os + +class TestTqdm(unittest.TestCase): + + def test_imports(self): + import mkidpipeline.steps.wavecal + import mkidpipeline.steps.lincal + import tqdm + + ''' + def test_progressbar(self): + # Test to see if progress bar shows up when passing data through + os.chdir(r'/home/dnearing/pipelineconfig/targets/HIP109427') + command1a = "echo 'BELOW IS THE PWD'" + command1b = 'pwd' + command1c = "echo 'ABOVE IS THE PWD'" + command2 = 'mkidpipe --make-outputs' + try: + # Run the command + os.system(command1a) + os.system(command1b) + os.system(command1c) + os.system(command2) + print("Command executed successfully.") + except: + print(f"An error occurred while running the command. ") + ''' + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file