Replaced progressbar with tqdm + Replaced pkg_resources with importlib.resources + Edit to .github/workflows/main.yaml

.github/workflows/main.yaml

@@ -46,6 +46,8 @@ jobs:
     name: upload release to PyPI
     runs-on: ubuntu-latest
     environment: publish-release
+    if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }}
+
     permissions:
       # This permission is needed for private repositories.
       contents: read
@@ -57,7 +59,6 @@ jobs:
       - uses: pdm-project/setup-pdm@v3
 
       - name: Publish package distributions to PyPI
-        if: ${{ github.repository == 'MazinLab/MKIDPipeline' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && !contains(github.event.ref, 'dev') }}
         run: pdm publish
 
   required: # group all required workflows into one to avoid reconfiguring this in Actions settings

mkidpipeline/definitions.py

@@ -364,7 +364,7 @@ def _metadata(self):
         try:
             d.update(dict(E_CXREFX=self.wcscal.conex_ref[0], E_CXREFY=self.wcscal.conex_ref[1],
                           E_PREFX=self.wcscal.pixel_ref[0], E_PREFY=self.wcscal.pixel_ref[1],
-                          E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale else self.photontable.query_header(
+                          E_PLTSCL=self.wcscal.platescale if self.wcscal.platescale is not None else self.photontable.query_header(
                               'E_PLTSCL'),
                           E_DPDCX=self.wcscal.dp_dcx if self.wcscal.dp_dcx else self.photontable.query_header(
                               'E_DPDCX'),

mkidpipeline/mkidpipe.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
+from importlib import resources as rs
 import argparse
 import os
 import sys
-import pkg_resources as pkg
 from datetime import datetime
 
 import mkidpipeline.definitions as definitions
@@ -71,7 +71,7 @@ def parser():
         dest="logcfg",
         help="Run the pipeline on the outputs",
         type=str,
-        default=pkg.resource_filename("mkidpipeline", "./config/logging.yaml"),
+        default=rs.files('mkidpipeline').joinpath('./config/logging.yaml'),
     )
 
     return parser

mkidpipeline/samples.py

@@ -1,6 +1,5 @@
-import pkg_resources
+from importlib import resources as rs
 from collections import defaultdict
-
 import mkidpipeline.definitions as d
 import mkidpipeline.config as config
 
@@ -47,8 +46,7 @@ def _namer(name='Thing'):
     # Speccal
     d.MKIDSpeccal(name=_namer('speccal'),
                   data=d.MKIDObservation(name=_namer('star'), start=1602049166, duration=10, wavecal='wavecal0',
-                                         spectrum=pkg_resources.resource_filename('mkidpipeline',
-                                                                                  'data/sample_spectrum.txt')),
+                                         spectrum=rs.files('mkidpipeline').joinpath('data/sample_spectrum.txt')),
                   aperture=('15h22m32.3', '30.32 deg', '200 mas')),
 
     # WCS cal
@@ -68,7 +66,7 @@ def _namer(name='Thing'):
                  header=dict(OBJECT="HIP 109427"),
                  flatcal='flatcal0', speccal='', use='0,2,4-9', wcscal='wcscal0'),
     d.MKIDDither(name=_namer('dither'),
-                 data=pkg_resources.resource_filename('mkidpipeline', 'data/dither_sample.log'),
+                 data=rs.files('mkidpipeline').joinpath('data/dither_sample.log'),
                  wavecal='wavecal0', flatcal='flatcal0', speccal='', use=(1,),
                  wcscal='wcscal0'),
     d.MKIDDither(name=_namer('dither'), flatcal='', speccal='', wcscal='', wavecal='',
@@ -140,4 +138,4 @@ def get_sample_output(dataset='default'):
     data.append(d.MKIDOutput(name=_namer('out'), data='dither0', min_wave='950 nm', max_wave='1375 nm',
                              kind='movie', duration=10.0, movie_format='mp4', movie_runtime=3,
                              movie_type='both'))
-    return data
+    return data

mkidpipeline/steps/lincal.py

@@ -9,7 +9,7 @@
 """
 import time
 import numpy as np
-from progressbar import ProgressBar, NullBar
+import tqdm  # Got rid of progressbar import in favor of this.
 
 import mkidpipeline.definitions as definitions
 from mkidcore.corelog import getLogger
@@ -62,15 +62,18 @@ def apply(o: definitions.MKIDTimerange, config=None):
     n_to_do = np.count_nonzero(~of.flagged(PROBLEM_FLAGS, all_flags=False))
     lastpct = 0
 
+
     if cfg.get('lincal.ncpu') > 1:
-        bar = NullBar()
-    else:
-        bar = ProgressBar(max_value=n_to_do)
+        disable_bar = True
+        getLogger(__name__).info('\nNo progress bar displayed due to multiple cpus used for lincal.\n')
 
+    else:
+        disable_bar = False
+
 
     #Not ram intensive ~250MB peak
 
-    for done, resid in bar(enumerate(of.resonators(exclude=PROBLEM_FLAGS))):
+    for done, resid in tqdm.tqdm(enumerate(of.resonators(exclude=PROBLEM_FLAGS)), disable = disable_bar):  # Using tqdm here for a progress bar.
         indices = of.photonTable.get_where_list('resID==resid')
         if not indices.size:
             continue

mkidpipeline/steps/wavecal.py

@@ -5,8 +5,8 @@
 import numpy as np
 import multiprocessing as mp
 import time
+import tqdm
 import shutil
-import progressbar as pb
 import scipy
 
 import astropy.constants
@@ -181,7 +181,6 @@ def __init__(self, configuration, solution_name='solution.npz', _shared_tables=N
         # define attributes
         self.solution_name = solution_name  # solution name for saving
         self.progress = None  # will be replaced with progress bar
-        self.progress_iteration = None  # will be replaced with progress bar counter
         self._acquired = 0  # counter for number of pixel data sets acquired
         self._max_queue_size = 300  # max queue size for _parallel() method
         self._shared_tables = _shared_tables  # shared photon tables
@@ -891,20 +890,13 @@ def _assign_best_calibration_model(self, tried_models, pixel):
     def _update_progress(self, number=None, initialize=False, finish=False, verbose=True):
         if verbose:
             if initialize:
-                percentage = pb.Percentage()
-                bar = pb.Bar()
-                timer = pb.Timer()
-                eta = pb.ETA()
-                self.progress = pb.ProgressBar(widgets=[percentage, bar, '  (', timer, ') ', eta, ' '],
-                                               max_value=number).start()
-                self.progress_iteration = -1
+                self.progress = tqdm.tqdm(total = number)
             elif finish:
-                self.progress_iteration += 1
-                self.progress.update(self.progress_iteration)
-                self.progress.finish()
+                # Note that the updating convention has changed slightly. 1 is now added to the progress bar counter prior to each operation in the iterable rather than after each item finishes. I.e. the progress bar at "3" means that we are currently working on the 3rd item.
+                self.progress.update() 
+                self.progress.close()
             else:
-                self.progress_iteration += 1
-                self.progress.update(self.progress_iteration)
+                self.progress.update()
 
     def _setup(self, pixels, wavelengths):
         # check inputs

mkidpipeline/tests/h5speed.py

@@ -6,7 +6,7 @@
 import numpy as np
 import tables
 import time, timeit, copy, pickle, hashlib, re, collections, ast
-from pkg_resources import resource_filename
+from importlib import resources as rs
 import matplotlib.pyplot as plt
 import matplotlib
 
@@ -268,7 +268,7 @@ def report(timesort=False):
     matplotlib.use('Qt5Agg')
     plt.ion()
     pipe.logtoconsole()
-    pipe.configure_pipeline(resource_filename('mkidpipeline',  os.path.join('tests','h5speed_pipe.yml')))
+    pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml')))
 
     results = SpeedResults('/scratch/baileyji/mec/speedtest/recovery.pickle')
     phot_per_RrID = results.determine_mean_photperRID()
@@ -365,8 +365,8 @@ def report(timesort=False):
 
     # from mkidpipeline.tests.h5speed import *
     pipe.logtoconsole(file='/scratch/baileyji/mec/speedtest/lastrun-pold.log')
-    pipe.configure_pipeline(resource_filename('mkidpipeline',  os.path.join('tests','h5speed_pipe.yml')))
-    d = pipe.load_data_description(resource_filename('mkidpipeline',  os.path.join('tests','h5speed_data.yml')))
+    pipe.configure_pipeline(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_pipe.yml')))
+    d = pipe.load_data_description(rs.files('mkidpipeline').joinpath(os.path.join('tests', 'h5speed_data.yml')))
 
     # Basic checks
     print(numexpr.get_vml_version())

mkidpipeline/tests/output_validation.py

@@ -2,7 +2,7 @@
 import mkidpipeline.config as config
 from astropy.io import fits
 import logging
-import pkg_resources as pkg
+from importlib import resources as rs
 import csv
 import astropy.units as u
 
@@ -39,7 +39,7 @@ def fits_keys(o):
     :param o:
     :return:
     """
-    with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f:
+    with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f:
         data = [row for row in csv.reader(f)]
     data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in
             data[1:]]

mkidpipeline/tests/test_config.py

@@ -3,10 +3,10 @@
 import mkidpipeline.config as config
 import mkidpipeline.pipeline as pipe
 import mkidcore.corelog
-import pkg_resources as pkg
+from importlib import resources as rs
 
 mkidcore.corelog.getLogger('mkidcore', setup=True,
-                           configfile=pkg.resource_filename('mkidpipeline', './config/logging.yaml'))
+                           configfile=rs.files('mkidpipeline').joinpath('./config/logging.yaml'))
 
 data = mkidpipeline.samples.get_sample_data('default')
 config.dump_dataconfig(data)

mkidpipeline/tests/test_wavecal.py

@@ -2,20 +2,20 @@
 import mkidpipeline.definitions as definitions
 import mkidpipeline.config as config
 import mkidpipeline.pipeline as pipe
-import pkg_resources as pkg
+from importlib import resources as rs
 import mkidcore.corelog
 
 mkidcore.corelog.getLogger('mkidcore', setup=True,
-                           configfile=pkg.resource_filename('mkidpipeline', './utils/logging.yaml'))
+                           configfile=rs.files('mkidpipeline').joinpath('./utils/logging.yaml'))
 pipe.getLogger('mkidpipeline.steps.wavecal').setLevel('INFO')
 
-config.configure_pipeline(pkg.resource_filename('mkidpipeline', 'pipe.yml'))
+config.configure_pipeline(rs.files('mkidpipeline').joinpath('pipe.yml'))
 paths = dict(data='/darkdata/ScienceData/Subaru/', out='/work/tmp/tests/out',
              database='/work/tmp/tests/database', tmp='')
 config.update_paths(paths)
 config.make_paths()
 config.config.update('ncpu', 10)
-dataset = definitions.MKIDObservingDataset(pkg.resource_filename('mkidpipeline', 'tests/test_data.yml'))
+dataset = definitions.MKIDObservingDataset(rs.files('mkidpipeline').joinpath('tests/test_data.yml'))
 
 bin2hdf.buildtables(dataset.timeranges, remake=False)
 pipe.wavecal.fetch(dataset.wavecals, verbose=False)

mkidpipeline/utils/output_validation.py

@@ -2,7 +2,7 @@
 import mkidpipeline.config as config
 from astropy.io import fits
 from mkidcore.corelog import getLogger
-import pkg_resources as pkg
+from importlib import resources as rs
 import csv
 
 
@@ -31,7 +31,7 @@ def fits_keys(o):
     :param o:
     :return:
     """
-    with open(pkg.resource_filename('mkidcore', 'mec_keys.csv')) as f:
+    with open(rs.files('mkidcore').joinpath('mec_keys.csv')) as f:
         data = [row for row in csv.reader(f)]
     data = [{k.strip().lower().replace(' ', '_').replace('?', ''): v.strip() for k, v in zip(data[0], l)} for l in
             data[1:]]

pyproject.toml

@@ -16,24 +16,20 @@ requires-python = "~=3.10"
 dependencies = [
   "mkidcore>=1.9,<2.0",
   "setuptools>=72.2.0",
-
   "numpy>=1.26",
   "mpmath>=1.0.0",
   "numexpr>=2.0.0",
   "lmfit>=1.3.0",
   "SharedArray>=3.0.0",
-
   "astropy>=5.0",
   "astroquery>=0.4.7",
   "specutils>=1.0",
   "photutils>=1.13",
   "scikit-image>=0.20.0",
   "drizzle>=1.0",
-
   "matplotlib>=3.5",
   "cycler>=0.11",
-  "progressbar2>=4.0.0",
-
+  "tqdm>=4.66.5",  
   "psutil>=5.0.0",
   "tables>=3.8.0",
   "ruamel.yaml>0.18",

run_mkidpipeline.py

@@ -0,0 +1,14 @@
+
+# Run script for mkidpipeline
+# Execute in command line with: python3 run_mkidpipeline.py
+import subprocess
+def run_mkidpipe():
+    command = ['mkidpipe', '--make-dir', '--make-outputs']
+    try:
+        # Run the command
+        subprocess.run(command, check=True)
+        print("Command executed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred while running the command: {e}")
+if __name__ == '__main__':
+    run_mkidpipe()

tests/test_tqdm.py

@@ -0,0 +1,35 @@
+# Unit test for tqdm (used in lincal.py and wavecal.py)
+# Note that I have seen the progress bar show up successfully, though only when manually changing verbose to verbose = True by default.
+
+import unittest
+import os
+
+class TestTqdm(unittest.TestCase):
+
+    def test_imports(self):
+        import mkidpipeline.steps.wavecal
+        import mkidpipeline.steps.lincal
+        import tqdm
+
+    '''
+    def test_progressbar(self):
+        # Test to see if progress bar shows up when passing data through
+        os.chdir(r'/home/dnearing/pipelineconfig/targets/HIP109427')
+        command1a = "echo 'BELOW IS THE PWD'"
+        command1b = 'pwd'
+        command1c = "echo 'ABOVE IS THE PWD'"
+        command2 = 'mkidpipe --make-outputs'
+        try:
+        # Run the command
+            os.system(command1a)
+            os.system(command1b)
+            os.system(command1c)
+            os.system(command2)
+            print("Command executed successfully.")
+        except:
+            print(f"An error occurred while running the command. ")
+    '''
+
+
+if __name__ == '__main__':
+    unittest.main()