Add Met Office Global Deterministic height level dataset

datasets/ecmwf-forecast/Dockerfile

@@ -21,10 +21,10 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 # See https://github.com/mapbox/rasterio/issues/1289
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-# Install Python 3.10
-RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
-     && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \
-     && rm -rf "Mambaforge-$(uname)-$(uname -m).sh"
+# Install Python 3.11
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \
+     && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \
+     && rm -rf "Miniforge3-$(uname)-$(uname -m).sh"
 
 ENV PATH /opt/conda/bin:$PATH
 ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
@@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
 
 COPY pctasks/core /opt/src/pctasks/core
 RUN cd /opt/src/pctasks/core && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/cli /opt/src/pctasks/cli
 RUN cd /opt/src/pctasks/cli && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/task /opt/src/pctasks/task
 RUN cd /opt/src/pctasks/task && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/client /opt/src/pctasks/client
 RUN cd /opt/src/pctasks/client && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/ingest /opt/src/pctasks/ingest
 RUN cd /opt/src/pctasks/ingest && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/dataset /opt/src/pctasks/dataset
 RUN cd /opt/src/pctasks/dataset && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY ./datasets/ecmwf-forecast/requirements.txt /opt/src/datasets/ecmwf-forecast/requirements.txt
 RUN python3 -m pip install -r /opt/src/datasets/ecmwf-forecast/requirements.txt

datasets/ecmwf-forecast/dataset.yaml

@@ -1,8 +1,8 @@
 id: ecmwf_forecast
-image: ${{ args.registry }}/pctasks-ecmwf-forecast:2024.6.13.0
+image: ${{ args.registry }}/pctasks-ecmwf-forecast:2026.01.12
 
 args:
-- registry
+  - registry
 
 code:
   src: ${{ local.path(./ecmwf_forecast.py) }}

datasets/gbif/Dockerfile

@@ -22,14 +22,14 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
 # Install Python 3.8
-RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
-    && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \
-    && rm -rf "Mambaforge-$(uname)-$(uname -m).sh"
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \
+    && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \
+    && rm -rf "Miniforge3-$(uname)-$(uname -m).sh"
 
 ENV PATH /opt/conda/bin:$PATH
 ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
 
-RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5
+RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy
 
 RUN python -m pip install --upgrade pip
 
@@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
 
 COPY pctasks/core /opt/src/pctasks/core
 RUN cd /opt/src/pctasks/core && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/cli /opt/src/pctasks/cli
 RUN cd /opt/src/pctasks/cli && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/task /opt/src/pctasks/task
 RUN cd /opt/src/pctasks/task && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/client /opt/src/pctasks/client
 RUN cd /opt/src/pctasks/client && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/ingest /opt/src/pctasks/ingest
 RUN cd /opt/src/pctasks/ingest && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/dataset /opt/src/pctasks/dataset
 RUN cd /opt/src/pctasks/dataset && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY ./datasets/gbif/requirements.txt /opt/src/datasets/gbif/requirements.txt
 RUN python3 -m pip install -r /opt/src/datasets/gbif/requirements.txt

datasets/gbif/dataset.yaml

@@ -1,8 +1,8 @@
 id: gbif
-image: ${{ args.registry }}/pctasks-gbif:20230607.1
+image: ${{ args.registry }}/pctasks-gbif:2026.01.12
 
 args:
-- registry
+  - registry
 
 code:
   src: ${{ local.path(./gbif.py) }}
@@ -24,4 +24,4 @@ collections:
             min_depth: 2
             max_depth: 2
     chunk_storage:
-      uri: blob://ai4edataeuwest/gbif-etl-data/pctasks-chunks
+      uri: blob://ai4edataeuwest/gbif-etl-data/pctasks-chunks

datasets/goes/goes-cmi/Dockerfile

@@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
 
 COPY pctasks/core /opt/src/pctasks/core
 RUN cd /opt/src/pctasks/core && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/cli /opt/src/pctasks/cli
 RUN cd /opt/src/pctasks/cli && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/task /opt/src/pctasks/task
 RUN cd /opt/src/pctasks/task && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/client /opt/src/pctasks/client
 RUN cd /opt/src/pctasks/client && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/ingest /opt/src/pctasks/ingest
 RUN cd /opt/src/pctasks/ingest && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY pctasks/dataset /opt/src/pctasks/dataset
 RUN cd /opt/src/pctasks/dataset && \
-    pip install .
+    pip install -r requirements.txt && \
+    pip install --no-deps .
 
 COPY datasets/goes/goes-cmi/requirements.txt /opt/src/datasets/goes/goes-cmi/requirements.txt
 RUN python3 -m pip install -r /opt/src/datasets/goes/goes-cmi/requirements.txt

datasets/goes/goes-cmi/dataset.yaml

@@ -1,9 +1,9 @@
 id: goes_cmi
-image: ${{ args.registry }}/pctasks-goes-cmi:2025.04.15.1
+image: ${{ args.registry }}/pctasks-goes-cmi:2026.01.12
 
 args:
-- registry
-- year-prefix
+  - registry
+  - year-prefix
 
 code:
   src: ${{ local.path(./goes_cmi) }}
@@ -19,20 +19,20 @@ collections:
     asset_storage:
       # The blob storage pattern is
       # | noaa-goes16
-      # |  ABI-L2-MCMIPC 
+      # |  ABI-L2-MCMIPC
       # |    2023
       # |      170    # day of year
       # |        23   # hour
       # |          OR_ABI-L2-MCMIPC-M6_G16_s20231702301183_e20231702303555_c20231702304136.nc
       - uri: blob://goeseuwest/noaa-goes16/
         chunks:
           splits:
-            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }}  # CONUS
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }}  # Mesoscale
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }}  # Full Disk
-              depth: 3  # daily
+            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk
+              depth: 3 # daily
           options:
             ends_with: .nc
       # # GOES-17 is parked
@@ -50,25 +50,25 @@ collections:
       - uri: blob://goeseuwest/noaa-goes18/
         chunks:
           splits:
-            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }}  # CONUS
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }}  # Mesoscale
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }}  # Full Disk
-              depth: 3  # daily
+            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk
+              depth: 3 # daily
           options:
             ends_with: .nc
 
       - uri: blob://goeseuwest/noaa-goes19/
         chunks:
           splits:
-            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }}  # CONUS
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }}  # Mesoscale
-              depth: 3  # daily
-            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }}  # Full Disk
-              depth: 3  # daily
+            - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale
+              depth: 3 # daily
+            - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk
+              depth: 3 # daily
           options:
             ends_with: .nc
     chunk_storage:
-      uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/cmi/
+      uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/cmi/

datasets/goes/goes-glm/Dockerfile

@@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt
 
 COPY pctasks/core /opt/src/pctasks/core
 RUN cd /opt/src/pctasks/core && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/cli /opt/src/pctasks/cli
 RUN cd /opt/src/pctasks/cli && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/task /opt/src/pctasks/task
 RUN cd /opt/src/pctasks/task && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/client /opt/src/pctasks/client
 RUN cd /opt/src/pctasks/client && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/ingest /opt/src/pctasks/ingest
 RUN cd /opt/src/pctasks/ingest && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY pctasks/dataset /opt/src/pctasks/dataset
 RUN cd /opt/src/pctasks/dataset && \
-     pip install .
+     pip install -r requirements.txt && \
+     pip install --no-deps .
 
 COPY datasets/goes/goes-glm /opt/src/datasets/goes-glm
 RUN python3 -m pip install -r /opt/src/datasets/goes-glm/requirements.txt

datasets/goes/goes-glm/dataset.yaml

@@ -1,9 +1,9 @@
 id: goes_glm
-image: ${{ args.registry }}/pctasks-goes-glm:2025.04.16.0
+image: ${{ args.registry }}/pctasks-goes-glm:2026.01.12
 
 args:
-- registry
-- year-prefix
+  - registry
+  - year-prefix
 
 code:
   src: ${{ local.path(./goes_glm.py) }}
@@ -60,4 +60,4 @@ collections:
               depth: 2
 
     chunk_storage:
-      uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/glm/
+      uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/glm/

datasets/hls2/dataset.yaml

@@ -1,5 +1,5 @@
 id: hls2
-image: ${{ args.registry }}/pctasks-task-base:2025.4.8.2
+image: ${{ args.registry }}/pctasks-task-base:2026.01.12
 
 args:
   - registry
@@ -42,7 +42,6 @@ collections:
             chunk_length: 20000
     chunk_storage:
       uri: blob://hls2euwest/hls2-l30-info/pctasks-chunks/
-
 # The blob storage pattern is:
 # | container
 # |   S30/ or L30/ depending on collection
@@ -57,4 +56,4 @@ collections:
 # |                   HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif
 # |                   HLS.S30.T56PPQ.2024005T001421.v2.0.B02.tif
 # |                   ...
-# | eg S30/56/P/PQ/2024/01/05/HLS.S30.T56PPQ.2024005T001421.v2.0/HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif
+# | eg S30/56/P/PQ/2024/01/05/HLS.S30.T56PPQ.2024005T001421.v2.0/HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif